diff --git a/docs/AGENTS.md b/docs/AGENTS.md index 92916db93a..93bb81be8f 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -100,3 +100,18 @@ and `logo_dark`. Keep cache-removal documentation focused on migration impact. Omit asset-fetch implementation walkthroughs and alternative override instructions. + +## RST restructuring + +When moving a guide, keep its setup content until any deferred replacement pages +exist. Preserve old URLs with compatibility pages and retain explicit labels in +the destination. Recompute RST heading underlines when renaming their text. +Device metadata containing docnames must come from `app.srcdir` so a multiversion +build uses paths that exist in that ref, even when `conf.py` comes from the deploy +checkout. + +Reuse the README architecture image in the RST overview so both entry points +share one maintained diagram. + +Group vendor device guides under category toctrees in the Devices navigation; +folder placement alone does not create sidebar nesting. diff --git a/docs/index.html b/docs/index.html index 153ab66a68..06fa578d42 100644 --- a/docs/index.html +++ b/docs/index.html @@ -3,7 +3,7 @@ - Redirecting to the latest Isaac Teleop documentation + Redirecting to the latest Isaac Capture documentation diff --git a/docs/source/_data/devices.yaml b/docs/source/_data/devices.yaml index b057eb5838..ea66b51c6f 100644 --- a/docs/source/_data/devices.yaml +++ b/docs/source/_data/devices.yaml @@ -139,7 +139,7 @@ devices: details: setup: - label: Body Tracking guide - doc: /device/body_tracking + doc: /device/body_tracking/index - Isaac Teleop Web Client requirements: - >- @@ -192,7 +192,7 @@ devices: - label: MANUS Gloves Plugin url: https://github.com/NVIDIA/IsaacTeleop/tree/main/src/plugins/manus - label: MANUS Gloves docs - doc: /device/manus + doc: /device/hand_tracking/manus requirements: - MANUS SDK - USB receiver on the workstation @@ -216,7 +216,7 @@ devices: - label: Haptikos Plugin url: https://github.com/NVIDIA/IsaacTeleop/tree/main/src/plugins/haptikos - label: Haptikos docs - doc: /device/haptikos + doc: /device/hand_tracking/haptikos requirements: - Haptikos Robotics API acquire: @@ -238,7 +238,7 @@ devices: - label: OGLO Tactile Glove Plugin url: https://github.com/NVIDIA/IsaacTeleop/tree/main/src/plugins/oglo_tactile - label: OGLO Tactile Glove docs - doc: /device/oglo + doc: /device/tactile/oglo requirements: - Linux with BlueZ - One BLE device per glove @@ -257,7 +257,7 @@ devices: details: setup: - label: WUJI Glove docs - doc: /device/wuji_glove + doc: /device/hand_tracking/wuji_glove - label: WUJI Glove Plugin url: https://github.com/NVIDIA/IsaacTeleop/tree/main/src/plugins/wuji_glove - label: WUJI Hand retargeting @@ -309,7 +309,7 @@ devices: - label: SpaceMouse Plugin url: https://github.com/NVIDIA/IsaacTeleop/tree/main/src/plugins/spacemouse - label: SpaceMouse docs - doc: /device/spacemouse + doc: /device/spatial_controllers/spacemouse requirements: - Linux workstation with USB or the 3Dconnexion Universal Receiver - Plugin udev rule installed on the host @@ -344,7 +344,7 @@ devices: - label: OAK-D Camera Plugin url: https://github.com/NVIDIA/IsaacTeleop/tree/main/src/plugins/oak - label: OAK Camera docs - doc: /device/oak + doc: /device/cameras/oak requirements: - USB 3 connection to the workstation acquire: @@ -365,7 +365,7 @@ devices: details: setup: - label: LeRobot integration - doc: /getting_started/lerobot/index + doc: /integrations/lerobot/index - label: LeRobot on GitHub url: https://github.com/huggingface/lerobot diff --git a/docs/source/_ext/ecosystem_grid.py b/docs/source/_ext/ecosystem_grid.py index 346142e7ac..b3370a1533 100644 --- a/docs/source/_ext/ecosystem_grid.py +++ b/docs/source/_ext/ecosystem_grid.py @@ -34,10 +34,10 @@ # an earlier tag is listed from here rather than from a 0.x version. SINCE_FLOOR = "v1.0.191" -# A link into the docs site or into the IsaacTeleop repository stays inside this +# A link into the docs site or into the Isaac Capture repository stays inside this # project, so it carries no external marker -- arrow or screen-reader label. Every # other domain gets one, including other GitHub organizations such as isaac-sim. -PROJECT_REPO = "https://github.com/NVIDIA/IsaacTeleop" +PROJECT_REPO = "https://github.com/NVIDIA/IsaacCapture" # Group header rows within a table. The input wording is the old page's own table # titles, so a reader arriving from an old link recognizes them. @@ -407,11 +407,10 @@ def _load_devices(env, relpath: str = DEVICE_DATA) -> tuple[list[dict], list[dic dependency, an incremental build reuses the cached doctree and renders the previous table after the data changes. """ - confdir = env.app.confdir - env.note_dependency(os.path.join(confdir, relpath)) - return _read( - confdir, relpath, lambda data: _validate_devices(data, relpath, confdir) - ) + # Device docnames must match the ref being built, not the deploy config's ref. + srcdir = env.app.srcdir + env.note_dependency(os.path.join(srcdir, relpath)) + return _read(srcdir, relpath, lambda data: _validate_devices(data, relpath, srcdir)) def _line(classes: list[str], text: str = "") -> nodes.paragraph: @@ -423,7 +422,9 @@ def _line(classes: list[str], text: str = "") -> nodes.paragraph: def _is_external(url: str) -> bool: - return not str(url).startswith(PROJECT_REPO) + return not str(url).startswith( + (PROJECT_REPO, "https://github.com/NVIDIA/IsaacTeleop") + ) def _mark_external(ref: nodes.Element, *, arrow: bool = True) -> None: diff --git a/docs/source/conf.py b/docs/source/conf.py index 8c0e78365a..ed6b522b97 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -13,7 +13,7 @@ # -- Project information ----------------------------------------------------- -project = "Isaac Teleop" +project = "Isaac Capture" build_time = datetime.now(timezone.utc) copyright = f"2025-{build_time.year}, NVIDIA CORPORATION & AFFILIATES" copyright += f", last updated on {build_time.strftime('%B %d, %Y')}" @@ -86,7 +86,7 @@ def _pip_pin(ref_name, fallback): # -- Options for HTML output --------------------------------------------------- -html_title = "Isaac Teleop Documentation" +html_title = "Isaac Capture Documentation" html_theme = "nvidia_sphinx_theme" html_favicon = "_static/favicon.ico" html_show_copyright = True @@ -101,7 +101,7 @@ def _pip_pin(ref_name, fallback): _DEFAULT_ICONS = { "teleop_version": "main", - "teleop_url": "https://github.com/NVIDIA/IsaacTeleop", + "teleop_url": "https://github.com/NVIDIA/IsaacCapture", "cloudxr_version": "6.2", "cloudxr_url": "https://docs.nvidia.com/cloudxr-sdk", "lab_version": "3.0", @@ -110,7 +110,7 @@ def _pip_pin(ref_name, fallback): _VERSION_ICON_MAP = { "release/1.0.x": { "teleop_version": "1.0", - "teleop_url": "https://github.com/NVIDIA/IsaacTeleop/tree/release/1.0.x", + "teleop_url": "https://github.com/NVIDIA/IsaacCapture/tree/release/1.0.x", "cloudxr_version": "6.1", "cloudxr_url": "https://docs.nvidia.com/cloudxr-sdk/release/6", "lab_version": "3.0", @@ -197,13 +197,13 @@ def _versioning_context(app, pagename, templatename, context, doctree): # Edit page button: link to GitHub so users can suggest edits (PyData theme uses html_context) html_context = { "github_user": "NVIDIA", - "github_repo": "IsaacTeleop", + "github_repo": "IsaacCapture", "github_version": _smv_name or "main", "doc_path": "docs/source", } # Base URL for linking to repository source (used by code-file and code-dir roles). -_GH_BASE = "https://github.com/NVIDIA/IsaacTeleop" +_GH_BASE = "https://github.com/NVIDIA/IsaacCapture" _GH_BRANCH = html_context["github_version"] diff --git a/docs/source/development/build_from_source/index.rst b/docs/source/development/build_from_source/index.rst new file mode 100644 index 0000000000..4d0ca8e286 --- /dev/null +++ b/docs/source/development/build_from_source/index.rst @@ -0,0 +1,458 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Build from Source +================= + +This page describes how to build Isaac Capture from source, including core libraries, plugins, and +examples. The instructions align with the project's CMake configuration and the CI workflow +(:code-file:`build-ubuntu.yml <.github/workflows/build-ubuntu.yml>` in the GitHub repository). + +.. contents:: Steps + :local: + :depth: 1 + +.. admonition:: Next Steps + + - To build and serve the **WebXR client** locally, see :doc:`webxr`. + +Prerequisites +------------- + +- **CMake** 3.24 or higher (Ubuntu 22.04's apt ``cmake`` is 3.22 — install a newer one + from `Kitware's APT repository `_ or with ``pip install cmake``) +- **C++20** compatible compiler +- **Python** 3.11, 3.12, or 3.13 (default 3.11; see ``ISAAC_TELEOP_PYTHON_VERSION`` in root ``CMakeLists.txt``) +- **uv** for Python dependency management and managed Python +- **Internet connection** for downloading dependencies via CMake FetchContent + +.. note:: + **Optional — only needed to build the Televiz visualization module,** ``BUILD_VIZ``. + ``BUILD_VIZ`` is auto-detected from the GPU stack: it defaults to ``ON`` when both of the + following are found at configure time and to ``OFF`` otherwise, so a core-only source + build still configures on a machine without them. Watch the + ``-- BUILD_VIZ: (Vulkan=... CUDAToolkit=...)`` configure line to see which one + is missing. + + - **Vulkan headers + loader** — ``libvulkan-dev`` on Linux, the LunarG SDK on Windows. + - **CUDA Toolkit** (cudart at link time) — ``nvidia-cuda-toolkit`` or the official NVIDIA + installer. + + ``BUILD_VIZ=ON`` additionally requires **glslangValidator** to compile shaders to SPIR-V + (``glslang-tools`` on Linux, ``brew install glslang`` on macOS; ships with the Vulkan SDK + on Windows), and on Linux **wayland-scanner** (``libwayland-dev``) for GLFW's Wayland + backend — pass ``-DGLFW_BUILD_WAYLAND=OFF`` to build Televiz against X11 only instead. + Neither gates the auto-default: if one is missing, the configure fails naming it rather + than quietly dropping the module. Most users do not need any of this: + ``pip install isaaccapture`` already ships the compiled ``isaaccapture.viz`` module. See + `Other Build options`_ for the full option table. + +.. _one-time-setup: + +One time setup +-------------- + +Install build tools and dependencies, such as CMake, clang-format. See :code-file:`build-ubuntu.yml <.github/workflows/build-ubuntu.yml>` in the GitHub repository for +the list of dependencies. On **Ubuntu**, install build tools and clang-format: + +.. code-block:: bash + + sudo apt-get update + sudo apt-get install -y build-essential cmake libegl-dev libx11-dev libwayland-dev clang-format-14 ccache patchelf pkg-config glslang-tools + +Runtime-only dependencies (needed to actually run teleop, not to build): + +.. code-block:: bash + + # adb — required for OOB teleop (``--setup-oob``) to talk to the headset over USB. + # coturn — required for USB-local mode (``--usb-local``); runs a local TURN server + # so WebRTC ICE can relay traffic from the headset to the CloudXR backend + # over the USB cable. + sudo apt-get install -y android-tools-adb coturn + +Our build system uses `uv`_ for Python version and dependency management. Install `uv`_ if not already installed: + +.. code-block:: bash + + curl -LsSf https://astral.sh/uv/install.sh | sh + +The installer drops ``uv`` in ``~/.local/bin``, which is not on ``PATH`` in most shells. Add it for +the current shell — and to your ``~/.bashrc`` to make it stick — or every ``uv`` command below +fails with ``uv: command not found``: + +.. code-block:: bash + + export PATH="$HOME/.local/bin:$PATH" + +.. note:: + While the build system uses `uv`_, the final Python packages can be installed via any Python package manager + such as `pip `_ or `conda `_. + +1. Clone the repository +----------------------- + +.. code-block:: bash + + git clone https://github.com/NVIDIA/IsaacCapture.git + cd IsaacCapture + +.. note:: + Dependencies (OpenXR SDK, pybind11, yaml-cpp) are automatically downloaded + during CMake configuration using FetchContent. No manual dependency installation or + git submodule initialization is required. + +Pre-download CloudXR SDK (Optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. note:: + + If you are using the default flow, skip this step. The + :code-file:`CMakeLists.txt ` + will automatically download the CloudXR SDK by calling the + :code-file:`download_cloudxr_runtime_sdk.sh ` + script. + +Sometimes NVIDIA might share early access CloudXR SDKs with you. In that case, you may get +tarballs such as: + +- ``CloudXR--Linux--sdk.tar.gz`` (CloudXR Runtime SDK) +- ``CloudXR-exp--Linux--sdk.tar.gz`` (experimental + runtime). Packaged by default as ``isaaccapture.cloudxr_exp``; needed for Jetson Orin + support (for example :doc:`/development/televiz`) until the default runtime covers + those platforms. +- ``nvidia-cloudxr-.tgz`` (CloudXR Web SDK) + +You can place them in the :code-dir:`deps/cloudxr/` directory and update the ``deps/cloudxr/.env`` +file to locally override the default version defined in :code-file:`deps/cloudxr/.env.default`, +like this: + +.. code-block:: bash + + CXR_RUNTIME_SDK_VERSION= + CXR_WEB_SDK_VERSION= + +The experimental runtime is packaged into the wheel as ``isaaccapture.cloudxr_exp`` by default +(``ENABLE_CLOUDXR_EXP_BUNDLE=ON``). Pass ``-DENABLE_CLOUDXR_EXP_BUNDLE=OFF`` to skip it. +Select it at runtime with ``ISAAC_TELEOP_CLOUDXR_EXP``. +See :ref:`dedicated-cloudxr-runtime`. + +2. CMake: Configure and build +----------------------------- + +From the project root: + +.. code-block:: bash + + cmake -B build # configure + cmake --build build --parallel # build + cmake --install build # install + +Add any other options as ``-D`` flags on the configure line, for example +``cmake -B build -DCMAKE_BUILD_TYPE=Debug``. + +.. important:: + + The Python version is baked into a build directory's CMake cache and its build + venv, so ``ISAAC_TELEOP_PYTHON_VERSION`` cannot be changed on an existing tree — + configuring again with a different value fails with an explanatory error. Select + it on the first configure, and delete the tree to switch later: + + .. code-block:: bash + + cmake -B build -DISAAC_TELEOP_PYTHON_VERSION=3.12 + cmake --build build --parallel + +This will: + +1. Fetch dependencies (OpenXR SDK, yaml-cpp, pybind11, FlatBuffers, MCAP, and optionally Catch2 for tests) via FetchContent in ``deps/third_party/CMakeLists.txt`` +2. Build core C++ libraries (schema, oxr_utils, plugin_manager, oxr, pusherio, deviceio, mcap, etc.) and Python bindings +3. Build the Python wheel +4. Build examples (if enabled) +5. Install to ``./install`` (default prefix set in root ``CMakeLists.txt``) + + +C++ Formatting Enforcement (Linux) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On Linux, **clang-format** is enforced by default; the build fails if formatting changes would be applied. The project +uses **clang-format-14** for consistent results across distributions (see ``cmake/ClangFormat.cmake``). + +To disable enforcement, set ``ENABLE_CLANG_FORMAT_CHECK`` to ``OFF``: + +.. code-block:: bash + + cmake -B build -DENABLE_CLANG_FORMAT_CHECK=OFF + +Useful targets: + +- ``clang_format_check`` — verifies formatting (part of ``ALL`` on Linux) +- ``clang_format_fix`` — applies formatting in place + +.. code-block:: bash + + cmake --build build --target clang_format_check + cmake --build build --target clang_format_fix + +Other Build options +~~~~~~~~~~~~~~~~~~~ + +The CMake options (defined in root :code-file:`CMakeLists.txt` and :code-file:`cmake/SetupPython.cmake`): + +.. list-table:: Common CMake Options + :widths: 20 36 44 + :header-rows: 1 + + * - Option + - CMake flag + - Default / Notes + * - **Build type** + - ``CMAKE_BUILD_TYPE`` + - ``Release`` or ``Debug`` + * - **Install prefix** + - ``CMAKE_INSTALL_PREFIX`` + - ``./install`` + +.. list-table:: Common Isaac Capture Options + :widths: 24 36 40 + :header-rows: 1 + + * - Option + - CMake flag + - Default / Notes + * - **Examples** + - ``BUILD_EXAMPLES`` + - ``ON`` + * - **Python bindings** + - ``BUILD_PYTHON_BINDINGS`` + - ``ON`` + * - **Python version** + - ``ISAAC_TELEOP_PYTHON_VERSION`` + - ``3.11`` (3.11, 3.12, or 3.13) + * - **Testing** + - ``BUILD_TESTING`` + - ``ON``; enables CTest and Catch2 + * - **Clang-format check** + - ``ENABLE_CLANG_FORMAT_CHECK`` + - ``ON`` on Linux + * - **Televiz visualization** + - ``BUILD_VIZ`` + - Auto: ``ON`` when Vulkan and the CUDA Toolkit are detected, else ``OFF``. Force with ``-DBUILD_VIZ=ON`` / ``-DBUILD_VIZ=OFF``. (Most users don't need this — ``pip install isaaccapture`` already ships the compiled ``isaaccapture.viz`` module.) + +.. list-table:: Plugin Specific Options + :widths: 26 34 40 + :header-rows: 1 + + * - Option + - CMake flag + - Default / Notes + * - **Plugins master switch** + - ``BUILD_PLUGINS`` + - ``ON`` + * - **OAK camera plugin** + - ``BUILD_PLUGIN_OAK_CAMERA`` + - ``OFF``; when ``ON``, builds DepthAI v3.x and pulls its dependencies through + vcpkg, so it also needs ``CMAKE_TOOLCHAIN_FILE`` on a fresh build directory. + See :doc:`/device/cameras/oak`. + * - **Teleop ROS2 example only** + - ``BUILD_EXAMPLE_TELEOP_ROS2`` + - ``OFF``; when ``ON``, only ``examples/teleop_ros2`` (e.g. Docker) + +Examples +~~~~~~~~ + +Build for a different Python version (``3.11``, ``3.12``, ``3.13`` are supported; +delete ``build/`` first if it is already configured for another one): + +.. code-block:: bash + + cmake -B build -DISAAC_TELEOP_PYTHON_VERSION=3.12 + cmake --build build --parallel + +Debug build: + +.. code-block:: bash + + cmake -B build -DCMAKE_BUILD_TYPE=Debug + cmake --build build + +Build without examples: + +.. code-block:: bash + + cmake -B build -DBUILD_EXAMPLES=OFF + cmake --build build + +Build without Python bindings: + +.. code-block:: bash + + cmake -B build -DBUILD_PYTHON_BINDINGS=OFF + cmake --build build + +Build with the OAK camera plugin. It needs the vcpkg toolchain, and CMake only +reads ``CMAKE_TOOLCHAIN_FILE`` on a build tree's **first** configure, so delete +``build/`` first if it already exists (see :doc:`/device/cameras/oak`): + +.. code-block:: bash + + rm -rf build + cmake -B build -DBUILD_PLUGIN_OAK_CAMERA=ON \ + -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake + cmake --build build --target camera_plugin_oak --parallel + +Build only the teleop_ros2 example (e.g. for Docker, as in :code-file:`build-ubuntu.yml <.github/workflows/build-ubuntu.yml>` teleop-ros2-docker job): + +.. code-block:: bash + + cmake -B build -DBUILD_EXAMPLES=OFF -DBUILD_EXAMPLE_TELEOP_ROS2=ON + cmake --build build + +Clean rebuild (``--fresh`` wipes the CMake cache and reconfigures): + +.. code-block:: bash + + cmake -B build --fresh + cmake --build build + +3. Running tests +---------------- + +When ``BUILD_TESTING`` is ``ON``, CTest is enabled at the top level. Run all tests either via the CMake ``test`` target or with ``ctest``: + +.. code-block:: bash + + cmake --build build --target test + + # Or with ctest (e.g. parallel, output on failure) + ctest --test-dir build --output-on-failure --parallel + +The CI uses ``ctest`` (see :code-file:`build-ubuntu.yml <.github/workflows/build-ubuntu.yml>`). + +4. Install the ``isaaccapture`` pip package +------------------------------------------- + +The wheels are built in the ``./install/wheels/`` directory. Install the package from the wheels. +Using ``pip``, you need to pass the ``--no-index`` option to automatically find the right wheel +based on the Python version. Note that ``pip`` and ``uv pip`` has slightly different options. + +.. code-block:: bash + + # Pass --no-index to use only wheels in ./install/wheels/; + # Pass --force-reinstall to replace an existing install. + pip install "isaaccapture[retargeters,cloudxr,ui]" --find-links=./install/wheels/ --no-index --force-reinstall + +With ``uv pip``, ``--find-links`` still consults the package index and installs the highest +version it finds, so a newer published ``isaaccapture`` there silently wins over your local build. +``--no-index`` is not a fix either: the wheel's dependencies are not in ``./install/wheels/``, so +the install fails. Install the local wheel by path instead. That always selects your build, while +the index stays available for its dependencies: + +.. code-block:: bash + + # Pick the wheel for your Python version (cp312 = Python 3.12; use cp311 or cp313 otherwise). + # -t sorts by modification time so head -1 selects the most recent build if older wheels remain. + WHEEL=$(ls -t ./install/wheels/isaaccapture-*-cp312-*.whl | head -1) + + # --reinstall-package replaces an already-installed isaaccapture, e.g. after a rebuild. + # --find-links exposes the sibling isaacteleop transition wheel built alongside isaaccapture. + uv pip install "${WHEEL}[retargeters,cloudxr,ui]" --find-links=./install/wheels/ --reinstall-package isaaccapture + +Check that the installed version ends in ``+local`` with ``uv pip list | grep isaaccapture``. + +The wheel carries the ``isaacteleop`` compatibility alias, which will be removed +in 1.9, so +``import isaacteleop`` resolves without installing anything else. See +:doc:`/development/migration`. + +Alternative: install directly from source with pip +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The repository also ships a root :code-file:`pyproject.toml` using the +`scikit-build-core `_ backend, which +drives the same top-level ``CMakeLists.txt``. This lets you build and install +with standard Python tooling, without running the ``cmake -B build`` steps +yourself; the classic flow above (which CI uses to produce ``install/wheels/`` and +the released wheels) is unchanged, so the two coexist. + +.. code-block:: bash + + pip install . # build + install the compiled wheel from source + pip install -e . # editable / developer install + +.. note:: + + The CMake build tree is kept under ``build-wheel//`` (e.g. + ``build-wheel/cpython-311/``) — one per Python version, so different + interpreters never share a configured CMake cache — instead of a temporary + directory. Re-installs are therefore incremental. It sits outside ``build/``, so + it never collides with a classic ``cmake -B build`` tree. Both are gitignored. + +What this path does and how it differs from the classic flow: + +- **Full build, no overrides.** The backend passes no ``cmake.define`` overrides, + so ``BUILD_EXAMPLES``, ``BUILD_TESTING``, and ``BUILD_PLUGINS`` stay at their + CMake defaults: a ``pip install`` builds the full default target — extensions, + examples, C++ tests, and plugins (a plugin without its SDK skips gracefully). It + installs only the ``isaacteleop_wheel`` and ``isaacteleop_binaries`` components, + so C++ library/header install rules do not leak into the wheel. +- **clang-format is enforced.** Because the gate is left on, the build **fails** if + ``clang-format-14`` is not installed or any C++ file is unformatted — a + ``pip install`` therefore requires ``clang-format-14`` on Linux. +- **All executables on PATH.** Every executable built from ``src/`` and + ``examples/`` — example programs, C++ test binaries, plugin tools + (``oxr_simple_api_demo``, ``se3_printer``, ``schema_tests``, …) — is installed + into the venv's ``bin/`` (the wheel's scripts scheme; see the executable-install + loop in the root ``CMakeLists.txt``), so they are on ``PATH`` once the venv is + active. They statically link the teleop core libraries, so they are + self-contained. Shared libraries (Python extension modules, plugin ``.so``) are + not executables and are **not** placed in ``bin/``. +- **ABI.** Extensions are compiled against the interpreter that runs the build + (so the wheel's ABI tag matches) and against NumPy 2.x, so a single wheel works + with both NumPy 1.x and 2.x at runtime. +- **No type stubs.** The pip-built wheel omits the ``.pyi`` stubs that the classic + and released wheels ship (stub generation shells out to ``uv``, which is not + guaranteed inside pip's build isolation). Imports and runtime behavior are + unaffected. +- **Version.** The pip-facing version is derived from the ``VERSION`` file as + ``MAJOR.MINOR+local`` — the same value the classic flow produces for a + non-CI/local build. The pip path is a from-source/dev build, so it is always + tagged ``+local``; the git-aware release versioning (tags, ``a1`` / ``rc1`` / + ``.devN`` labels) stays with the classic flow via + :code-file:`cmake/IsaacTeleopVersion.cmake`. + +.. admonition:: ``import isaacteleop`` under this path + + Both ``pip install .`` and ``pip install -e .`` serve the alias: the shim's + source (:code-file:`src/compat/isaacteleop.py`) is mapped beside ``isaaccapture``'s + and ships in the same wheel (:doc:`/development/migration`). + +.. admonition:: Editable installs and iterating on pure-Python subpackages + + An editable install (``pip install -e .``) never recompiles on import. Pure-Python + subpackages resolve straight to ``src/python/isaaccapture/``, so edits **take effect + live** — a fresh interpreter is enough. + + Compiled extensions are the exception: ``.so``/``.pyd`` still come from the CMake + build tree, so C++ changes need a ``cmake --build`` (or a re-run of + ``pip install -e .``) first. + +See :doc:`/references/build` for the full build-system reference. + +.. admonition:: ``retargeters`` extra on aarch64 (Jetson / DGX) + + The full ``retargeters`` extra does **not** resolve on ``aarch64`` (some of its + pinned dependencies have no aarch64 wheels). On Jetson/DGX-class robotics + targets, install the ``retargeters-lite`` extra instead: + + .. code-block:: bash + + pip install "isaaccapture[retargeters-lite]" + +.. toctree:: + :hidden: + + webxr + +.. + References +.. _`uv`: https://docs.astral.sh/uv/ diff --git a/docs/source/development/build_from_source/webxr.rst b/docs/source/development/build_from_source/webxr.rst new file mode 100644 index 0000000000..184fbf062f --- /dev/null +++ b/docs/source/development/build_from_source/webxr.rst @@ -0,0 +1,101 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Build WebXR Client +================== + +.. tip:: + + A prebuilt CloudXR web client is available at |web_client_url|. + You only need to build from source if you want to customize the client. + +The Isaac Teleop WebXR client is built with React, Three.js, and WebXR. It streams VR/AR content +from a CloudXR Runtime server to the browser. For more details about CloudXR.js SDK and its advanced features, see the +`CloudXR.js documentation`_. + +The source is located at :code-dir:`deps/cloudxr/webxr_client/`. + +.. contents:: Steps + :local: + :depth: 1 + +Prerequisites +------------- + +- Node.js (v20 or higher). Ubuntu's ``nodejs`` apt package is older (18.x on Ubuntu 24.04), so + install a current release from `Node.js downloads `_ (for + example with nvm) and check it with ``node --version``. ``npm`` ships with it. + +1. Download CloudXR.js SDK +-------------------------- + +From the project root: + +.. code-block:: bash + + source scripts/setup_cloudxr_env.sh + scripts/download_cloudxr_sdk.sh + +This will automatically download the CloudXR.js SDK and place it in ``deps/cloudxr/``. The +`package.json` is configured to install the SDK from the downloaded local package. + +2. Install dependencies +----------------------- + +From the ``deps/cloudxr/webxr_client/`` directory: + +.. code-block:: bash + + npm install + +3. Build & Run +-------------- + +Production build +~~~~~~~~~~~~~~~~ + +From ``deps/cloudxr/webxr_client/``: + +.. code-block:: bash + + npm run build + +Output in ``build/``: + +- ``index.html`` — entry page +- ``bundle.js`` — main application (UIKit, MSDF text, icons; loaded on first paint) +- ``bundle.emulator.js`` — desktop XR / IWER stack (DevUI, synthetic environments, + and related deps); loaded only when emulation is used + + +Development build (one-shot) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Outputs to ``./build/`` with source maps for debugging: + +.. code-block:: bash + + npm run dev + +Development server (with hot reload) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Starts a webpack dev server on ``http://localhost:8080`` with hot module replacement: + +.. code-block:: bash + + npm run dev-server + +For **HTTPS** (required by WebXR on most browsers): + +.. code-block:: bash + + npm run dev-server:https + +This starts the dev server with a self-signed certificate on ``https://localhost:8080``. You can follow the same +instructions to use the WebXR client as documented in the :ref:`Quick Start Guide `. + + +.. + References +.. _`CloudXR.js documentation`: https://docs.nvidia.com/cloudxr-sdk/latest/usr_guide/cloudxr_js/index.html diff --git a/docs/source/development/contributing.rst b/docs/source/development/contributing.rst new file mode 100644 index 0000000000..a411af25cc --- /dev/null +++ b/docs/source/development/contributing.rst @@ -0,0 +1,77 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Contributing +============ + +We welcome contributions. Please see the repository's `CONTRIBUTING.md `_ for: + +- Code of conduct and how to contribute +- Development setup and coding standards +- Pull request process + +Previewing documentation changes +-------------------------------- + +Local build +~~~~~~~~~~~ + +Build the docs locally to catch broken links and rendering issues before opening +a pull request: + +.. code-block:: bash + + cd docs + pip install -r requirements.txt + make current-docs + +The output is written to ``docs/build/current/``. Open ``index.html`` in a +browser to inspect it. Sphinx is run with ``-W --keep-going``, so warnings are +treated as errors — fix them locally before pushing. + +PR preview on GitHub Pages +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Every PR preview is published to a single canonical location: + +.. code-block:: text + + https://nvidia.github.io/IsaacCapture/preview/pr-/ + +How the preview gets built depends on where the PR's branch lives. + +PRs from a branch on ``NVIDIA/IsaacCapture`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``Build & deploy docs`` workflow runs automatically on every push to the +PR branch and publishes the preview. The preview URL is added to the workflow +run's *Summary* tab and refreshed on every push. No extra action required. + +PRs from a fork +^^^^^^^^^^^^^^^ + +GitHub Actions on PRs from forks run with a read-only token, so the workflow +cannot push to ``gh-pages`` automatically. Instead: + +1. ``Build & deploy docs`` still runs and uploads the built artifacts; only + the deploy step is skipped. +2. When a fork PR is opened, a bot comments with instructions. +3. A maintainer (anyone with write access to ``NVIDIA/IsaacCapture``) deploys + the preview by commenting on the PR: + + .. code-block:: text + + /preview-docs + +4. The maintainer-triggered workflow downloads the artifacts that + ``Build & deploy docs`` already produced for that commit and pushes them + to ``preview/pr-/``, reacting to the comment with 👀 while deploying + and 👍 once published. A follow-up comment posts the preview URL. + +Because the deploy reuses the same artifacts that the build job validated, +``/preview-docs`` does not run any PR code itself. Re-comment +``/preview-docs`` after new commits land to redeploy from the latest build. + +Previews are not auto-cleaned; maintainers can run the +``Cleanup docs PR previews`` workflow from the *Actions* tab to clear the +``preview/`` tree. diff --git a/docs/source/development/license.rst b/docs/source/development/license.rst new file mode 100644 index 0000000000..62a40de509 --- /dev/null +++ b/docs/source/development/license.rst @@ -0,0 +1,38 @@ +.. _license: + +License +======== + +Unless otherwise noted, the code in this repository is licensed under the +`Apache 2.0 license `_. + +Known exceptions +---------------- + +- Certain headers from the `OpenXR SDK `_ or related + `extensions `_ + are imported from their original source and are licensed under the + `MIT license `_ or `Boost Software License 1.0 `_. + +- CloudXR SDK is NVIDIA's proprietary software, licensed under the `NVIDIA CloudXR License `_. +- The documentation is built using `NVIDIA Sphinx Theme `_. + +License headers +--------------- + +We use `REUSE `_ to enforce SPDX license headers on source files. +`Pre-commit `_ runs +``reuse lint-file`` on changed files (e.g. ``.py``, ``.cpp``, ``.md``, ``.yaml``): + +1. **Required fields**: Each file must have ``SPDX-FileCopyrightText`` and ``SPDX-License-Identifier``. +2. **License**: ``SPDX-License-Identifier`` must be ``Apache-2.0`` (full text in ``LICENSES/Apache-2.0.txt``). + +Example header (for Python files): + +.. code-block:: python + :class: code-100col + + # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + # SPDX-License-Identifier: Apache-2.0 + +You can also run ``reuse spdx -o project.spdx`` to generate an SPDX report for the project. diff --git a/docs/source/development/migration.rst b/docs/source/development/migration.rst new file mode 100644 index 0000000000..fec4275041 --- /dev/null +++ b/docs/source/development/migration.rst @@ -0,0 +1,117 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Migrating to ``isaaccapture`` +============================= + +In 1.6 the Python import package and the distribution were both renamed from +``isaacteleop`` to ``isaaccapture``. ``import isaacteleop`` keeps working for now +through an alias shipped inside the ``isaaccapture`` wheel, which **will be +removed in 1.9**. + +Depend on ``isaaccapture`` and pin ``isaaccapture<1.9`` if you need the alias +to stay available while you migrate imports. + +Isaac Capture is the umbrella library; Isaac Teleop remains its teleoperation +component. The package migration does not deprecate the Isaac Teleop brand. + +In short +-------- + +#. Upgrade with ``pip install -U isaaccapture[...]`` -- see + :ref:`migration-upgrading`. +#. Rewrite ``isaacteleop`` to ``isaaccapture`` in imports, ``python -m`` commands + and dependency pins; ``isaacteleop_examples`` to ``isaaccapture_examples``. + +.. _migration-upgrading: + +Upgrading an existing install +----------------------------- + +Installing ``isaaccapture`` replaces the old ``isaacteleop`` distribution with +a small transition package that contains no code. Pip removes the legacy +implementation automatically: + +.. code-block:: console + + $ pip install -U "isaaccapture[cloudxr]" + +The same command works on a fresh environment. Both import names resolve to +``isaaccapture``. Existing ``pip install -U isaacteleop[...]`` commands also +install ``isaaccapture`` and forward the requested extras. + +Imports +------- + +.. code-block:: diff + + -import isaacteleop + -from isaacteleop.teleop_session_manager import DeviceState + +import isaaccapture + +from isaaccapture.teleop_session_manager import DeviceState + +Every submodule keeps its name; only the top-level package changes. Both names +resolve to the same module objects until 1.9, so a partially migrated process +shares state, singletons and extension registrations rather than loading two +copies. + +The same rename applies to ``python -m`` entry points, for example +``python -m isaaccapture.cloudxr.service run``. + +.. warning:: + + ``isaacteleop_examples`` became ``isaaccapture_examples``, and the alias + does **not** cover it -- it serves ``isaacteleop`` alone. Every + ``python -m isaacteleop_examples.*`` command fails with a bare + ``ModuleNotFoundError`` and no notice attached. The example distributions + renamed with it: ``isaacteleop-examples-*`` to ``isaaccapture-examples-*``. + +What importing the old name does +-------------------------------- + +Until 1.9 the old name emits a standard ``DeprecationWarning`` on first import. +Python's warning filters control its visibility; use ``-W default`` to see it. +With ``-W error::DeprecationWarning``, the import raises until migrated or filtered. + +Installing +---------- + +.. code-block:: diff + + -pip install isaacteleop[cloudxr] + +pip install isaaccapture[cloudxr] + +Update dependency pins and extras to the new name: + +.. code-block:: diff + + -isaacteleop[retargeters,ui,cloudxr]~=1.5.0 + +isaaccapture[retargeters,ui,cloudxr]~=1.6.0 + +The ``isaaccapture`` wheel ships the ``isaacteleop`` alias inside it, so +``import isaacteleop`` keeps working after the requirement is renamed. Pip +installs the transition dependency automatically. + +A ``sys.meta_path`` finder is invisible to a type checker, so everything under +``isaacteleop`` is typed ``Any`` and **pyright reports every submodule import as +unresolved** (``reportMissingImports``). The alias promises runtime identity only. + +Logging +------- + +Loggers are named after the module they live in, and the alias hands back the +real module, so the whole hierarchy moved to ``isaaccapture.*`` even for code +that still imports the old name. No alias can reach a string key, so a +``dictConfig`` or ``getLogger`` call naming ``isaacteleop`` silently stops +matching -- the logger is created, and nothing ever logs to it: + +.. code-block:: diff + + -logging.getLogger("isaacteleop.cloudxr").setLevel(logging.DEBUG) + +logging.getLogger("isaaccapture.cloudxr").setLevel(logging.DEBUG) + +The asset cache +--------------- + +The SO101 and reBot asset-directory overrides have been removed. The +``ISAAC_TELEOP_*`` runtime switches keep their names. diff --git a/docs/source/development/televiz.rst b/docs/source/development/televiz.rst new file mode 100644 index 0000000000..1cee56cd37 --- /dev/null +++ b/docs/source/development/televiz.rst @@ -0,0 +1,763 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Televiz +======== + +Televiz (``isaaccapture.viz``) is the shared visualization module for Isaac Capture. It composites what the +operator sees — camera and sensor feeds, plus 3D rendered content such as gsplat or nvblox — and +presents it in stereo to an XR headset over :doc:`CloudXR `. Desktop-window and +offscreen output are also available, mainly for development and debugging. + +It is a **compositor**, not a capture or streaming layer: it consumes GPU frames and assembles them +into a final image. Camera capture, decode, and network transport live in the application (see +:doc:`/references/camera_streaming`). + +The compositor is implemented in C++ (``namespace viz``, built on Vulkan + OpenXR + CUDA with no +external rendering-framework dependency) and exposed through a pybind11 binding. This page uses the +Python API, which mirrors the C++ names one-to-one — see `C++ API`_. + +.. contents:: On this page + :local: + :depth: 2 + +Installation +------------ + +Televiz ships inside the ``isaaccapture`` wheel — install it from PyPI: + +.. code-block:: bash + + pip install isaaccapture + +The published wheels (Linux x86_64 / aarch64, CPython 3.11–3.13) bundle the compiled +``isaaccapture.viz`` module, so **no source build is required**. Verify with: + +.. code-block:: python + + import isaaccapture.viz as televiz + +You only need to :doc:`build from source ` when +developing Isaac Capture itself — that build enables Televiz automatically when Vulkan and the +CUDA Toolkit are detected. ``glslangValidator`` is additionally required to compile the shaders: +if it is missing, configuration fails rather than quietly dropping the module — install it, or +pass ``-DBUILD_VIZ=OFF``. + +Overview +-------- + +The central object is :code-file:`VizSession `, +which owns the Vulkan context, the display target, the OpenXR session (in XR mode), and a registry +of **layers**. Content producers submit GPU buffers to layers; the session composites every layer +into one frame each time you call ``render()``. + +Four layer types are available: + +* :code-file:`QuadLayer ` — a CUDA-fed 2D texture + plane (mono or stereo), optionally placed in 3D space. Use it for camera feeds. +* :code-file:`CylinderLayer ` — the same + CUDA-fed texture curved onto the inside of a cylinder arc, so wide-FOV feeds keep a constant + viewing distance edge to edge. XR-only. +* :code-file:`EquirectLayer ` — an + equirectangular texture mapped onto the inside of a sphere, for 360°/180° panorama and VR-video + sources. XR-only. +* :code-file:`ProjectionLayer ` — a full-view + RGBD layer for external renderers (gsplat, nvblox, neural reconstruction) that produce per-view + ``(color, depth)`` buffers. Use it to present a rendered 3D scene from the current head pose. + +A session holds **either** one ``ProjectionLayer`` **or** any number of texture layers +(``QuadLayer`` / ``CylinderLayer`` / ``EquirectLayer``), never both — see `Layers`_. + +All symbols are imported from the top-level module:: + + import isaaccapture.viz as televiz + +Display modes +------------- + +A session runs in exactly one display mode, set on the config: + +.. list-table:: + :header-rows: 1 + :widths: 20 80 + + * - ``DisplayMode`` + - Behavior + * - ``kXr`` + - OpenXR + Vulkan. Per-eye swapchains, stereo rendering, depth composition layer. Requires a + running OpenXR runtime (e.g. CloudXR). + * - ``kWindow`` + - GLFW desktop window. Layers are aspect-fit tiled; stereo layers show the left eye. + * - ``kOffscreen`` + - No display. Composite to an internal target and pull pixels back with + ``readback_to_host()``. Useful for tests and headless rendering. + +Quick start +----------- + +A minimal offscreen render-and-readback (no GPU display, no headset): + +.. code-block:: python + + import cupy as cp + import isaaccapture.viz as televiz + + viz_cfg = televiz.VizSessionConfig() + viz_cfg.mode = televiz.DisplayMode.kOffscreen + viz_cfg.window_width = 1024 + viz_cfg.window_height = 1024 + + session = televiz.VizSession.create(viz_cfg) + + layer_cfg = televiz.QuadLayerConfig() + layer_cfg.name = "cam" + layer_cfg.resolution = televiz.Resolution(1024, 1024) + layer = session.add_quad_layer(layer_cfg) + + # Any __cuda_array_interface__ array (CuPy / PyTorch / Numba) or a VizBuffer. + frame = cp.zeros((1024, 1024, 4), dtype=cp.uint8) # RGBA8 + layer.submit(frame) + + info = session.render() # wait + composite + present + img = session.readback_to_host() # HostImage; numpy.asarray(img) for pixels + + session.destroy() + +For a window or headset, set ``mode`` to ``DisplayMode.kWindow`` or ``DisplayMode.kXr`` instead. The +layer setup is identical; you just drive a frame loop (see `Frame loop`_) rather than the one-shot +``readback_to_host()``, which is offscreen-only. + +.. _orin-openxr-composition: + +.. note:: + + **Jetson Orin, XR.** The default ``openxr_composition = True`` path presents a solid + color, or black, in place of the submitted image. Until this is addressed in a future + release, set ``openxr_composition = False`` so Televiz composites the quad itself: + + .. code-block:: python + + layer_cfg.openxr_composition = False + + Cylinder and equirect layers cannot opt out. In :doc:`/references/camera_streaming` the + same switch is YAML ``compositor: televiz`` (quads only). + +Session configuration +--------------------- + +``VizSessionConfig`` fields: + +.. list-table:: + :header-rows: 1 + :widths: 28 16 56 + + * - Field + - Default + - Description + * - ``mode`` + - ``DisplayMode.kOffscreen`` + - ``DisplayMode.kXr`` / ``kWindow`` / ``kOffscreen``. Set it explicitly; the default is + offscreen. + * - ``window_width`` / ``window_height`` + - ``1024`` + - Render size for window and offscreen modes. Ignored in XR (the runtime dictates per-eye + resolution; query it with ``get_recommended_resolution()``). + * - ``app_name`` + - ``"televiz"`` + - OpenXR application name. + * - ``required_extensions`` + - ``[]`` + - Extra OpenXR instance extensions to enable when Televiz hosts the session and downstream + components (e.g. ``TeleopSession`` trackers) need them. Televiz already enables its own + rendering extensions. See `Sharing the XR session`_. + * - ``xr_near_z`` / ``xr_far_z`` + - ``0.05`` / ``100.0`` + - Near / far planes for the XR projection. + * - ``xr_system_wait_seconds`` + - ``0`` + - How long to wait for the OpenXR system (headset) to become available at create time. + Zero fails fast when no headset is present. + * - ``clear_color`` + - ``(0, 0, 0, 0)`` + - Background color as an ``(r, g, b, a)`` sequence in ``[0, 1]``. Transparent by default. + * - ``gpu_timing`` + - ``False`` + - Enable GPU timestamp queries, surfaced via ``get_gpu_timing()``. + +Construct the session with the factory; never call the class directly: + +.. code-block:: python + + session = televiz.VizSession.create(cfg) + +Layers +------ + +Layers come in two families, and a session holds one family or the other: + +* **Texture layers** — ``QuadLayer``, ``CylinderLayer``, ``EquirectLayer``. A session can hold any + number of them. All three take the same configuration, are fed by the same ``submit()``, and are + composited the same way; they differ *only* in the surface the texture is mapped onto — a flat + plane, a curved arc, or a sphere. The three sections at the end of this chapter cover just that + difference; everything before them applies to all of them. +* **Projection layer** — one ``ProjectionLayer``, a full-view RGBD layer for an in-loop renderer. + A session holds at most one, and never alongside a texture layer. See `ProjectionLayer`_. + +Layers render in **insertion order** — the first added renders first (underneath). A layer is owned +by the session; ``add_quad_layer`` returns a **non-owning** handle, so don't keep it past the +session's lifetime. + +.. _openxr-composition-layers: + +Composition model +^^^^^^^^^^^^^^^^^ + +In XR, each texture layer is handed to the OpenXR runtime as its own composition layer +(``XrCompositionLayerQuad`` / ``CylinderKHR`` / ``Equirect2KHR``) rather than drawn by Televiz; the +runtime places, samples, and reprojects it at display rate. That keeps text and fine detail sharp +under head motion, and it lets CloudXR stream color-only video when every visible layer is +runtime-composited and opaque — the headset then rebuilds the composition from the layer geometry +at lower bandwidth. + +Three consequences, in rough order of how often they bite: + +* **Runtime-composited layers carry no depth.** They blend in insertion order, so a foreground + overlay added before a camera feed renders *behind* it no matter where the two sit in 3D. Add + backgrounds — an ``EquirectLayer`` panorama, say — first. +* **Alpha is ignored unless you ask for it.** ``alpha_blend`` (default off) makes the runtime honor + the texture's alpha channel — turn it on for translucent content such as a HUD. Keep it off on + opaque camera feeds: as long as no visible layer uses source alpha, CloudXR can stream the layers + themselves and let the headset rebuild the composition, rather than sending a fully composited + stereo frame. That is less to encode and less on the wire every frame — lower bandwidth, and + lower latency on a constrained link. A single visible alpha-blended layer disqualifies the whole + frame. +* **Mipmaps only exist on the built-in path.** ``generate_mipmaps`` applies when Televiz does the + drawing; the runtime samples native layers itself. On the default XR path the flag has no effect. + +``QuadLayer`` is the one layer with a choice, which is why ``openxr_composition`` and +``generate_mipmaps`` appear on its config and on no other. Set ``openxr_composition = False`` to +draw the quad with Televiz's built-in compositor instead, where 3D-placed quads depth-test against +each other in a shared render target — the way out of the no-depth constraint above, and the +setting to use on Jetson Orin in XR (see :ref:`Jetson Orin, XR `). Cylinder and +equirect layers have no such switch: they exist only as native runtime layers, and therefore only in +XR. Window and offscreen modes always use the built-in compositor. + +Shared configuration +^^^^^^^^^^^^^^^^^^^^ + +``QuadLayerConfig``, ``CylinderLayerConfig``, and ``EquirectLayerConfig`` all carry these fields: + +.. list-table:: + :header-rows: 1 + :widths: 26 14 60 + + * - Field + - Default + - Description + * - ``name`` + - — + - Layer name (used as the placement key in app config). + * - ``resolution`` + - — + - Source texture size, a ``Resolution``. Submitted buffers must match it. + * - ``format`` + - — + - ``PixelFormat`` of the source. ``kRGBA8`` is the only value texture layers accept; + anything else raises ``ValueError``. + * - ``placement`` + - — + - Where and how big the surface is. Each layer has its own placement type describing its own + geometry — see the per-layer sections below. + * - ``stereo`` + - ``False`` + - Per-eye stereo. When ``True``, ``submit`` requires both eyes' buffers; view 0 (left) samples + the left buffer, view 1 (right) the right. Memory doubles. + * - ``stereo_baseline_mm`` + - ``0`` + - Horizontal disparity between the per-eye surfaces (mm), along the placement's local +x axis. + ``0`` → both eyes see the same surface in the world, and any parallax comes from the frames + themselves. XR + stereo only, and no effect on an infinite-radius equirect sphere. + * - ``alpha_blend`` + - ``False`` + - Honor the texture's alpha channel (translucent content). Runtime composition only; ignored + on the built-in compositor path. + +Stereo works the same way on every surface, following the VR-video convention: per-eye textures on +the *same* surface, with ``stereo_baseline_mm`` adding an optional per-eye pose shift on top. + +Submitting frames +^^^^^^^^^^^^^^^^^ + +Every texture layer (``QuadLayer`` / ``CylinderLayer`` / ``EquirectLayer``) takes content through +the same ``submit()``. What a frame must be: + +* **GPU-resident.** A ``VizBuffer`` in device memory, or any object exposing + ``__cuda_array_interface__`` (CuPy, PyTorch, Numba). A host buffer is rejected — the teleop hot + path never round-trips through system memory. +* **RGBA8**, matching the layer's ``format``. +* **Exactly the layer's resolution.** Submitted dimensions are checked against ``resolution``, not + scaled — resize upstream. + +Three properties of ``submit()`` decide how you structure the producer around it: + +**It is a latest-wins mailbox, not a queue.** Each layer owns a small ring of device images. +``submit()`` copies your pixels into a slot no in-flight frame is reading, then publishes it; the +renderer always picks up the most recent completed publish. A producer faster than the display +rate therefore **drops** frames rather than queueing or blocking, and a producer slower than the +display rate has its last frame re-presented. Neither side waits on the other, and no frame is +ever shown half-updated. + +**It blocks until the copy completes.** ``submit()`` synchronizes ``stream`` before returning, so +your source buffer is free to reuse the moment it does — no fences to manage, no lifetime rules +past the call. The cost lands on the calling thread (see `Performance and diagnostics`_), not on +the render path, which is why capture threads should call ``submit()`` directly rather than +funnelling frames to the render thread. + +**One producer per layer.** ``submit()`` is safe against the renderer consuming the same layer +concurrently, but *not* against a second thread submitting to the same layer. Give each producer +its own layer. + +.. warning:: + + **Stereo submits carry a stream precondition.** ``submit(left, right, stream=...)`` copies both + eyes on the single ``stream`` you pass. CUDA orders work only within one stream, so if either + buffer was produced on a *different* stream, synchronize that stream first — either + ``cudaStreamSynchronize`` on the producer stream, or record an event there and + ``cudaStreamWaitEvent`` it on ``stream``. Skip it and that eye can be copied mid-write: torn or + stale pixels, with no error raised. The in-tree ZED and OAK-D sources synchronize per eye + before publishing, which is what makes their plain ``submit(left, right)`` (stream 0) safe. + + +QuadLayer +^^^^^^^^^ + +A flat plane — the default surface, and the right one for normal-FOV camera feeds. Beyond the +`Shared configuration`_ fields, ``QuadLayerConfig`` adds the two composition knobs, which exist here +because the quad is the only layer that can be drawn either way: + +.. list-table:: + :header-rows: 1 + :widths: 26 14 60 + + * - Field + - Default + - Description + * - ``generate_mipmaps`` + - ``True`` + - Allocate + regenerate a capped mip chain each frame; sampler uses trilinear filtering. Only + applies on the built-in compositor path. + * - ``openxr_composition`` + - ``True`` + - ``True`` = the OpenXR runtime composites the quad, ``False`` = Televiz's built-in + compositor. On Jetson Orin in XR, set ``False`` (see :ref:`Jetson Orin, XR `). + See `Composition model`_. + +Its placement is a ``QuadLayerPlacement`` — a ``pose`` plus ``size_meters``. In window mode it +is optional: a quad with no placement fills its tile of the window (the whole window when it is +the only layer). In XR it is **required**: set it in the +config or with ``set_placement()`` before a submitted frame is rendered, or ``render()`` raises +``RuntimeError: ... requires Config::placement to be set``. ``add_quad_layer()`` does not reject +a missing placement, so a placement strategy can set it after the layer is created. + +.. code-block:: python + + layer = session.add_quad_layer(layer_cfg) + + # Mono: pass exactly one buffer. Stereo: layer.submit(left, right). + layer.submit(rgba_array) # optional: stream= + + # 3D placement (XR). Pose is OpenXR stage space: position (x,y,z), + # orientation quaternion (w,x,y,z). size_meters is (width, height). + placement = televiz.QuadLayerPlacement( + televiz.Pose3D(position=(0.0, 0.0, -1.5), orientation=(1.0, 0.0, 0.0, 0.0)), + size_meters=(1.0, 0.5625), + ) + layer.set_placement(placement) + layer.set_visible(True) + +Lock-mode placement strategies (``world`` / ``head`` / ``lazy`` / ``gimbal``) are **application +policy** and ship in the sample, not in the module. + +CylinderLayer +^^^^^^^^^^^^^ + +The texture curved onto the inside of a vertical cylinder arc. Every point on the surface sits at +the same distance from the cylinder's axis, which makes it the natural surface for wide-FOV camera +feeds — a flat quad wide enough for a 110° image would put its edges much farther from the eye than +its center. ``CylinderLayerPlacement`` describes the arc: + +.. list-table:: + :header-rows: 1 + :widths: 26 14 60 + + * - Field + - Default + - Description + * - ``pose`` + - identity + - Cylinder center; the arc bows out along the pose's ``-z``, cylinder axis is ``+y``. + * - ``radius_m`` + - ``1.0`` + - Radius in meters — the viewing distance to the surface. ``0`` / ``+inf`` = infinite. + * - ``central_angle_rad`` + - ``π/2`` + - Visible arc in radians, ``(0, 2π)``. + * - ``aspect_ratio`` + - ``0`` + - Arc width / height. ``0`` derives it from ``resolution`` (square texels). + +XR only, and the runtime must support ``XR_KHR_composition_layer_cylinder`` (CloudXR does). +``add_cylinder_layer`` raises ``ValueError`` otherwise. + +EquirectLayer +^^^^^^^^^^^^^ + +An equirectangular texture mapped onto the inside of a sphere centered on (by default) the +operator, for 360°/180° panoramas and VR-video sources. The ``EquirectLayerPlacement`` defaults +describe a full 360°×180° sphere at infinite radius, so a panorama needs no placement at all: + +.. list-table:: + :header-rows: 1 + :widths: 30 14 56 + + * - Field + - Default + - Description + * - ``pose`` + - identity + - Sphere center; the texture's horizontal center maps to the pose's ``-z``. + * - ``radius_m`` + - ``0`` + - Radius in meters; ``0`` / ``+inf`` = infinite sphere. + * - ``central_horizontal_angle_rad`` + - ``2π`` + - Horizontal span (``π`` for VR180). + * - ``upper_vertical_angle_rad`` / ``lower_vertical_angle_rad`` + - ``π/2`` / ``−π/2`` + - Vertical span from the horizon, upper > lower. + +XR only, like the cylinder; the required extension here is +``XR_KHR_composition_layer_equirect2``. + +Combining the two — a panorama background with a camera feed on an arc in front of it, background +added first so it composites underneath: + +.. code-block:: python + + eq_cfg = televiz.EquirectLayerConfig() + eq_cfg.name = "sky" + eq_cfg.resolution = televiz.Resolution(4096, 2048) + sky = session.add_equirect_layer(eq_cfg) # default = full sphere + + cyl_cfg = televiz.CylinderLayerConfig() + cyl_cfg.name = "cam" + cyl_cfg.resolution = televiz.Resolution(1920, 1080) + cyl_cfg.placement = televiz.CylinderLayerPlacement(radius_m=2.0) # 2 m arc in front of the user + cam = session.add_cylinder_layer(cyl_cfg) + + while running: + sky.submit(panorama_rgba) + cam.submit(camera_rgba) + session.render() + +ProjectionLayer +^^^^^^^^^^^^^^^ + +A full-view RGBD layer for **in-loop** renderers — gsplat, nvblox, or neural reconstruction engines +that produce per-view ``(color, depth)`` buffers. Configure it with ``ProjectionLayerConfig``: + +.. list-table:: + :header-rows: 1 + :widths: 25 75 + + * - Field + - Description + * - ``name`` + - Layer name. + * - ``view_resolution`` + - Per-view render resolution. **Must equal** ``session.get_recommended_resolution()`` — the + layer's images are copied 1:1 into the presentation swapchains (per-eye in XR). A mismatch + is rejected by ``add_projection_layer``. + * - ``color_format`` + - ``PixelFormat.kRGBA8``. + * - ``depth_format`` + - ``PixelFormat.kD32F`` (default) so the depth reaches the XR runtime for positional + reprojection, or ``None`` to present color only. + * - ``stereo`` + - ``True`` for per-eye buffers. A stereo (XR) display **requires** a stereo layer; a mono layer + is rejected at ``add_projection_layer``. + +Unlike ``QuadLayer``, a projection layer is **direct-present**: each view's ``(color, depth)`` is +copied straight into the presentation swapchains (no shared render target). Because of that a session +holds *either* one ``ProjectionLayer`` *or* any number of texture layers, never both. + +The renderer runs **in-loop** with the frame loop: read the predicted view poses from the +``FrameInfo`` returned by ``begin_frame()``, render against them, then ``submit()`` before +``end_frame()``: + +.. code-block:: python + + cfg = televiz.ProjectionLayerConfig() + cfg.view_resolution = session.get_recommended_resolution() + cfg.stereo = session.is_xr_mode() + layer = session.add_projection_layer(cfg) + + while running: + info = session.begin_frame() + if info.should_render: + # Render against THIS frame's per-eye poses (info.views[i].pose + .fov). + color, depth = renderer.render(info.views) # RGBA8 + D32F CUDA buffers + if layer.stereo: + layer.submit(left_color, left_depth, right_color, right_depth, stream=cuda_stream) + else: + layer.submit(color, depth, stream=cuda_stream) + session.end_frame() + +If the renderer is slower than display rate, the runtime / CloudXR paces the app via ``xrWaitFrame`` +and reprojects the last submitted frame at display rate. In XR, a visible layer that does **not** +submit for a frame presents nothing (the swapchains are cleared) rather than reproject stale RGBD +under a new pose. + +Frame loop +---------- + +Two API levels drive the frame loop. Both release the GIL during blocking waits. + +**Convenience** — ``render()`` does wait + composite + present in one call and returns a +``FrameInfo``. Internally it checks ``should_render`` and skips the GPU pass when the runtime +says the frame won't be visible; producers' ``submit`` writes still land in the back buffer. + +.. code-block:: python + + while running: + cam_layer.submit(camera_frame) + info = session.render() + +**Explicit** — ``begin_frame()`` / ``end_frame()``, when the app needs the +``FrameInfo`` *before* submitting (e.g. to read the predicted view poses before rendering, or to +skip expensive decode when not visible): + +.. code-block:: python + + while running: + info = session.begin_frame() + if info.should_render: + cam_layer.submit(decode_camera()) # skip decode when not visible + session.end_frame() + +``FrameInfo`` carries ``frame_index``, ``predicted_display_time`` (XR time in ns; 0 outside +XR), ``delta_time`` (CPU wall-clock seconds — usable without any XR knowledge), ``should_render``, +``resolution``, and ``views``. Each ``ViewInfo`` in ``views`` has ``viewport``, ``fov``, and +``pose`` — 2 entries in XR stereo, 1 (identity pose) in window / offscreen. + +Performance and diagnostics +--------------------------- + +``get_frame_timing_stats()`` is the first thing to reach for when the view stutters. It costs +nothing to call and needs no configuration: + +.. list-table:: + :header-rows: 1 + :widths: 24 76 + + * - Field + - Meaning + * - ``render_fps`` + - Smoothed frame rate over a recent window — what you are actually achieving. + * - ``target_fps`` + - What you should be achieving: the headset's display rate in XR (60 / 72 / 90 / 120), the + vsync rate in window mode. + * - ``missed_frames`` + - Cumulative count of frames whose GPU work didn't finish inside the budget. Rising while + ``render_fps`` holds means you are on the edge; rising with ``render_fps`` below + ``target_fps`` means you are over it. + * - ``avg_frame_time_ms`` + - Mean total frame time. + * - ``gpu_time_ms`` + - Last frame's GPU-side render time. + * - ``stale_layers`` + - Layers whose producer missed the stale timeout this frame — the signal that a *capture* + thread, not the compositor, is behind. Non-zero here points you upstream. + +``get_gpu_timing()`` breaks the GPU side into ``total_ms`` / ``render_pass_ms`` / ``post_pass_ms``. +It requires ``gpu_timing = True`` on ``VizSessionConfig`` (timestamp queries are off by default); +without it the fields read zero. + +.. code-block:: python + + last_missed = 0 + + while running: + session.render() + + stats = session.get_frame_timing_stats() + if stats.missed_frames > last_missed or stats.stale_layers: + print(f"{stats.render_fps:.1f}/{stats.target_fps:.0f} fps, " + f"gpu {stats.gpu_time_ms:.2f} ms, stale layers {stats.stale_layers}") + last_missed = stats.missed_frames + +Session state +------------- + +A session moves through ``SessionState``: + +``kUninitialized → kReady → kRunning → kStopping → kLost → kDestroyed`` + +- ``kReady`` after ``create`` — add layers and submit content. +- ``kRunning`` once the frame loop is active. +- ``kStopping`` (XR) — the runtime is stopping; ``end_frame`` submits empty frames. +- ``kLost`` (XR) — the session was lost; ``render`` / ``begin_frame`` raise. Destroy and recreate + the ``VizSession`` (Televiz supports clean in-process recreation). +- ``kDestroyed`` after ``destroy``. + +Query it with ``get_state()``; in window mode ``should_close()`` reports the window-close +request. OpenXR events are polled inside ``begin_frame``, which drives the XR-specific transitions. + +.. _sharing-the-xr-session: + +Sharing the XR session +---------------------- + +Only one OpenXR session is allowed per process. In XR mode ``VizSession`` creates a **graphics-bound** +session (Isaac Teleop's own ``OpenXRSession`` is headless and cannot render). When you use Televiz +*and* ``TeleopSession`` together, let Televiz own the session and hand its live handles to +``TeleopSession`` so trackers attach to the same session — one CloudXR connection, synchronized +timing. + +Declare the extensions your trackers need in ``required_extensions`` (Televiz adds its own rendering +extensions automatically), then pass the handles through: + +.. code-block:: python + + import isaaccapture.viz as televiz + from isaaccapture.teleop_session_manager import TeleopSession, TeleopSessionConfig + from isaaccapture.deviceio import DeviceIOSession + from isaaccapture.oxr import OpenXRSessionHandles + + viz_cfg = televiz.VizSessionConfig() + viz_cfg.mode = televiz.DisplayMode.kXr + # Aggregate the XR extensions downstream trackers need so they're present + # on the XrInstance Televiz is about to create. + viz_cfg.required_extensions = DeviceIOSession.get_required_extensions(trackers) + viz_session = televiz.VizSession.create(viz_cfg) + + teleop_cfg = TeleopSessionConfig( + app_name="MyApp", + pipeline=pipeline, + oxr_handles=OpenXRSessionHandles(*viz_session.get_oxr_handles()), + ) + with TeleopSession(teleop_cfg) as session: + while running: + session.step() + cam_layer.submit(camera_frame) + viz_session.render() + +``get_oxr_handles()`` returns ``(instance, session, space, proc_addr)`` as raw ``uint64`` +values (or ``None`` outside ``kXr``); wrap them with ``OpenXRSessionHandles(*tuple)`` for +``TeleopSessionConfig.oxr_handles``. ``VizSession`` and ``TeleopSession`` keep **independent** state +machines and lifecycles — either can run without the other. In the unified pattern the underlying +OpenXR session is shared, so if it is lost both must be recreated. See +:doc:`../teleop/teleop_session` for the ``TeleopSession`` side. + +API reference +------------- + +VizSession +^^^^^^^^^^ + +- ``create(config) -> VizSession`` *(static)* — validate config + initialize Vulkan / display backend. +- ``render() -> FrameInfo`` — wait + composite + present. +- ``begin_frame() -> FrameInfo`` / ``end_frame()`` — explicit two-phase frame loop. +- ``add_quad_layer(config) -> QuadLayer`` — construct + register a layer; returns a non-owning handle. +- ``add_cylinder_layer(config) -> CylinderLayer`` / ``add_equirect_layer(config) -> EquirectLayer`` — + shaped texture layers (``kXr`` only; raise ``ValueError`` elsewhere). +- ``readback_to_host() -> HostImage`` — most recent frame as RGBA8 host pixels (``kOffscreen`` only). +- ``get_state() -> SessionState``, ``should_close() -> bool``, ``is_xr_mode() -> bool``. +- ``get_recommended_resolution() -> Resolution`` — runtime per-eye resolution (XR). +- ``head_pose_now() -> Optional[Pose3D]`` — current head pose (``kXr`` only; ``None`` on tracking loss). +- ``get_oxr_handles() -> Optional[tuple]`` — ``(instance, session, space, proc_addr)`` as raw ``uint64``. +- ``get_frame_timing_stats() -> FrameTimingStats`` / ``get_gpu_timing() -> GpuFrameTiming``. +- ``destroy()`` — release all resources (idempotent). +- Properties ``vk_device`` / ``vk_physical_device`` / ``vk_queue_family_index`` — raw handles for + wiring Televiz into a foreign Vulkan app. Most users won't touch these. + +QuadLayer / CylinderLayer / EquirectLayer +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- ``submit(left, right=None, stream=0)`` — submit a frame (mono: ``left`` only; stereo: both). +- ``set_placement(placement)`` / ``placement()`` — placement swap, thread-safe vs the frame loop. +- ``set_stereo_baseline_mm(mm)`` / ``stereo_baseline_mm`` — live per-eye offset, thread-safe vs + the frame loop; applies on the next frame. Inert while the layer is mono, since ``stereo`` is + fixed at construction. Raises ``ValueError`` on a non-finite value. +- ``set_stereo_convergence_deg(deg)`` / ``stereo_convergence_deg`` (``CylinderLayer`` / + ``EquirectLayer``) — the same idea as a per-eye yaw instead of a translation. Prefer it on a + curved surface: translating one gives full disparity dead ahead and less toward the edges, and + none at all at infinite radius, while a rotation is uniform across the arc and works at any + radius. + ``QuadLayer`` accepts ``None`` (fullscreen, window mode); the shaped layers validate and raise + ``ValueError`` on bad shape parameters. +- ``set_visible(visible)`` / ``is_visible()``. +- Properties ``resolution``, ``format``, ``name`` (plus ``aspect_ratio`` on ``QuadLayer``). + +Data types +^^^^^^^^^^ + +- ``VizBuffer`` — non-owning 2D pixel buffer descriptor. Device buffers expose + ``__cuda_array_interface__`` (``cupy.asarray(buf)``); host buffers expose ``__array_interface__`` + (``numpy.asarray(buf)``). +- ``HostImage`` — owning host pixel buffer returned by ``readback_to_host``; wrap with + ``numpy.asarray``. +- ``Resolution`` ``(width, height)``, ``Pose3D`` (``position``, ``orientation`` as + ``(w, x, y, z)``), ``Fov``, ``ViewInfo``. +- Enums ``DisplayMode``, ``PixelFormat`` (``kRGBA8`` / ``kD32F``), + ``MemorySpace``, ``SessionState``. + +C++ API +------- + +Televiz is a C++ library; ``isaaccapture.viz`` is a thin pybind11 binding over it. The Python and C++ +APIs share the same type and method names (``VizSession``, ``QuadLayer``, ``submit``, +``set_placement``, ``DisplayMode::kXr``, …), so everything on this page maps directly to C++. All +symbols live in ``namespace viz``, and headers use nested include paths:: + + #include + #include + #include + +``BUILD_VIZ`` auto-enables when Vulkan and the CUDA toolkit are detected +(force it with ``-DBUILD_VIZ=ON`` / ``-DBUILD_VIZ=OFF``); link the relevant CMake target: + +.. list-table:: + :header-rows: 1 + :widths: 18 18 64 + + * - Target + - Alias + - Provides + * - ``viz_core`` + - ``viz::core`` + - Core types (``VizBuffer``, ``Pose3D``, ``HostImage``, ``DeviceImage``) and Vulkan / CUDA infrastructure + * - ``viz_layers`` + - ``viz::layers`` + - The built-in layers (``QuadLayer``, ``CylinderLayer``, ``EquirectLayer``, + ``ProjectionLayer``) and their shared ``ImageLayerBase`` mailbox + * - ``viz_session`` + - ``viz::session`` + - ``VizSession``, the compositor, ``FrameInfo``, window / offscreen backends + * - ``viz_xr`` + - ``viz::xr`` + - OpenXR backend — per-eye swapchains, depth composition layer + +Public headers live under :code-dir:`src/viz//cpp/inc/viz/ `. One difference from +the Python bindings: in C++, layers are added with a single templated +``VizSession::add_layer(args...)`` method, which also accepts your own ``LayerBase`` subclasses — +the route for plugging in a custom renderer. See :doc:`/references/build` for build options and +output locations. + +More information +---------------- + +- :doc:`/references/camera_streaming` — the reference ``camera_viz`` sample built on Televiz +- :doc:`../teleop/teleop_session` — how ``TeleopSession`` works and how to share its OpenXR session +- :code-dir:`src/viz/ ` — module source, organized as ``core`` / ``layers`` / ``session`` / + ``xr`` / ``shaders`` / ``python`` sub-modules diff --git a/docs/source/device/add_device.rst b/docs/source/device/add_device.rst index e85aba119f..e3c008fe9b 100644 --- a/docs/source/device/add_device.rst +++ b/docs/source/device/add_device.rst @@ -3,11 +3,11 @@ Add a New Device ================ -For new hardware that requires a custom driver or SDK, create an Isaac Teleop device plugin +For new hardware that requires a custom driver or SDK, create an Isaac Capture device plugin (C++ level). Plugins push data via OpenXR tensor collections. Existing plugins include Manus gloves, OAK-D camera, Haptikos exoskeletons, controller synthetic hands, and foot pedals. After creating the plugin, update the retargeting pipeline config to consume data from the new plugin's -source node. See the `Plugins directory `_ +source node. See the `Plugins directory `_ for examples. To add a new device that streams typed data over the OpenXR runtime, follow these four steps. @@ -261,7 +261,7 @@ Both exit after 1000 samples, or press Ctrl+C to exit early. The same build installs ``se3_printer``, ``full_body_printer`` and ``frame_metadata_printer`` next to them. Those read other schemas and need their own data source, so they are not part of this -example; ``full_body_printer`` is covered in :doc:`body_tracking`. +example; ``full_body_printer`` is covered in :doc:`body_tracking/index`. **Components** @@ -281,3 +281,6 @@ example; ``full_body_printer`` is covered in :doc:`body_tracking`. - **DeviceIOSession** — Session manager: collects required OpenXR extensions from registered trackers, creates tracker implementations with session handles, and calls ``update()`` on all trackers during the update loop. + +For the shared library build and contribution workflow, see +:doc:`/development/build_from_source/index` and :doc:`/development/contributing`. diff --git a/docs/source/device/body_tracking.rst b/docs/source/device/body_tracking.rst index 6fb559203e..dfb07c82b9 100644 --- a/docs/source/device/body_tracking.rst +++ b/docs/source/device/body_tracking.rst @@ -1,280 +1,9 @@ .. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 +:orphan: + Body Tracking ============= -Isaac Teleop supports streaming full-body tracking data from an XR headset -through the CloudXR WebXR client to the teleop server. The server exposes the -body skeleton to applications through the ``FullBodyTracker`` tracker and -the OpenXR ``XR_BD_body_tracking`` extension. - -Body tracking support currently targets the **PICO 4 Ultra Enterprise** with -**PICO Motion Trackers**. The rest of this guide covers PICO hardware setup -and the BD skeleton format used on the server. - -.. important:: - - Full body tracking requires PICO's **enterprise features**. They are - available out of the box on the - `PICO 4 Ultra Enterprise `_ - headset. On a consumer PICO 4 Ultra, the browser does not grant the - ``body-tracking`` WebXR feature by default — no body data reaches the - server, even with Motion Trackers paired and calibrated. Consumer devices - can gain the feature through enterprise activation: contact PICO, then - use the **Activate enterprise account** option in the headset settings. - -.. note:: - - Meta Quest 3/3S also expose body tracking to WebXR via inside-out body - tracking (IOBT), and the CloudXR client will stream it. However the Quest - skeleton is mapped to the PICO BD 24-joint layout in the current version, - and accuracy is lower since there are no physical trackers. See - `Quest Body Tracking (Limited Support)`_ for details. - -.. seealso:: - - `GR00T-WholeBodyControl (SONIC)`_ can consume this CloudXR skeleton on a G1 robot as well. - See that project's setup guide for additional details. - -.. _`GR00T-WholeBodyControl (SONIC)`: https://nvlabs.github.io/GR00T-WholeBodyControl/tutorials/isaac_teleop_publisher_setup.html - -.. contents:: On this page - :local: - :depth: 2 - -Setting Up PICO Motion Trackers -------------------------------- - -Full body tracking uses -`PICO Motion Trackers `_ -paired with a PICO 4 Ultra Enterprise headset. Make sure the PICO browser is -updated to a version that supports body tracking. The following configurations -are supported: - -- **5 trackers:** best tracking quality across all 24 joints. - Two wear modes are available: 2 ankles + 2 wrists + 1 waist, or 2 ankles + - 2 thighs + 1 waist. -- **3 trackers:** 2 ankles + 1 waist. Upper-body joints are estimated from the - headset and controllers; lower-body tracking remains accurate. -- **2 trackers:** 2 ankles only. Provides lower-body tracking with upper-body - estimation. Least hardware, but reduced accuracy for waist and sitting/lying - postures. - -Instructions -~~~~~~~~~~~~ - -1. Tap the **clock / Wi-Fi / battery icon** in the lower-right corner of the - PICO menu. A picture of the headset appears; above it is a small circular - button with a wristband-and-tracker icon. If the button is missing, open the - **Motion Tracker** app directly. -2. Select a **wear mode** that matches the number of trackers you have and best - fits your use case. -3. Click the **Pair** button. -4. On each tracker, press and hold the top button until the lights start - flashing (pairing mode). -5. Strap the trackers on following the placement shown in the app for your - chosen wear mode. Scrunch down any baggy clothing so the trackers are visible - to the headset cameras. -6. Click **Calibrate** and follow the instructions in the Motion Tracker app. - -Before each session, confirm all trackers are powered on, paired, and -calibrated. - -How Full Body Tracking Works ----------------------------- - -Skeleton profile -~~~~~~~~~~~~~~~~ - -Body tracking data uses the -`XR_BD_body_tracking `_ -OpenXR extension profile defined by ByteDance (BD). This is the only body -tracking profile currently supported by the CloudXR runtime. - -When the CloudXR WebXR client starts a session, it requests the ``body-tracking`` -feature from the browser. Each frame, the CloudXR client library reads the body -joint data provided by the PICO runtime and streams it to the server. No -additional configuration is needed in the WebXR client; body tracking is -enabled automatically when the device supports it. - -Joint skeleton (24 joints) -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The BD profile defines a 24-joint skeleton. Each joint provides a position -(vec3 in meters), an orientation (quaternion), and a per-joint ``is_valid`` flag -indicating whether the tracker successfully resolved that joint in the current -frame. - -.. list-table:: - :header-rows: 1 - :widths: 10 25 25 - - * - Index - - Joint - - Parent - * - 0 - - Pelvis - - *(root)* - * - 1 - - Left Hip - - Pelvis - * - 2 - - Right Hip - - Pelvis - * - 3 - - Spine 1 - - Pelvis - * - 4 - - Left Knee - - Left Hip - * - 5 - - Right Knee - - Right Hip - * - 6 - - Spine 2 - - Spine 1 - * - 7 - - Left Ankle - - Left Knee - * - 8 - - Right Ankle - - Right Knee - * - 9 - - Spine 3 - - Spine 2 - * - 10 - - Left Foot - - Left Ankle - * - 11 - - Right Foot - - Right Ankle - * - 12 - - Neck - - Spine 3 - * - 13 - - Left Collar - - Spine 3 - * - 14 - - Right Collar - - Spine 3 - * - 15 - - Head - - Neck - * - 16 - - Left Shoulder - - Left Collar - * - 17 - - Right Shoulder - - Right Collar - * - 18 - - Left Elbow - - Left Shoulder - * - 19 - - Right Elbow - - Right Shoulder - * - 20 - - Left Wrist - - Left Elbow - * - 21 - - Right Wrist - - Right Elbow - * - 22 - - Left Hand - - Left Wrist - * - 23 - - Right Hand - - Right Wrist - -Server-side access -~~~~~~~~~~~~~~~~~~ - -On the server, body tracking data is consumed through the -``FullBodyTracker`` tracker (see :doc:`trackers` for the full tracker -reference). The tracker exposes a ``get_body_pose()`` method that returns the -24-joint skeleton each frame (or null when body tracking is not available). -Joint data follows the ``FullBodyPose`` FlatBuffers schema defined in -``src/core/schema/fbs/full_body.fbs``. - -The ``all_joint_poses_tracked`` quality flag indicates whether every joint was -successfully tracked in the current frame. When it is false, consult individual -joint ``is_valid`` flags to determine which joints have valid poses. - -For a minimal C++ reader see ``examples/schemaio/full_body_printer.cpp``, which -creates the tracker, queries the required OpenXR extensions, and prints the -joint data each frame through ``DeviceIOSession``. The Python equivalent is -``examples/oxr/python/test_full_body_tracker.py``. Neither embeds -``CloudXRLauncher``, so start the runtime first and source its environment in -the shell that launches them: - -.. code-block:: bash - - python -m isaaccapture.cloudxr.service start # runs in the background - source ~/.cloudxr/run/cloudxr.env - ./install/examples/schemaio/full_body_printer - -See :ref:`dedicated-cloudxr-runtime` and -:ref:`load-cloudxr-environment-variables` for the full service setup. - -Troubleshooting -~~~~~~~~~~~~~~~ - -- **Body tracking does not work on a consumer PICO 4 Ultra.** WebXR body - tracking is an enterprise feature; consumer devices do not enable it by - default. Contact PICO about enterprise activation, then use the - **Activate enterprise account** option in the headset settings. -- **No body tracking data arrives on the server.** Confirm the headset has - enterprise features (see above). Verify all PICO motion trackers are - paired, powered on, and calibrated. Confirm the PICO browser is up to - date. -- **The printer prints** ``[body tracking inactive]``. The headset is - connected but body tracking is unavailable: check that the Motion Trackers - are paired and calibrated and that the headset has enterprise body-tracking - support. A printer that never reaches ``[Step 4] Reading samples...`` is - still waiting for the runtime / headset instead. -- **Some joints report** ``is_valid: false``. The PICO runtime may - temporarily lose tracking for individual joints during fast movement or - partial occlusion. These joints will recover automatically once tracking is - re-acquired. - -Recording and Replay --------------------- - -Full body sessions can be previewed live, captured to MCAP, and replayed -offline through the same retargeting pipeline — no headset required during -replay. See :doc:`../references/mcap_record_replay` for the API and the -``live_full_body.py`` / ``record_full_body.py`` / ``replay_full_body.py`` -example. - -Recording is also available directly from C++: pass a ``McapRecordingConfig`` -to ``DeviceIOSession::run()`` with the tracker mapped to the ``full_body`` -channel base name. ``examples/mcap_record_replay/cpp/record_full_body.cpp`` -demonstrates this — a file recorded there replays unchanged with -``replay_full_body.py``. Give it an output directory -(``record_full_body 5 recordings/``) and each run writes a fresh timestamped -take into it; ``replay_full_body.py`` run from the same directory picks the -newest one from ``./recordings/`` by default. - -.. figure:: ../_static/full-body-replay.gif - :alt: Full body skeleton replayed from an MCAP recording in viser - :class: no-image-zoom - - ``replay_full_body.py`` visualizing a recorded BD 24-joint skeleton in viser. - -Quest Body Tracking (Limited Support) -------------------------------------- - -Meta Quest headsets expose inside-out body tracking (IOBT) to WebXR sessions. -The CloudXR WebXR client will stream this data when the Quest browser grants the -``body-tracking`` feature. - -However, in the current version the Quest IOBT skeleton is mapped to the PICO BD -24-joint layout before being sent to the server. This means the server always -receives data in the ``XR_BD_body_tracking`` joint format regardless of the -source headset. - -.. note:: - - Quest body tracking does not require external trackers. It uses the - headset's built-in cameras. Tracking quality may differ from the - tracker-based PICO solution, particularly for lower-body joints. +This page has moved to :doc:`/device/body_tracking/index`. diff --git a/docs/source/device/body_tracking/index.rst b/docs/source/device/body_tracking/index.rst new file mode 100644 index 0000000000..a041f264a9 --- /dev/null +++ b/docs/source/device/body_tracking/index.rst @@ -0,0 +1,290 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Body Tracking +============= + +Isaac Teleop supports streaming full-body tracking data from an XR headset +through the CloudXR WebXR client to the teleop server. The server exposes the +body skeleton to applications through the ``FullBodyTracker`` tracker and +the OpenXR ``XR_BD_body_tracking`` extension. + +Body tracking support currently targets the **PICO 4 Ultra Enterprise** with +**PICO Motion Trackers**. The rest of this guide covers PICO hardware setup +and the BD skeleton format used on the server. + +.. important:: + + Full body tracking requires PICO's **enterprise features**. They are + available out of the box on the + `PICO 4 Ultra Enterprise `_ + headset. On a consumer PICO 4 Ultra, the browser does not grant the + ``body-tracking`` WebXR feature by default — no body data reaches the + server, even with Motion Trackers paired and calibrated. Consumer devices + can gain the feature through enterprise activation: contact PICO, then + use the **Activate enterprise account** option in the headset settings. + +.. note:: + + Meta Quest 3/3S also expose body tracking to WebXR via inside-out body + tracking (IOBT), and the CloudXR client will stream it. However the Quest + skeleton is mapped to the PICO BD 24-joint layout in the current version, + and accuracy is lower since there are no physical trackers. See + `Quest Body Tracking (Limited Support)`_ for details. + +.. seealso:: + + `GR00T-WholeBodyControl (SONIC)`_ can consume this CloudXR skeleton on a G1 robot as well. + See that project's setup guide for additional details. + +.. _`GR00T-WholeBodyControl (SONIC)`: https://nvlabs.github.io/GR00T-WholeBodyControl/tutorials/isaac_teleop_publisher_setup.html + +.. contents:: On this page + :local: + :depth: 2 + +Setting Up PICO Motion Trackers +------------------------------- + +Full body tracking uses +`PICO Motion Trackers `_ +paired with a PICO 4 Ultra Enterprise headset. Make sure the PICO browser is +updated to a version that supports body tracking. The following configurations +are supported: + +- **5 trackers:** best tracking quality across all 24 joints. + Two wear modes are available: 2 ankles + 2 wrists + 1 waist, or 2 ankles + + 2 thighs + 1 waist. +- **3 trackers:** 2 ankles + 1 waist. Upper-body joints are estimated from the + headset and controllers; lower-body tracking remains accurate. +- **2 trackers:** 2 ankles only. Provides lower-body tracking with upper-body + estimation. Least hardware, but reduced accuracy for waist and sitting/lying + postures. + +Instructions +~~~~~~~~~~~~ + +1. Tap the **clock / Wi-Fi / battery icon** in the lower-right corner of the + PICO menu. A picture of the headset appears; above it is a small circular + button with a wristband-and-tracker icon. If the button is missing, open the + **Motion Tracker** app directly. +2. Select a **wear mode** that matches the number of trackers you have and best + fits your use case. +3. Click the **Pair** button. +4. On each tracker, press and hold the top button until the lights start + flashing (pairing mode). +5. Strap the trackers on following the placement shown in the app for your + chosen wear mode. Scrunch down any baggy clothing so the trackers are visible + to the headset cameras. +6. Click **Calibrate** and follow the instructions in the Motion Tracker app. + +Before each session, confirm all trackers are powered on, paired, and +calibrated. + +How Full Body Tracking Works +---------------------------- + +Skeleton profile +~~~~~~~~~~~~~~~~ + +Body tracking data uses the +`XR_BD_body_tracking `_ +OpenXR extension profile defined by ByteDance (BD). This is the only body +tracking profile currently supported by the CloudXR runtime. + +When the CloudXR WebXR client starts a session, it requests the ``body-tracking`` +feature from the browser. Each frame, the CloudXR client library reads the body +joint data provided by the PICO runtime and streams it to the server. No +additional configuration is needed in the WebXR client; body tracking is +enabled automatically when the device supports it. + +Joint skeleton (24 joints) +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The BD profile defines a 24-joint skeleton. Each joint provides a position +(vec3 in meters), an orientation (quaternion), and a per-joint ``is_valid`` flag +indicating whether the tracker successfully resolved that joint in the current +frame. + +.. list-table:: + :header-rows: 1 + :widths: 10 25 25 + + * - Index + - Joint + - Parent + * - 0 + - Pelvis + - *(root)* + * - 1 + - Left Hip + - Pelvis + * - 2 + - Right Hip + - Pelvis + * - 3 + - Spine 1 + - Pelvis + * - 4 + - Left Knee + - Left Hip + * - 5 + - Right Knee + - Right Hip + * - 6 + - Spine 2 + - Spine 1 + * - 7 + - Left Ankle + - Left Knee + * - 8 + - Right Ankle + - Right Knee + * - 9 + - Spine 3 + - Spine 2 + * - 10 + - Left Foot + - Left Ankle + * - 11 + - Right Foot + - Right Ankle + * - 12 + - Neck + - Spine 3 + * - 13 + - Left Collar + - Spine 3 + * - 14 + - Right Collar + - Spine 3 + * - 15 + - Head + - Neck + * - 16 + - Left Shoulder + - Left Collar + * - 17 + - Right Shoulder + - Right Collar + * - 18 + - Left Elbow + - Left Shoulder + * - 19 + - Right Elbow + - Right Shoulder + * - 20 + - Left Wrist + - Left Elbow + * - 21 + - Right Wrist + - Right Elbow + * - 22 + - Left Hand + - Left Wrist + * - 23 + - Right Hand + - Right Wrist + +Server-side access +~~~~~~~~~~~~~~~~~~ + +On the server, body tracking data is consumed through the +``FullBodyTracker`` tracker (see :doc:`../trackers` for the full tracker +reference). The tracker exposes a ``get_body_pose()`` method that returns the +24-joint skeleton each frame (or null when body tracking is not available). +Joint data follows the ``FullBodyPose`` FlatBuffers schema defined in +``src/core/schema/fbs/full_body.fbs``. + +The ``all_joint_poses_tracked`` quality flag indicates whether every joint was +successfully tracked in the current frame. When it is false, consult individual +joint ``is_valid`` flags to determine which joints have valid poses. + +For a minimal C++ reader see ``examples/schemaio/full_body_printer.cpp``, which +creates the tracker, queries the required OpenXR extensions, and prints the +joint data each frame through ``DeviceIOSession``. The Python equivalent is +``examples/oxr/python/test_full_body_tracker.py``. Neither embeds +``CloudXRLauncher``, so start the runtime first and source its environment in +the shell that launches them: + +.. code-block:: bash + + python -m isaaccapture.cloudxr.service start # runs in the background + source ~/.cloudxr/run/cloudxr.env + ./install/examples/schemaio/full_body_printer + +See :ref:`dedicated-cloudxr-runtime` and +:ref:`load-cloudxr-environment-variables` for the full service setup. + +Troubleshooting +~~~~~~~~~~~~~~~ + +- **Body tracking does not work on a consumer PICO 4 Ultra.** WebXR body + tracking is an enterprise feature; consumer devices do not enable it by + default. Contact PICO about enterprise activation, then use the + **Activate enterprise account** option in the headset settings. +- **No body tracking data arrives on the server.** Confirm the headset has + enterprise features (see above). Verify all PICO motion trackers are + paired, powered on, and calibrated. Confirm the PICO browser is up to + date. +- **The printer prints** ``[body tracking inactive]``. The headset is + connected but body tracking is unavailable: check that the Motion Trackers + are paired and calibrated and that the headset has enterprise body-tracking + support. A printer that never reaches ``[Step 4] Reading samples...`` is + still waiting for the runtime / headset instead. +- **Some joints report** ``is_valid: false``. The PICO runtime may + temporarily lose tracking for individual joints during fast movement or + partial occlusion. These joints will recover automatically once tracking is + re-acquired. + +Recording and Replay +-------------------- + +Full body sessions can be previewed live, captured to MCAP, and replayed +offline through the same retargeting pipeline — no headset required during +replay. See :doc:`../../references/mcap_record_replay` for the API and the +``live_full_body.py`` / ``record_full_body.py`` / ``replay_full_body.py`` +example. + +Recording is also available directly from C++: pass a ``McapRecordingConfig`` +to ``DeviceIOSession::run()`` with the tracker mapped to the ``full_body`` +channel base name. ``examples/mcap_record_replay/cpp/record_full_body.cpp`` +demonstrates this — a file recorded there replays unchanged with +``replay_full_body.py``. Give it an output directory +(``record_full_body 5 recordings/``) and each run writes a fresh timestamped +take into it; ``replay_full_body.py`` run from the same directory picks the +newest one from ``./recordings/`` by default. + +.. figure:: ../../_static/full-body-replay.gif + :alt: Full body skeleton replayed from an MCAP recording in viser + :class: no-image-zoom + + ``replay_full_body.py`` visualizing a recorded BD 24-joint skeleton in viser. + +Quest Body Tracking (Limited Support) +------------------------------------- + +Meta Quest headsets expose inside-out body tracking (IOBT) to WebXR sessions. +The CloudXR WebXR client will stream this data when the Quest browser grants the +``body-tracking`` feature. + +However, in the current version the Quest IOBT skeleton is mapped to the PICO BD +24-joint layout before being sent to the server. This means the server always +receives data in the ``XR_BD_body_tracking`` joint format regardless of the +source headset. + +.. note:: + + Quest body tracking does not require external trackers. It uses the + headset's built-in cameras. Tracking quality may differ from the + tracker-based PICO solution, particularly for lower-body joints. + +SOMA pose transport +------------------- + +For body and hand data from vendor adapters, see the SOMA transport guide. + +.. toctree:: + :maxdepth: 1 + + soma diff --git a/docs/source/device/body_tracking/soma.rst b/docs/source/device/body_tracking/soma.rst new file mode 100644 index 0000000000..b0349a181b --- /dev/null +++ b/docs/source/device/body_tracking/soma.rst @@ -0,0 +1,245 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +SOMA Pose Transport +=================== + +Isaac Capture can transport SOMA body and hand data through FlatBuffers and +``SchemaPusher``. Producers publish either joint rotations or evaluated joint +poses. Consumer-side sources normalize both representations to global joint +poses for retargeting or visualization. + +The schemas and examples are defined and verified against the SOMA-X v0.3.1 +pose interface. See the `SOMA-X repository `_ +for the upstream model and assets. + +.. contents:: On this page + :local: + :depth: 2 + +Data flow +--------- + +Body and hand data use separate schemas and collections. Applications can +select their body and hand sources independently. + +.. code-block:: text + + vendor body SDK -> SOMA body FlatBuffer -> SchemaPusher -> SOMA body tracker + vendor hand SDK -> SOMA hand FlatBuffer -> SchemaPusher -> SOMA hand tracker + +The body schema contains SOMA's complete body control set, including its hand +controls. The standalone hand schemas provide independent left and right hand +streams, identified by ``hand_side`` (``SomaHandSide.LEFT`` or +``SomaHandSide.RIGHT``). Isaac Capture does not automatically fuse standalone +hands with the hand controls in a body payload. + +Choose a representation +----------------------- + +Use the representation closest to the producer's native data. + +.. list-table:: + :header-rows: 1 + :widths: 20 30 50 + + * - Representation + - FlatBuffer types + - Behavior + * - ``joint-rotations`` + - ``SomaBodyJointRotations`` and ``SomaHandJointRotations`` + - Publishes SOMA joint rotations and the global Hips or Wrist translation. + The consumer evaluates forward kinematics against a prepared SOMA + identity. + * - ``joint-poses`` + - ``SomaBodyJointPoses`` and ``SomaHandJointPoses`` + - Publishes global position and orientation for each provided joint. The + consumer maps these poses directly and does not run per-frame forward + kinematics. + +``joint-rotations`` most directly matches the SOMA-X pose-control interface. +``joint-poses`` is an Isaac Capture transport profile for producers that already +have evaluated global joint transforms. + +Both representations use positions in meters and unit XYZW quaternions. They +use a right-handed reference frame with positive Y up and positive Z forward. +The publisher must convert its native skeleton and reference frame before +serialization. See :ref:`tracker-reference` for the complete field and +validity contracts. + +Consumers normalize finite quaternions whose norm is greater than ``1e-12``. A +zero, degenerate, or non-finite quaternion, position, or global translation is +unavailable. Quaternion signs are not canonicalized, so ``q`` and ``-q`` remain +equivalent. + +Rotation consumers resolve the SOMA v0.3.1 reference through SOMA-X and pass it +explicitly during forward kinematics. The prepared identity and subject scale +remain caller-selected calibration and are not constrained by this check. + +Body and hand joint values are sorted, unique keyed vectors. Every entry carries +an explicit ``SomaBodyJoint`` or ``SomaHandJoint`` identifier. A missing joint is +unavailable for that frame; producers do not serialize placeholder values for +joints they cannot provide. Consumers expand the keyed values into SOMA-X's +canonical dense order when invoking its body or hand helpers. + +Build the examples +------------------ + +Start with a configured Isaac Capture checkout. See +:doc:`../../development/build_from_source/index` for the complete build +prerequisites. Install the live viewer and both deterministic publishers: + +.. code-block:: bash + + uv pip install -e ./examples/deviceio_live_view + uv pip install -e ./examples/soma_body_publisher + uv pip install -e ./examples/soma_hand_publisher + + .venv/bin/cmake -S . -B build + .venv/bin/cmake --build build \ + --target isaacteleop_python \ + -j4 + +The example packages install ``py-soma-x==0.3.1``. +The publishers use the Python-accessible ``TensorPushTracker`` and its native +``SchemaPusher`` transport; no separate sender executable is needed. Demo samples +use the session's monotonic clock for both timestamps. + +Run the body example +-------------------- + +Start the viewer first. It launches the CloudXR runtime and prints the viewer +URL. Open that URL before starting the publisher. On a remote host, replace +``localhost`` in the URL with the host name or IP address. + +.. code-block:: bash + + uv run --no-sync python -m isaaccapture_examples.deviceio_live_view \ + --accept-eula \ + --body-schema soma + +In another terminal, load the runtime environment and start the body +publisher: + +.. code-block:: bash + + source ~/.cloudxr/run/cloudxr.env + uv run --no-sync python -m isaaccapture_examples.soma_body_publisher \ + --loop + +These commands use ``joint-rotations``. To publish evaluated joint poses, add +``--body-representation joint-poses`` to the publisher and +``--soma-body-representation joint-poses`` to the viewer. + +Run the hand example +-------------------- + +Start the viewer with SOMA hands: + +.. code-block:: bash + + uv run --no-sync python -m isaaccapture_examples.deviceio_live_view \ + --accept-eula \ + --hand-schema soma + +In another terminal, load the runtime environment and start the hand +publisher: + +.. code-block:: bash + + source ~/.cloudxr/run/cloudxr.env + uv run --no-sync python -m isaaccapture_examples.soma_hand_publisher \ + --loop + +These commands use ``joint-rotations``. To publish evaluated joint poses, add +``--hand-representation joint-poses`` to the publisher and +``--soma-hand-representation joint-poses`` to the viewer. + +Collection IDs and schema identifiers +------------------------------------- + +A collection ID pairs a producer with a consumer and is not encoded in the +FlatBuffer payload. The tensor identifier selects the representation carried by +that collection. The demo publishers use these fixed collection IDs; another +producer can choose different IDs as long as its consumers use the same values. + +.. list-table:: + :header-rows: 1 + :widths: 30 30 40 + + * - Data + - Collection ID + - Viewer override + * - Body + - ``soma_body_demo`` + - ``--soma-body-collection-id`` + * - Left hand + - ``soma_hand_left_demo`` + - ``--soma-left-hand-collection-id`` + * - Right hand + - ``soma_hand_right_demo`` + - ``--soma-right-hand-collection-id`` + +The representation selected by the publisher and consumer must match. The +transport identifiers are ``soma_body_joint_rotations``, +``soma_body_joint_poses``, ``soma_hand_joint_rotations``, and +``soma_hand_joint_poses``. + +Vendor integration +------------------ + +A vendor integration converts its native skeleton to one of the checked-in +SOMA contracts and publishes it through ``SchemaPusher``. Use these schemas as +the serialization source: + +- :code-file:`src/core/schema/fbs/soma_common.fbs` +- :code-file:`src/core/schema/fbs/soma_body_common.fbs` +- :code-file:`src/core/schema/fbs/soma_body_joint_rotations.fbs` +- :code-file:`src/core/schema/fbs/soma_body_joint_poses.fbs` +- :code-file:`src/core/schema/fbs/soma_hand_joint_rotations.fbs` +- :code-file:`src/core/schema/fbs/soma_hand_joint_poses.fbs` + +The FlatBuffer transport and generated trackers do not depend on SOMA-X or +PyTorch. A rotation consumer uses SOMA-X and its resolved neutral identity to +evaluate forward kinematics. A joint-pose consumer does not need SOMA-X for +per-frame evaluation. + +Current limitations +------------------- + +- The examples evaluate rotations against a neutral identity. They do not + perform subject calibration. +- Body and standalone hand streams are not fused automatically. +- A producer and consumer must select the same representation. +- The live viewer selects sources at launch. It does not switch schemas while + running. + +Troubleshooting +--------------- + +No module named ``isaaccapture_examples`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Install the corresponding example with ``uv pip install -e`` and launch it +with ``uv run --no-sync`` from the repository root. + +The publisher cannot connect +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Start the viewer, open its client, and then source +``~/.cloudxr/run/cloudxr.env`` in the publisher terminal. The environment file +identifies the active CloudXR OpenXR runtime. + +The viewer receives no skeleton +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Confirm that the publisher and viewer use the same representation and +collection names. + +See also +-------- + +- :code-file:`examples/soma_body_publisher/README.md` +- :code-file:`examples/soma_hand_publisher/README.md` +- :code-file:`examples/deviceio_live_view/README.md` +- :doc:`../trackers` diff --git a/docs/source/device/cameras/index.rst b/docs/source/device/cameras/index.rst new file mode 100644 index 0000000000..e94ec7b3d2 --- /dev/null +++ b/docs/source/device/cameras/index.rst @@ -0,0 +1,14 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Cameras +======= + +Camera plugins publish image and frame data through DeviceIO. For transmitting +camera feeds to clients, see :doc:`/references/camera_streaming`. + +.. toctree:: + :maxdepth: 1 + :titlesonly: + + oak diff --git a/docs/source/device/cameras/oak.rst b/docs/source/device/cameras/oak.rst new file mode 100644 index 0000000000..a302179114 --- /dev/null +++ b/docs/source/device/cameras/oak.rst @@ -0,0 +1,280 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +OAK Camera Plugin +================= + +C++ plugin that captures H.264 video from OAK cameras and saves to raw H.264 +files. Source and plugin README: +:code-file:`src/plugins/oak/README.md`. + +.. contents:: On this page + :local: + :depth: 2 + +Features +-------- + +- **Hardware H.264 Encoding** — Uses OAK's built-in video encoder +- **Raw H.264 Recording** — Writes H.264 NAL units directly to file (no + container overhead) +- **OpenXR Integration** — Optional CloudXR runtime integration +- **Self-contained build** — DepthAI built automatically via CMake + +Build +----- + +DepthAI v3.x is fetched and built automatically via FetchContent. Its transitive +dependencies are managed by **vcpkg**: rather than tracking our own ``vcpkg.json``, +the build downloads **DepthAI's own vcpkg manifest** for the pinned tag at configure +time (see ``cmake/DepthAIVcpkgManifest.cmake``) and enables its ``usb`` feature for +libusb. The first build takes ~10–15 minutes (mostly vcpkg deps + DepthAI); +subsequent builds are fast. + +Prerequisites +~~~~~~~~~~~~~ + +.. code-block:: bash + + # Install build tools for vcpkg to build libusb (Linux) + sudo apt install libudev-dev autoconf automake libtool pkg-config + + # Install vcpkg (one-time) + git clone https://github.com/microsoft/vcpkg ~/.vcpkg + ~/.vcpkg/bootstrap-vcpkg.sh + echo 'export VCPKG_ROOT=~/.vcpkg' >> ~/.bashrc && source ~/.bashrc + +Configure and Build +~~~~~~~~~~~~~~~~~~~ + +.. warning:: + + ``CMAKE_TOOLCHAIN_FILE`` is a CMake constraint: it is only read on the + **first** configure of a build tree and is silently ignored on any + re-configure. **If** ``build/`` **already exists, delete it before running + the commands below**, otherwise the vcpkg toolchain will not be picked up and + configure will fail. + +.. code-block:: bash + + cd IsaacCapture + + rm -rf build/ # required if build/ already exists or was previously configured without CMAKE_TOOLCHAIN_FILE + cmake -B build -DBUILD_PLUGIN_OAK_CAMERA=ON \ + -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake + cmake --build build --target camera_plugin_oak --parallel + +Usage +----- + +.. code-block:: bash + + # Record a single color stream + ./build/src/plugins/oak/camera_plugin_oak --add-stream=camera=Color,output=./color.h264 + + # Record multiple streams + ./build/src/plugins/oak/camera_plugin_oak \ + --add-stream=camera=Color,output=./color.h264 \ + --add-stream=camera=MonoLeft,output=./left.h264 \ + --add-stream=camera=MonoRight,output=./right.h264 + + # Record with a live preview window + ./build/src/plugins/oak/camera_plugin_oak \ + --add-stream=camera=Color,output=./color.h264 --preview + + # Show help + ./build/src/plugins/oak/camera_plugin_oak --help + +Press ``Ctrl+C`` to stop recording. + +Record camera metadata in MCAP +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Each captured frame emits metadata (sequence number, timestamps) as FlatBuffer +``FrameMetadataOak`` messages. There are two ways to record it — +``--collection-prefix`` and ``--mcap-filename`` are **mutually exclusive**. + +Local MCAP file (``--mcap-filename``) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The plugin writes per-stream metadata directly to an MCAP file. No host-side +tracker or TeleopSession is required. + +.. code-block:: bash + + ./build/src/plugins/oak/camera_plugin_oak \ + --add-stream=camera=Color,output=./color.h264 \ + --add-stream=camera=MonoLeft,output=./left.h264 \ + --mcap-filename=./metadata.mcap + +TeleopSession (multi-device recording) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Record camera metadata in the same MCAP file as other teleop data (hands, head, +controllers, etc.): + +1. **Plugin** — launch ``oak_camera`` via PluginManager or + :doc:`TeleopSession <../../teleop/teleop_session>`'s ``PluginConfig``, + passing ``--collection-prefix`` so the plugin pushes metadata via OpenXR + ``SchemaPusher``. +2. **Host trackers** — create one ``FrameMetadataTrackerOak`` **per stream**, + each with collection id ``{collection_prefix}/{StreamName}`` using the + **same** prefix the plugin was given (see :doc:`../trackers`). TeleopSession's + DeviceIO layer uses the live tracker implementation to read the pushed + tensors and write MCAP channels. +3. **MCAP config** — add every tracker to both ``TeleopSessionConfig.trackers`` + and ``McapRecordingConfig.tracker_names``, each under its own base name. See + also :doc:`../../references/mcap_record_replay`. + +.. code-block:: python + + from pathlib import Path + + from isaaccapture.deviceio import FrameMetadataTrackerOak, McapRecordingConfig + from isaaccapture.teleop_session_manager import PluginConfig, TeleopSession, TeleopSessionConfig + + PLUGIN_ROOT = Path("build/src/plugins") # or your installed plugin search path + COLLECTION_PREFIX = "oak_camera" + STREAM_NAMES = ["Color", "MonoLeft"] + + # One tracker per stream, each on the collection the plugin publishes it under. + oak_trackers = { + name: FrameMetadataTrackerOak(f"{COLLECTION_PREFIX}/{name}") + for name in STREAM_NAMES + } + + config = TeleopSessionConfig( + app_name="OakTeleop", + pipeline=pipeline, # your retargeting pipeline + trackers=list(oak_trackers.values()), + mcap_config=McapRecordingConfig( + "recording.mcap", + [(tracker, f"oak_metadata_{name.lower()}") for name, tracker in oak_trackers.items()], + ), + plugins=[ + PluginConfig( + plugin_name="oak_camera", + plugin_root_id="oak_camera", + search_paths=[PLUGIN_ROOT], + plugin_args=[ + "--add-stream=camera=Color,output=./color.h264", + "--add-stream=camera=MonoLeft,output=./left.h264", + f"--collection-prefix={COLLECTION_PREFIX}", + ], + ), + ], + ) + + with TeleopSession(config) as session: + while True: + session.step() # metadata is recorded each update + +See also :code-file:`examples/oxr/python/test_oak_camera.py` (``--mode +schema-pusher``) for a standalone PluginManager + DeviceIOSession example of +the same SchemaPusher flow. + +Configuration +------------- + +.. list-table:: + :widths: 30 15 55 + :header-rows: 1 + + * - Option + - Default + - Description + * - ``--add-stream=camera=,output=`` + - (at least one required) + - Add a capture stream. ``camera`` is one of ``Color``, ``MonoLeft``, + ``MonoRight``. Repeatable. + * - ``--fps=N`` + - 30 + - Frame rate for all streams + * - ``--bitrate=N`` + - 8000000 + - H.264 bitrate (bps) + * - ``--quality=N`` + - 80 + - H.264 quality (1–100) + * - ``--device-id=ID`` + - first available + - OAK device MxId + * - ``--preview`` + - off + - Open a live SDL2 window showing the color camera feed + * - ``--collection-prefix=PREFIX`` + - + - Push per-frame metadata via OpenXR tensor extensions + * - ``--mcap-filename=PATH`` + - + - Record per-frame metadata to an MCAP file + +Architecture +------------ + +.. code-block:: text + + ┌─────────────────┐ ┌──────────────────┐ ┌───────────────┐ ┌──────────────┐ + │ OakCamera │────>│ FrameSink │────>│ RawDataWriter │────>│ .h264 File │ + │ (H.264 encode) │ │ (write + push) │ │ (file writer) │ │ │ + └─────────────────┘ └──────┬───────────┘ └───────────────┘ └──────────────┘ + core/ │ core/ core/ + v + ┌──────────────────┐ + │ MetadataPusher │ + │ (OpenXR tensor) │ + └──────────────────┘ + +Dependencies +------------ + +Transitive dependencies via **vcpkg**, DepthAI and SDL2 via FetchContent: + +- **DepthAI v3.x** — OAK camera interface (FetchContent) +- **SDL2** — Live preview window (FetchContent, used by ``--preview``) +- **vcpkg** — DepthAI's transitive deps (nlohmann-json, spdlog, libusb, + libarchive, etc.), resolved from DepthAI's own auto-downloaded manifest + (``usb`` feature enabled for libusb) + +Output Format +------------- + +The plugin writes raw H.264 NAL units (Annex B format) to ``.h264`` files. To +play or convert: + +.. code-block:: bash + + # Play with ffplay + ffplay -f h264 recording.h264 + + # Convert to MP4 + ffmpeg -f h264 -i recording.h264 -c copy output.mp4 + + # Convert with specific framerate + ffmpeg -f h264 -framerate 30 -i recording.h264 -c copy output.mp4 + +Troubleshooting +--------------- + +Device USB Connection +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + # Check OAK camera connection + lsusb | grep 03e7 + +For more connection troubleshooting, see the +`OAK USB deployment guide `_. + +Inspect the Recording +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + # Verify recording (convert to MP4 first) + ffmpeg -f h264 -i recording.h264 -c copy recording.mp4 + ffprobe recording.mp4 + + # Check frame count + ffprobe -show_entries stream=nb_frames recording.mp4 diff --git a/docs/source/device/gamepads/index.rst b/docs/source/device/gamepads/index.rst new file mode 100644 index 0000000000..bcd8b5c0dd --- /dev/null +++ b/docs/source/device/gamepads/index.rst @@ -0,0 +1,13 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Gamepads +======== + +Gamepads provide button, stick, and trigger input through the DeviceIO path. + +.. toctree:: + :maxdepth: 1 + :titlesonly: + + /device/gamepad diff --git a/docs/source/device/hand_tracking/haptikos.rst b/docs/source/device/hand_tracking/haptikos.rst new file mode 100644 index 0000000000..94d535e38f --- /dev/null +++ b/docs/source/device/hand_tracking/haptikos.rst @@ -0,0 +1,128 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Haptikos Exoskeletons +===================== + +.. raw:: html + +
+ +.. image:: ../../_static/haptikos-exoskeletons.gif + :alt: Haptikos exoskeletons used with Isaac Teleop + :class: align-center + :width: 100% + +.. raw:: html + +
+ +Use the `Haptikos `_ Exoskeletons with the Isaac Capture framework. Currently only Linux is supported. Tested on Meta Quest headsets. Other headsets with controllers may work as well. + +.. contents:: On this page + :local: + :depth: 2 + +Components +---------- + +- **Core library** (``haptikos_core``) — wraps the Haptikos Core App(``libHaptikosCore.so``) and provides access to per-joint tracking data and haptics. +- **Plugin executable** (``haptikos_plugin``) — the primary plugin binary that enables integration with the Teleop system via CloudXR and OpenXR. + +Prerequisites +------------- +- **Linux** — x86_64 (tested on Ubuntu 22.04 / 24.04). +- **Haptikos API** for Linux. + + +Quick Start +----------- + +Step 1: Get the Haptikos API +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The Haptikos API must be downloaded separately due to licensing. Check our `website `_. + +1. Obtain a Haptikos account. +2. Clone the `repository `_. +3. Copy the ``HaptikosCpp_API_Shared`` folder from our HaptikosAPI repository into the ``src/plugins/haptikos`` folder. The folder structure should be the following: + +.. code-block:: text + + src/plugins/haptikos/ + ├── CMakeLists.txt + ├── HaptikosCpp_API_Shared + │ ├── include + │ └── lib + ├── haptikos_hands_plugin.cpp + ├── haptikos_hands_plugin.hpp + ├── main.cpp + ├── plugin.yaml + └── README.md + +Step 2: Build the plugin +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + cd ../../.. # Navigate to TeleopCore root + cmake -S . -B build + cd build + make haptikos_hands_plugin + +Step 3: Build the entire project +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. code-block:: bash + + cmake -B build -DENABLE_CLANG_FORMAT_CHECK=OFF #From Project root + cmake --build build --parallel 4 + cmake --install build + +We use the ``-DENABLE_CLANG_FORMAT_CHECK=OFF`` because the headers in the HaptikosAPI library are not clang-formatted. You can clang format the headers to not include the flag by running: + +.. code-block:: bash + + clang-format -i src/plugins/haptikos/HaptikosCpp_API_Shared/include/* + +Step 4: Run the plugin +~~~~~~~~~~~~~~~~~~~~~~ + +The Haptikos plugin connects to the Teleop session through the CloudXR / OpenXR runtime, so the runtime must be running and its environment sourced in the shell that launches the plugin. + +Start the CloudXR runtime from the Python environment where ``isaaccapture`` is installed. It runs as a background service that outlives this shell, until you stop it with ``python -m isaaccapture.cloudxr.service stop``: + +.. code-block:: bash + + python -m isaaccapture.cloudxr.service start --accept-eula + +In the shell that runs the plugin, source the environment file that the runtime writes on startup. This points the OpenXR loader at CloudXR: + +.. code-block:: bash + + cd IsaacCapture/ + source ~/.cloudxr/run/cloudxr.env + +In the same terminal, run the plugin with the following: + +.. code-block:: bash + + ./install/plugins/haptikos/haptikos_hands_plugin + +Important Information +--------------------- + +1. To use the plugin properly it is necessary to attach the controllers on our exoskeletons using the included mount. + +2. The orientation calibration defines the direction the Haptikos gloves define as forward. This needs to be aligned with the Head Mounted Display's forward direction. + +3. To inject the hand tracking data and haptic feedback, the controllers, the Haptikos App and the exoskeletons need to be active. + +4. The executable will be located in the ``build/src/plugins/haptikos``. If you installed the project by running ``cmake --install build`` the executable will also be found in the ``install/plugins/haptikos`` folder. + + +License +------- + +Source files are covered by their stated licenses (Apache-2.0). The Haptikos API is +proprietary to Haptikos and is subject to its own license; it is **not** redistributed +by this project. diff --git a/docs/source/device/hand_tracking/index.rst b/docs/source/device/hand_tracking/index.rst new file mode 100644 index 0000000000..21a38a3374 --- /dev/null +++ b/docs/source/device/hand_tracking/index.rst @@ -0,0 +1,17 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Hand tracking +============= + +Hand tracking includes native XR hand poses and vendor devices such as gloves +and hand exoskeletons. See :doc:`/device/trackers` for the common tracking API. + +.. toctree:: + :maxdepth: 1 + :titlesonly: + + manus + wuji_glove + sharpa_avatar + haptikos diff --git a/docs/source/device/hand_tracking/manus.rst b/docs/source/device/hand_tracking/manus.rst new file mode 100644 index 0000000000..f39b53d927 --- /dev/null +++ b/docs/source/device/hand_tracking/manus.rst @@ -0,0 +1,330 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +MANUS Gloves +============ + +A Linux-only plugin for integrating `MANUS `_ gloves +into the Isaac Capture framework. It provides full hand-joint tracking via the +Manus SDK and injects the resulting poses into the OpenXR hand-tracking layer so +any downstream retargeter can consume them transparently. Optionally it also +publishes Manus flex-sensor (RawDeviceData) tip poses as ``JointSe3PoseOutput`` +tensors and consumes inbound haptic commands for vibration gloves. + +.. contents:: On this page + :local: + :depth: 2 + +Components +---------- + +- **Core library** (``manus_plugin_core``) — wraps the Manus SDK + (``libIsaacTeleopPluginsManus.so``) and exposes per-joint tracking data. +- **Plugin executable** (``manus_hand_plugin``) — the main plugin binary that + integrates with the Teleop system via CloudXR / OpenXR. +- **CLI tool** (``manus_hand_tracker_printer``) — a standalone diagnostic tool + that prints tracked joint data to the terminal and opens a real-time + **MANUS Data Visualizer** window showing the hand skeleton from two orthographic + views per hand. + +Prerequisites +------------- + +- **Linux** — x86_64 or aarch64 (tested on Ubuntu 22.04 / 24.04). +- **Manus SDK** for Linux — downloaded automatically by the install script. +- **System dependencies** — the install script installs required packages automatically. + +Installation +------------ + +MANUS access has two halves: **device permissions** (kernel/udev, lives on the +host) and **SDK + plugin build** (lives wherever you build, typically a +container). The two scripts below split along that line. + +Step 1: grant the host access to the Manus dongle (one-time) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Run this **on the host machine**, not inside a container. udev rules are +processed by ``systemd-udevd``, which does not run inside Docker — so +installing rules from a container has no effect. + +.. code-block:: bash + + cd src/plugins/manus + ./install_udev_rules.sh + # then unplug + replug the Manus dongle + +If you're using the Isaac ROS dev container (``isaac_ros run_dev``), it +bind-mounts ``/dev/bus/usb`` from the host, so once the host has the rules +applied the container will see the dongle with the right permissions. + +Step 2: build the SDK and plugin +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Run this **inside the build environment** (devcontainer or Isaac ROS container): + +.. code-block:: bash + + cd src/plugins/manus + ./install_manus.sh + +Pass ``--build-dir `` to reuse a non-default top-level CMake build +directory. + +The script will: + +1. Install the required system packages for MANUS Core Integrated. +2. Download MANUS SDK v3.2.1. +3. Extract and place the SDK for the host architecture in the correct location. +4. Build the plugin and the diagnostic tool. + +When run inside a container, ``install_manus.sh`` skips the udev step and +reminds you to run ``install_udev_rules.sh`` on the host. Pass ``--container`` +when the build environment does not expose standard container markers, such as +during a Docker BuildKit build. + +Manual installation (fallback only) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Steps 1 and 2 above are the supported path — most users should never need +this section. Only follow it if ``install_manus.sh`` fails (for example, the +SDK download is unreachable from your network) and you need to place the SDK +by hand. + +1. Download the MANUS Core SDK from + `MANUS Downloads `_. +2. From ``C++/SDKClient/ManusSDK`` in the archive, place ``include/`` and the + library for your architecture (``lib/amd64/libManusSDK-amd64.so`` or + ``lib/aarch64/libManusSDK-aarch64.so``, renamed to ``lib/libManusSDK.so``) + into a ``ManusSDK`` folder inside ``src/plugins/manus/``, or point CMake at a + different path by setting ``MANUS_SDK_ROOT``. +3. Follow the + `MANUS Getting Started guide for Linux `_ + to install the dependencies and configure device permissions. + +Expected directory layout after placing the SDK: + +.. code-block:: text + + src/plugins/manus/ + app/ + main.cpp + core/ + manus_hand_tracking_plugin.cpp + inc/ + manus/ + manus_glove_collection.hpp + manus_hand_tracking_plugin.hpp + tools/ + manus_hand_tracker_printer.cpp + ManusSDK/ <-- placed here + include/ + lib/ + libManusSDK.so + +Then build from the root: + +.. code-block:: bash + + cd ../../.. # navigate to root + cmake -S . -B build + cmake --build build --target manus_hand_plugin manus_hand_tracker_printer -j + cmake --install build --component manus + +Running the Plugin +------------------ + +1. Start the CloudXR runtime and load its environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The MANUS plugin connects to the Teleop session through the CloudXR / OpenXR +runtime, so the runtime must be running and its environment sourced in the +shell that launches the plugin. + +Start the CloudXR runtime. It runs as a background service that outlives +this shell, until you stop it with ``python -m isaaccapture.cloudxr.service stop``: + +.. code-block:: bash + + python -m isaaccapture.cloudxr.service start + +In the shell you will use to run the plugin, source the environment file +that the runtime writes on startup. This points the OpenXR loader at CloudXR: + +.. code-block:: bash + + source ~/.cloudxr/run/cloudxr.env + +See :ref:`dedicated-cloudxr-runtime` for the full CloudXR service setup, +including EULA acceptance, and :ref:`whitelist-firewall-ports` in the Quick +Start for firewall configuration. + +2. Verify with the CLI tool +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Verify that the gloves are working using the CLI tool. It runs from the build +tree, while the plugin in the next step runs from the install tree; both come +from the same build: + +.. code-block:: bash + + ./build/bin/manus_hand_tracker_printer + +The tool prints joint positions to the terminal and opens a **MANUS Data +Visualizer** window showing a top-down and side view of each hand. + +3. Run the plugin +~~~~~~~~~~~~~~~~~~ + +The plugin is installed to the ``install`` directory. Ensure the CLI tool is +not running when you launch the plugin — only one process can hold the Manus +SDK connection at a time. + +.. code-block:: bash + + ./install/plugins/manus/manus_hand_plugin + +By default the plugin enables human hand injection, flex-sensor push, and +haptic read. Restrict datasets with ``--datasets=`` (comma-separated): + +.. code-block:: bash + + ./install/plugins/manus/manus_hand_plugin --datasets=human,sensors,haptic + ./install/plugins/manus/manus_hand_plugin --datasets=human # skeleton only + ./install/plugins/manus/manus_hand_plugin --datasets=sensors # flex sensors only + +Data paths +---------- + +Human (OpenXR hand injection) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The Manus raw skeleton stream is mapped to 26 OpenXR hand joints and pushed +through ``HandInjector`` (requires ``XR_NVX1_device_interface_base``). Downstream +hosts consume hands via ``HandsSource`` as with any other hand-tracking plugin. + +Sensors (flex tips via SchemaPusher) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When gloves report at least five RawDeviceData flex sensors, the plugin pushes +a ``JointSe3PoseOutput`` (tensor id ``joint_se3_pose``) on: + +- ``manus_sensors_left`` +- ``manus_sensors_right`` + +Each frame is stamped ``type = JointType.HAND_RAW`` and carries the five +fingertips keyed by ``JointName`` +(``HAND_RAW_THUMB_TIP`` .. ``HAND_RAW_LITTLE_TIP``), position in meters and +orientation as an xyzw quaternion, in the Manus SDK frame after the plugin's +VUH coordinate setup. A joint absent from a frame is not tracked. Poses are raw +Manus flex transforms; hosts that mask against a re-framed skeleton must apply +their own sensor-pose processing. + +This path requires CloudXR tensor **push** extensions +(``XR_NVX1_push_tensor`` and ``XR_NVX1_tensor_data``). + +Host-side consumption example: + +.. code-block:: python + + from isaaccapture.deviceio_trackers import JointSe3PoseTracker + from isaaccapture.schema import JointName, JointType + + left = JointSe3PoseTracker("manus_sensors_left") + # ... register with a DeviceIOSession, then each tick: + data = left.get_data(session).data + assert data is None or data.type == JointType.HAND_RAW + thumb = data.lookup(JointName.HAND_RAW_THUMB_TIP) if data else None + if thumb is not None: + position, orientation = thumb.pose.position, thumb.pose.orientation + +See :code-file:`examples/mcap_record_replay/python/isaaccapture_examples/mcap_record_replay/record_joint_se3_pose.py` for a +runnable recorder. Run ``python -m isaaccapture_examples.mcap_record_replay.replay_joint_se3_pose`` +to play one back without hardware. + +Haptic (inbound vibration) +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The plugin reads ``HapticCommand`` on collection ``manus_glove_haptic`` +(``XR_NVX1_tensor_data``) and drives five finger motors via the Manus SDK. +See the haptic feedback example and +``isaaccapture.haptic_devices.glove.haptic_glove_device``. + +Wrist Positioning — Controllers vs Optical Hand Tracking +--------------------------------------------------------- + +Two sources are available for positioning the MANUS gloves in 3D space: + +- **Controller adapters** — attach Quest 3 controllers to the MANUS Universal + Mount on the back of the glove. The controller pose drives wrist placement. +- **Optical hand tracking** — use the HMD's built-in optical hand tracking to + position the hands. No physical controller adapter required. + +The plugin selects the source automatically at runtime: optical hand tracking is +preferred when ``XR_MNDX_xdev_space`` is supported and the runtime reports an +actively tracked wrist pose; otherwise it falls back to the controller pose. + +.. note:: + + When using controller adapters it is advisable to disable the HMD's automatic + hand-tracking–to–controller switching to avoid unexpected source changes + mid-session. + +Troubleshooting +--------------- + +.. list-table:: + :widths: 40 60 + :header-rows: 1 + + * - Symptom + - Resolution + * - SDK download fails + - Check your internet connection and re-run the install script. + * - ``uv not found. Please install uv: ...`` + - Install uv and re-run: ``curl -LsSf https://astral.sh/uv/install.sh | sh``. + It installs to ``~/.local/bin``, so also add that to ``PATH`` for the + current shell (and ``~/.bashrc``, to make it stick): + ``export PATH="$HOME/.local/bin:$PATH"``. + * - ``Missing build tools: ... patchelf ...`` + - CMake's dependency preflight lists every missing tool and the exact + install command in the same message, e.g. + ``sudo apt-get install -y patchelf``. Run the printed command and + re-run the install script. + * - Manus SDK not found at build time + - With manual installation, ensure ``ManusSDK`` is inside + ``src/plugins/manus/`` or set ``MANUS_SDK_ROOT`` to your installation path. + * - Manus SDK not found at runtime + - The build configures RPATH automatically. If you moved the SDK after + building, set ``LD_LIBRARY_PATH`` to its ``lib/`` directory. + * - No data received + - Ensure MANUS Core is running and the gloves are connected and calibrated. + The plugin logs ``Left glove connected (ID ); start streaming.`` (or + ``Right ...``) when MANUS Core reports a glove; without that line it has + no glove to stream. + * - CloudXR runtime errors + - Check that a runtime is serving with ``python -m isaaccapture.cloudxr.service status``, + and that ``~/.cloudxr/run/cloudxr.env`` has been sourced in the same + terminal as the plugin. + * - Permission denied for USB devices + - udev rules must be installed on the host. Run ``./install_udev_rules.sh`` + from the host (not inside a container), then unplug and replug the + MANUS dongle. Verify on the host with ``ls -l /dev/hidraw*`` — entries for the + dongle should be mode ``0666``. + * - ``udevadm control --reload-rules`` fails with "No such file or directory" + - You're inside a container. ``systemd-udevd`` doesn't run in containers, + so this command can never succeed there. Run ``install_udev_rules.sh`` + on the host instead. + * - Dongle not visible inside the Isaac ROS container (``lsusb`` doesn't + show vendor ``3325``) + - The container needs ``/dev/bus/usb`` bind-mounted from the host. The + Isaac ROS dev container does this automatically; for a custom + ``docker run``, add ``-v /dev/bus/usb:/dev/bus/usb`` (or + ``--device=/dev/hidraw`` for a specific device). + +License +------- + +Source files are covered by their stated licenses (Apache-2.0). The Manus SDK is +proprietary to MANUS and is subject to its own license; it is **not** redistributed +by this project. diff --git a/docs/source/device/hand_tracking/sharpa_avatar.rst b/docs/source/device/hand_tracking/sharpa_avatar.rst new file mode 100644 index 0000000000..a7cde9d791 --- /dev/null +++ b/docs/source/device/hand_tracking/sharpa_avatar.rst @@ -0,0 +1,235 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2025-2026 Avatar SDK contributors. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Sharpa AvatarExo Glove +====================== + +A Linux-only plugin for integrating Sharpa AvatarExo (AE) Gloves into the +Isaac Capture framework. The gloves stream IMU and encoder data through the +official Avatar SDK over USB. The plugin maps that stream into Isaac Capture's +usual surfaces: OpenXR hand tracking for the HUMAN skeleton, +``JointStateOutput`` tensors for the RAW and ROBOT 22-DoF joint sets, and +inbound per-finger vibration on a haptic collection. + +Any downstream consumer that already reads OpenXR hands (``HandsSource``) or +schema joint-state collections can use Sharpa AE Gloves through the same DeviceIO +interfaces. The plugin talks to the gloves directly, so do not run +``avatar-backend`` or Avatar Desktop at the same time. + +Plugin sources and the operator README live under +:code-dir:`src/plugins/sharpa_avatar`. A TeleopSession visualization example +is :code-dir:`examples/sharpa_avatar`. + +.. contents:: On this page + :local: + :depth: 2 + +Data flow +--------- + +The glove SDK exposes three datasets plus haptics. HUMAN landmarks are placed +at a fused wrist pose and injected as OpenXR hands; RAW and ROBOT stay as +named joint-state collections: + +.. code-block:: text + + Sharpa AvatarExo Gloves (USB) + │ + ▼ + avatar_hand_plugin (Avatar SDK, production package) + │ + ├── HUMAN ──► OpenXR /hand/left, /hand/right (26 joints) + ├── RAW ──► avatar_raw_left / avatar_raw_right + ├── ROBOT ──► avatar_robot_left / avatar_robot_right + └── haptic ◄── avatar_glove_haptic (five finger vibration motors) + +.. seealso:: + + :doc:`/references/retargeting/sharpa` maps OpenXR hand tracking onto a + Sharpa robot hand. Avatar HUMAN output is the same OpenXR layer that + retargeter consumes. + +Prerequisites +------------- + +- **Linux x86_64** (tested on Ubuntu 22.04). +- **Pinned production Avatar SDK** (``avatar-sdk`` 1.7.3-17) from Sharpa's + signed APT repository. ``install_avatar_sdk.sh`` retrieves it; do not substitute + ``avatar-sdk-dev`` or ``avatar-sdk-beta``. +- **Sharpa AvatarExo Gloves** powered and connected over USB. Transport settings + are selected in ``/share/sdk_config.json`` (default root + ``/opt/avatar-sdk``). +- **CloudXR runtime** on the plugin host, unless a ``TeleopSession`` example + launches it for you. +- A built Isaac Capture checkout. CMake 3.24 or newer is required. + +The SDK stays external: headers, libraries, and runtime assets are not copied +into this repository or the plugin install prefix. + +Installation +------------ + +Install the pinned production SDK first if it is not already present: + +.. code-block:: bash + + ./src/plugins/sharpa_avatar/install_avatar_sdk.sh + +Then use the canonical top-level CMake workflow from the Isaac Capture root. +``-DAVATAR_SDK_ROOT`` selects a non-default SDK tree; that tree must already +be complete: + +.. code-block:: bash + + cmake -B build \ + -DBUILD_PLUGIN_SHARPA_AVATAR=ON \ + -DAVATAR_SDK_ROOT=/opt/avatar-sdk + cmake --build build --parallel + cmake --install build + +``-DBUILD_PLUGIN_SHARPA_AVATAR=ON`` without a usable SDK under the selected +root skips this plugin and the rest of Isaac Capture still configures. See +:doc:`/development/build_from_source/index`. + +USB access (one-time, on the host) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +udev rules must run **on the host**, not inside a container: + +.. code-block:: bash + + ./src/plugins/sharpa_avatar/install_udev_rules.sh + # then unplug and reconnect the glove + +Running the plugin +------------------ + +The plugin publishes through the CloudXR / OpenXR runtime. Start the runtime +and source its environment in the shell that launches the plugin: + +.. code-block:: bash + + python -m isaaccapture.cloudxr.service start # runs in the background + source ~/.cloudxr/run/cloudxr.env + ./install/plugins/sharpa_avatar/avatar_hand_plugin + +See :ref:`dedicated-cloudxr-runtime` and +:ref:`load-cloudxr-environment-variables` for the full service setup. + +``TeleopSession`` can launch the same binary through ``plugin.yaml``. The +visualization example does that for you (CloudXR and the plugin start unless +you opt out): + +.. code-block:: bash + + uv pip install -e ./examples/sharpa_avatar + python -m isaaccapture_examples.sharpa_avatar + +Gloves reconnect from ``update()`` if they drop offline. Only one process +may hold the Avatar SDK at a time. + +Datasets +-------- + +By default every path is on. Restrict them with ``--datasets=`` +(comma-separated: ``human``, ``raw``, ``robot``, ``haptic``): + +.. code-block:: bash + + ./install/plugins/sharpa_avatar/avatar_hand_plugin --datasets=human,raw,robot,haptic + ./install/plugins/sharpa_avatar/avatar_hand_plugin --datasets=human + +.. list-table:: + :widths: 38 62 + :header-rows: 1 + + * - Device or collection + - Contents + * - ``/hand/left``, ``/hand/right`` + - OpenXR 26-joint poses from Avatar HUMAN landmarks + * - ``avatar_raw_left``, ``avatar_raw_right`` + - RAW 22-DoF encoder joint state + * - ``avatar_robot_left``, ``avatar_robot_right`` + - ROBOT 22-DoF retargeted joint state + * - ``avatar_glove_haptic`` + - Inbound per-finger vibration commands + +HUMAN (OpenXR hands) +~~~~~~~~~~~~~~~~~~~~ + +HUMAN landmarks are mapped onto the 26 OpenXR hand slots (unsupported slots +stay invalid) and pushed with ``HandInjector``. Hosts read them through +``HandsSource`` like any other hand-tracking plugin. + +RAW and ROBOT (joint-state tensors) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Each side pushes a ``JointStateOutput`` (tensor id ``joint_state``) with the +22 named DoF from the pinned SDK. Consume them with ``JointStateSource`` +and the collection ids above. + +Haptic +~~~~~~ + +The plugin reads ``HapticCommand`` on ``avatar_glove_haptic`` and drives +five finger vibration motors. The visualization example uses the same +per-finger pinch path as the generic glove haptic sample. + +Wrist pose +---------- + +Sharpa AE Glove data is wrist-relative, so the plugin anchors HUMAN landmarks +with a pose from ``WristPoseSource`` before publishing OpenXR hands. It prefers +optical hand tracking and falls back to the controller aim pose when needed. +If no wrist pose is available yet, the plugin still publishes a local skeleton +with VALID-only flags; TRACKED is set once the selected wrist source is actively +tracked. + +The example places that local skeleton into a stable left/right display +frame. Pass ``--world-frame`` to leave the OpenXR poses unmodified. + +Troubleshooting +--------------- + +.. list-table:: + :widths: 40 60 + :header-rows: 1 + + * - Symptom + - Fix + * - Avatar SDK installation fails + - Confirm access to Sharpa's production APT endpoint and re-run + ``install_avatar_sdk.sh``. + * - CMake skipped the plugin + - The SDK is missing under the selected root. Run + ``install_avatar_sdk.sh`` and reconfigure with + ``-DBUILD_PLUGIN_SHARPA_AVATAR=ON``. + * - CMake rejected the Avatar SDK + - Install the pinned production package, or point ``AVATAR_SDK_ROOT`` + at a complete SDK tree. A ``-dev`` / ``-beta`` package at + ``/opt/avatar-sdk`` is not the pin. + * - USB glove is not detected + - Run ``install_udev_rules.sh`` on the host, then unplug and reconnect + the AE Gloves. + * - Plugin binary is not found + - Build and install through the top-level CMake workflow. The example looks under + ``install/plugins/sharpa_avatar/``. + * - HUMAN / RAW / ROBOT stay offline + - Power the gloves, and stop Avatar Desktop, ``avatar-backend``, or + any other process using the Avatar SDK. + * - Viser shows an empty grid + - HUMAN must be enabled (default). RAW-only is joint angles, not a + 3D skeleton. Confirm the example is not launched with + ``--datasets`` that omit ``human``. + * - No vibration + - Keep ``haptic`` in ``--datasets`` and close a fingertip toward the + thumb (example pinch path). + * - CloudXR runtime errors + - Check ``python -m isaaccapture.cloudxr.service status``, and source + ``~/.cloudxr/run/cloudxr.env`` in the same terminal as the plugin. + +License +------- + +Plugin sources are Apache-2.0. The Avatar SDK is proprietary to Sharpa and +is subject to its own license; this project does not redistribute it. diff --git a/docs/source/device/hand_tracking/wuji_glove.rst b/docs/source/device/hand_tracking/wuji_glove.rst new file mode 100644 index 0000000000..0ca61dac4f --- /dev/null +++ b/docs/source/device/hand_tracking/wuji_glove.rst @@ -0,0 +1,133 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 Wuji Technology. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Wuji Glove +========== + +A Linux-only plugin for integrating `Wuji `_ +data gloves into the Isaac Capture framework. It reads the glove's 21-joint hand +skeleton over the network via the wuji_sdk C API and injects the resulting +poses into the OpenXR hand-tracking layer, so any downstream consumer can read +them as standard hand tracking. + +.. contents:: On this page + :local: + :depth: 2 + +Data flow +--------- + +The plugin combines the glove's wrist-relative skeleton with an absolute wrist +pose and publishes the result through OpenXR: + +.. code-block:: text + + Wuji glove (21-joint shape) ─┐ + ├─► wuji_glove plugin ──► CloudXR OpenXR runtime + Optical hand / controller ───┘ (wrist-pose fusion) (spatial hand joints) + +.. seealso:: + + :doc:`/references/retargeting/wuji` explains how to map OpenXR hand tracking + to the 20 joint commands of a Wuji Hand or Wuji Hand 2. + +Prerequisites +------------- + +- **Linux** — x86_64 or aarch64 (tested on Ubuntu 22.04 / 24.04). +- **wuji_sdk C SDK v2026.7.14** — the + ``wuji-sdk-c-2026.7.14--linux-gnu.tar.gz`` GitHub release asset, which + extracts to ``include/wuji_sdk.h`` plus ``lib/libwuji_sdk_c.so`` and its + companion ``lib/libwujihandcpp.so``. This is the C FFI, *not* the + ``wuji-sdk`` pip package; the install script downloads it automatically. +- **Wuji glove(s)** reachable on the same LAN as the plugin host. +- **CloudXR runtime** running on the plugin host. + +Installation +------------ + +``install.sh`` downloads the pinned C SDK release, verifies it against a +per-architecture SHA-256, extracts it next to the script (re-runs reuse the +extracted copy), and configures, builds, and installs the plugin: + +.. code-block:: bash + + ./src/plugins/wuji_glove/install.sh + +Pass ``--build-dir `` to reuse a non-default top-level CMake build +directory. + +To build offline, point ``WUJI_SDK_C_DIR`` at an already-extracted SDK copy — +or drive CMake directly: + +.. code-block:: bash + + cmake -B build -DBUILD_PLUGINS=ON -DBUILD_PLUGIN_WUJI_GLOVE=ON \ + -DWUJI_SDK_INCLUDE_DIR=/path/to/wuji-sdk-c/include \ + -DWUJI_SDK_LIB=/path/to/wuji-sdk-c/lib/libwuji_sdk_c.so + cmake --build build --target wuji_glove_plugin + cmake --install build --component wuji_glove + +Running the Plugin +------------------ + +The plugin reaches the Teleop session through the CloudXR / OpenXR runtime, so +start the runtime first and source its environment in the shell that launches +the plugin: + +.. code-block:: bash + + python -m isaaccapture.cloudxr.service start # runs in the background + source ~/.cloudxr/run/cloudxr.env + ./install/plugins/wuji_glove/wuji_glove_plugin + +See :ref:`dedicated-cloudxr-runtime` and +:ref:`load-cloudxr-environment-variables` for the full service setup. The plugin resolves each glove's side from device +metadata and reconnects automatically after a disconnect; applications can also +launch it through its ``plugin.yaml`` using ``TeleopSession``. + +Wrist pose +---------- + +The glove stream contains wrist-relative joints but no absolute 6DoF wrist +pose. Before injection, the plugin places that skeleton at a wrist pose from +optical hand tracking or from a controller aim pose combined with a calibrated +per-hand rigid offset. + +Set ``WUJI_GLOVE_WRIST_SOURCE`` to ``auto`` (the default), ``hand_tracking``, or +``controller``. In ``auto`` mode, optical tracking is preferred; the controller +is used when optical tracking provides no valid pose. Override the built-in +controller offsets when needed: + +.. code-block:: bash + + export WUJI_GLOVE_AIM_TO_WRIST_LEFT="px,py,pz,qx,qy,qz,qw" + export WUJI_GLOVE_AIM_TO_WRIST_RIGHT="px,py,pz,qx,qy,qz,qw" + +Position values are in meters and quaternion values use ``(x, y, z, w)``. +The built-in offsets assume mirrored left/right mounts. + +Troubleshooting +--------------- + +.. list-table:: + :widths: 40 60 + :header-rows: 1 + + * - Symptom + - Fix + * - ``install.sh`` fails with a SHA-256 mismatch + - The downloaded tarball does not match the pinned release. Re-run to + rule out a corrupted download; if it persists, do not bypass the check + — verify the release assets upstream. + * - No glove is discovered + - Confirm the glove is powered and on the same LAN/subnet as the plugin + host, then check the plugin log for ``wuji_scan`` errors. + * - Log says a second same-side glove is ignored + - The plugin binds one glove per side (first discovered wins) and ignores + later same-side gloves until restart. Power off the spare, or restart + the plugin to rebind. + * - Plugin runs but a downstream client sees no hand data + - The plugin and the client must talk to the same CloudXR runtime; check + that the runtime was up before both, and that hands are streaming + (plugin prints per-glove connect lines). diff --git a/docs/source/device/haptic_feedback.rst b/docs/source/device/haptic_feedback.rst index 9e7800cb75..0aef2d17e2 100644 --- a/docs/source/device/haptic_feedback.rst +++ b/docs/source/device/haptic_feedback.rst @@ -1,99 +1,9 @@ -.. SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 +:orphan: + Haptic Feedback =============== -Drive a haptic actuator from a simulator signal through the existing Isaac Teleop -retargeting pipeline. Motion-controller vibration (Quest, Vive Index, Pico, …) is -the first integration; the abstraction is vendor-neutral so haptic gloves and -force-feedback devices fit the same contract. - -.. contents:: On this page - :local: - :depth: 2 - -Architecture ------------- - -Haptic output is a graph phase symmetric to the input phase: where an input -source feeds device data *into* the retargeting graph, a haptic **sink** consumes -graph outputs and writes them *out* to a device after the graph runs. - -- **Device-side schemas** (``TensorGroupType``) describe what a device consumes. - ``ControllerHapticPulse`` carries ``[amplitude, frequency_hz, duration_s]``; - ``EndEffectorForce`` carries a 3-DoF force for future grounded devices. They - live in ``isaaccapture.retargeting_engine.tensor_types``. -- **Retargeters** in ``isaaccapture.retargeters.tactile_retargeters`` map - sim-side tactile data (a ``TactileVector`` / ``TactileHeatmap``) to a - device-side schema — e.g. ``TactileVectorToControllerPulse``. -- **HapticSink** (``IDeviceIOSink``) is the vendor-neutral output node. It stores - each frame's values per endpoint, and ``TeleopSession`` flushes it to the - device after the graph, with the active session in scope. Register it with - ``TeleopSessionConfig(sinks=[...])``. -- **IHapticDevice** is the vendor adapter. ``ControllerHapticDevice`` is the - in-process reference; it drives the controller's vibration actuator through the - same ``ControllerTracker`` that ``ControllersSource`` reads on the input side - (via the per-side ``apply_left_haptic_feedback`` / ``apply_right_haptic_feedback``). - OpenXR specifics stay in the live tracker layer, so the device, sink, and schema - are runtime-neutral. - -.. code-block:: text - - Sim / input signal Retargeter Sink (IDeviceIOSink) - (contact force, --> (TactileVectorTo- --> HapticSink - trigger value) ControllerPulse) -> ControllerHapticDevice - (flushed after the graph) - -> ControllerTracker - .apply_left/right_haptic_feedback - -Example -------- - -``examples/haptic_feedback/python/controller_haptic_example.py`` is the minimal -end-to-end wiring: pull a controller's trigger and that **same** controller -rumbles. ``TriggerToTactile`` turns the trigger value into a ``TactileVector``, -``TactileVectorToControllerPulse`` turns that into a ``ControllerHapticPulse``, -and the ``HapticSink`` drives the controller. Swap ``TriggerToTactile`` for any -``TactileVector``-producing source (e.g. an Isaac Lab ``ContactSensor`` fetch) to -rumble from sim contact instead. - -``ControllerHapticDevice`` must reuse the ``ControllerTracker`` owned by -``ControllersSource`` (pass ``controllers.get_tracker()``) so the session creates -a single controller tracker and there is no OpenXR action-set contention. - -Running -------- - -The example connects through the CloudXR / OpenXR runtime, which it starts -itself if nothing is serving (see :ref:`dedicated-cloudxr-runtime` to manage -that service yourself). Run it from the ``examples/haptic_feedback/python`` -directory: - -.. code-block:: bash - - uv run controller_haptic_example.py - -No arguments — pull either trigger to rumble that controller. Press Ctrl+C to -exit. Runtimes that do not expose ``xrApplyHapticFeedback`` silently no-op rather -than tearing down the session. - -Adding a new haptic device ---------------------------- - -A new haptic device implements ``IHapticDevice`` (``accepted_type``, -``endpoints``, ``apply``, ``flush``, ``get_tracker``) and is wired into a -``HapticSink``. Endpoints are named — ``"left"`` / ``"right"`` by convention, or -``"device"`` for a single grounded device — so single-actuator and -multi-actuator rigs share the same contract without a hardcoded handedness -assumption. Devices that run their vendor SDK in a separate process (haptic -gloves, exoskeletons) implement the same interface but exchange data with their -plugin over a tensor collection; those integrations land on top of this -foundation. - -See also --------- - -- Example + tests: ``examples/haptic_feedback/python/`` and - ``tests/python/core/retargeting_engine/test_haptic_devices.py`` / - ``test_haptic_sink.py``. +This page has moved to :doc:`/references/haptic_feedback`. diff --git a/docs/source/device/haptikos.rst b/docs/source/device/haptikos.rst index d2f152da8f..1acf696454 100644 --- a/docs/source/device/haptikos.rst +++ b/docs/source/device/haptikos.rst @@ -1,128 +1,9 @@ -.. SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 +:orphan: + Haptikos Exoskeletons ===================== -.. raw:: html - -
- -.. image:: ../_static/haptikos-exoskeletons.gif - :alt: Haptikos exoskeletons used with Isaac Teleop - :class: align-center - :width: 100% - -.. raw:: html - -
- -Use the `Haptikos `_ Exoskeletons with the Isaac Teleop framework. Currently only Linux is supported. Tested on Meta Quest headsets. Other headsets with controllers may work as well. - -.. contents:: On this page - :local: - :depth: 2 - -Components ----------- - -- **Core library** (``haptikos_core``) — wraps the Haptikos Core App(``libHaptikosCore.so``) and provides access to per-joint tracking data and haptics. -- **Plugin executable** (``haptikos_plugin``) — the primary plugin binary that enables integration with the Teleop system via CloudXR and OpenXR. - -Prerequisites -------------- -- **Linux** — x86_64 (tested on Ubuntu 22.04 / 24.04). -- **Haptikos API** for Linux. - - -Quick Start ------------ - -Step 1: Get the Haptikos API -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The Haptikos API must be downloaded separately due to licensing. Check our `website `_. - -1. Obtain a Haptikos account. -2. Clone the `repository `_. -3. Copy the ``HaptikosCpp_API_Shared`` folder from our HaptikosAPI repository into the ``src/plugins/haptikos`` folder. The folder structure should be the following: - -.. code-block:: text - - src/plugins/haptikos/ - ├── CMakeLists.txt - ├── HaptikosCpp_API_Shared - │ ├── include - │ └── lib - ├── haptikos_hands_plugin.cpp - ├── haptikos_hands_plugin.hpp - ├── main.cpp - ├── plugin.yaml - └── README.md - -Step 2: Build the plugin -~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: bash - - cd ../../.. # Navigate to TeleopCore root - cmake -S . -B build - cd build - make haptikos_hands_plugin - -Step 3: Build the entire project -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. code-block:: bash - - cmake -B build -DENABLE_CLANG_FORMAT_CHECK=OFF #From Project root - cmake --build build --parallel 4 - cmake --install build - -We use the ``-DENABLE_CLANG_FORMAT_CHECK=OFF`` because the headers in the HaptikosAPI library are not clang-formatted. You can clang format the headers to not include the flag by running: - -.. code-block:: bash - - clang-format -i src/plugins/haptikos/HaptikosCpp_API_Shared/include/* - -Step 4: Run the plugin -~~~~~~~~~~~~~~~~~~~~~~ - -The Haptikos plugin connects to the Teleop session through the CloudXR / OpenXR runtime, so the runtime must be running and its environment sourced in the shell that launches the plugin. - -Start the CloudXR runtime from the Python environment where ``isaaccapture`` is installed. It runs as a background service that outlives this shell, until you stop it with ``python -m isaaccapture.cloudxr.service stop``: - -.. code-block:: bash - - python -m isaaccapture.cloudxr.service start --accept-eula - -In the shell that runs the plugin, source the environment file that the runtime writes on startup. This points the OpenXR loader at CloudXR: - -.. code-block:: bash - - cd IsaacTeleop/ - source ~/.cloudxr/run/cloudxr.env - -In the same terminal, run the plugin with the following: - -.. code-block:: bash - - ./install/plugins/haptikos/haptikos_hands_plugin - -Important Information ---------------------- - -1. To use the plugin properly it is necessary to attach the controllers on our exoskeletons using the included mount. - -2. The orientation calibration defines the direction the Haptikos gloves define as forward. This needs to be aligned with the Head Mounted Display's forward direction. - -3. To inject the hand tracking data and haptic feedback, the controllers, the Haptikos App and the exoskeletons need to be active. - -4. The executable will be located in the ``build/src/plugins/haptikos``. If you installed the project by running ``cmake --install build`` the executable will also be found in the ``install/plugins/haptikos`` folder. - - -License -------- - -Source files are covered by their stated licenses (Apache-2.0). The Haptikos API is -proprietary to Haptikos and is subject to its own license; it is **not** redistributed -by this project. +This page has moved to :doc:`/device/hand_tracking/haptikos`. diff --git a/docs/source/device/joint_space.rst b/docs/source/device/joint_space.rst index b6ffa29cf3..11aa6b49b2 100644 --- a/docs/source/device/joint_space.rst +++ b/docs/source/device/joint_space.rst @@ -1,220 +1,9 @@ .. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 +:orphan: + Generic Joint-Space Device ========================== -A reusable device path for any **joint-encoder source** -- leader arms, exoskeletons, haptic -gloves, or other articulated input devices. A device streams a name-keyed ``JointStateOutput`` -FlatBuffer over the OpenXR tensor transport; one schema, one tracker, one source, and one -retargeter serve them all, so adding a new joint-space device is just a new **plugin** plus a -small **config**. - -The **SO-101 leader arm** (`TheRobotStudio SO-ARM100 `_, -6 Feetech STS3215 bus servos) is the reference instance; the **reBot DevArm leader** -(`Seeed reBot DevArm `_, 7 Damiao DM-series or -RobStride RS-series motors) is a second instance built from the same recipe. - -At a glance ------------ - -.. list-table:: - :header-rows: 1 - :widths: 18 82 - - * - Layer - - Component - * - Schema - - :code-file:`src/core/schema/fbs/joint_state.fbs` -- ``JointState`` (name + position + - optional velocity/effort) and ``JointStateOutput`` (a vector of joints + ``device_id``). - * - Plugin - - :code-dir:`src/plugins/so101_leader` -- pushes ``JointStateOutput`` via ``SchemaPusher``. - Reads the FEETECH STS3215 servos over serial (``FeetechBus``); synthetic fallback when no - device path is given. - * - Tracker - - ``JointStateTracker`` (facade) with live (``LiveJointStateTrackerImpl``) and MCAP-replay - (``ReplayJointStateTrackerImpl``) backends, registered in the live/replay factories. - * - Source - - ``JointStateSource`` (``IDeviceIOSource``) -- converts the FlatBuffer into a name-keyed - group of joint positions for the retargeting graph. - * - Retargeter - - ``JointStateRetargeter`` -- ``joint`` (mirror) or ``ee_pose`` (URDF FK) mode. See - :doc:`/references/retargeting/joint_space`. - -Data schema ------------ - -Joints are modeled as **name -> value** records so consumers read them by name, independent of -wire order: - -.. code-block:: idl - :class: code-100col - - table JointState { - name: string (id: 0, key); // e.g. "shoulder_pan", "gripper" - position: float (id: 1); // [rad] revolute, [m] prismatic - velocity: float (id: 2); // optional (JointStateOutput.has_velocity) - effort: float (id: 3); // optional (JointStateOutput.has_effort) - valid: bool = true (id: 4); - } - - table JointStateOutput { - joints: [JointState] (id: 0); - device_id: string (id: 1); - has_velocity: bool (id: 2); - has_effort: bool (id: 3); - ee_pose: Pose (id: 4); // RESERVED: device-side FK; not consumed yet - ee_pose_valid: bool (id: 5); - } - -The gripper is just another named DOF (conventionally ``"gripper"``). ``velocity``, ``effort``, -and ``ee_pose`` are optional/reserved: the reference plugin and ``JointStateSource`` populate and -surface joint **positions** only. - -.. _so101-leader-plugin: - -The SO-101 leader plugin ------------------------- - -``so101_leader`` reads the six SO-101 servos (``shoulder_pan, shoulder_lift, elbow_flex, -wrist_flex, wrist_roll, gripper``) and pushes them to a tensor collection. With a serial device -path it talks to the FEETECH STS3215 bus servos directly via ``FeetechBus`` -- the same SMS/STS -wire protocol the FEETECH SCServo SDK / LeRobot's ``FeetechMotorsBus`` use, with no SDK dependency: -it disables torque (so the leader can be back-driven) and reads ``Present_Position`` each frame, -converting ticks to radians with per-joint calibration. With no device path it falls back to a -**synthetic** trajectory so the pipeline runs hardware-free (CI and the headless example). - -.. code-block:: bash - - # Synthetic backend (no hardware), default collection id "so101_leader": - ./install/plugins/so101_leader/so101_leader_plugin - - # Real SO-101 leader on a serial port (Linux), optional calibration file: - ./install/plugins/so101_leader/so101_leader_plugin /dev/ttyACM0 so101_leader so101_leader.calib - -See the :code-file:`plugin README ` for hardware setup -(unique servo ids, gear removal, back-driving) and the calibration file format. - -The consumer side creates a ``JointStateSource(name=..., collection_id="so101_leader", -joint_names=[...])`` on the same ``collection_id``; ``TeleopSession`` discovers and polls the -``JointStateTracker`` each frame. - -.. _rebot-devarm-leader-plugin: - -The reBot DevArm leader plugin ------------------------------- - -``rebot_devarm_leader`` reads the seven joints of the Seeed reBot DevArm (``joint1 .. joint6, -gripper``) and pushes them to a tensor collection, mirroring the SO-101 plugin's structure and -CLI shape. The arm ships in **two motor builds**, and the plugin picks the backend from the shape -of the device argument: a serial path containing ``/`` (e.g. ``/dev/ttyACM0``) selects the -**Damiao** build (:ref:`below `), a bare SocketCAN interface name -(e.g. ``can0``) selects the **RobStride** build (:ref:`below `). -With no device argument it falls back to a **synthetic** trajectory, exactly like -``so101_leader``: - -.. code-block:: bash - - # Synthetic backend (no hardware), default collection id "rebot_devarm_leader": - ./install/plugins/rebot_devarm_leader/rebot_devarm_leader_plugin - -Both builds share the calibration file format; see the -:code-file:`plugin README ` for the format -(name, command/feedback CAN ids, motor model, sign, zero offset) and hardware notes. - -.. _rebot-devarm-damiao-build: - -The Damiao build -~~~~~~~~~~~~~~~~ - -The **Damiao build** (7 DM-series MIT-protocol motors: DM4340P on joints 1-3, DM4310 on -joints 4-6 and the gripper) puts the motors on a CAN bus behind a Damiao USB-to-CAN serial adapter -(USB CDC-ACM); ``DamiaoBus`` speaks the adapter's fixed-size binary framing directly -- no SDK -dependency. As a *leader*, the plugin sends the **disable** control frame so the arm can be -back-driven by hand (Damiao motors keep answering feedback requests while disabled), then -requests one feedback frame per motor per cycle (command ``0xCC`` addressed via CAN id ``0x7FF``) -and decodes the fixed-point position/velocity feedback, which lands directly in radians -- no -tick conversion, only an optional per-joint sign and zero offset from a calibration file. - -.. code-block:: bash - - # Real reBot DevArm on the Damiao USB-to-CAN adapter (Linux), optional calibration file: - ./install/plugins/rebot_devarm_leader/rebot_devarm_leader_plugin /dev/ttyACM0 rebot_devarm_leader rebot_devarm.calib - - # Probe wiring, motor ids, and the decode path -- no OpenXR runtime needed: - ./install/plugins/rebot_devarm_leader/rebot_devarm_leader_plugin probe /dev/ttyACM0 - -``probe`` exits ``0`` when every motor replied, and ``3`` when the motors replied but the gripper -reads outside its physical travel: the Damiao multi-turn counter is volatile across power cycles, -so the gripper (whose geared travel exceeds one turn) can wake up reading ``physical + 2*pi*k`` -and must be re-homed (closed against the mechanical stop and re-zeroed) before teleoperating. -While wrapped, the running plugin streams the gripper joint with ``valid = false`` so consumers -hold it instead of executing garbage. - -.. _rebot-devarm-robstride-build: - -The RobStride build -~~~~~~~~~~~~~~~~~~~ - -The **RobStride build** (7 RS-series motors) speaks classic CAN at **1 Mbps** through any -SocketCAN adapter (PCAN, candleLight, ...); this backend is Linux-only. ``RobStrideBus`` -implements the leader subset of the RobStride private 29-bit extended-id CAN protocol directly -- -again no SDK dependency: it sends the **stop** frame (comm type ``0x04``) so the arm can be -back-driven by hand, then alternates single-parameter reads (comm type ``0x11``) of ``mechPos`` -(``0x7019``) and ``mechVel`` (``0x701A``) per motor per cycle. Replies carry exact little-endian -IEEE ``f32`` radians / rad/s, so the decode is **model-independent** -- no fixed-point limit -tables. Each channel refreshes at 45 Hz with the plugin's 90 Hz loop, under 20% load of the -1 Mbps bus for 7 motors. - -.. code-block:: bash - - # Bring up the SocketCAN interface (once per boot): - sudo ip link set can0 up type can bitrate 1000000 - - # Real reBot DevArm (RobStride build) on a SocketCAN interface: - ./install/plugins/rebot_devarm_leader/rebot_devarm_leader_plugin can0 - - # Probe wiring, motor ids, and the decode path -- no OpenXR runtime needed: - ./install/plugins/rebot_devarm_leader/rebot_devarm_leader_plugin probe can0 - -``probe`` uses the same exit codes as the Damiao build (``0`` all motors replied, ``1`` some -missing, ``2`` no device, ``3`` gripper out of travel). The calibration file format is shared; -on the RobStride backend only ``motor_id`` is used (replies are matched by device id, the -``feedback_id`` column is ignored), and ``rs-*`` model names are accepted and ignored since the -decode is model-independent. Factory RobStride device ids are ``1..7``; the host id is ``0xFD``. - -Record and replay ------------------ - -The live tracker records to MCAP, and ``ReplayJointStateTrackerImpl`` replays it back with no -OpenXR runtime, so a recorded session drives the retargeting graph headlessly: - -.. code-block:: python - - from isaaccapture.deviceio import McapRecordingConfig, McapReplayConfig - from isaaccapture.teleop_session_manager import SessionMode, TeleopSession, TeleopSessionConfig - - # Record (live): TeleopSessionConfig(..., mcap_config=McapRecordingConfig("leader.mcap")) - # Replay (headless): TeleopSessionConfig(..., mode=SessionMode.REPLAY, - # mcap_config=McapReplayConfig("leader.mcap")) - -Add another joint-space device ------------------------------- - -Reuse everything above by writing only: - -#. A **plugin** that reads your hardware and fills ``JointStateOutput`` (positions; optionally - velocity/effort), modeled on :code-dir:`src/plugins/so101_leader` (or - :code-dir:`src/plugins/rebot_devarm_leader`, a second instance of the same recipe on a - different motor bus). -#. A **config**: a ``collection_id``, the device joint names, and -- for ``ee_pose`` mode -- a URDF - and end-effector link. - -The schema, ``JointStateTracker``, ``JointStateSource``, and ``JointStateRetargeter`` are unchanged. - -.. seealso:: - - :doc:`add_device` -- the general four-step device-plugin recipe (foot-pedal reference). - - :doc:`/references/retargeting/joint_space` -- the ``JointStateRetargeter`` (joint / EE modes), - the end-to-end example, and validation. +This page has moved to :doc:`/device/joint_space/index`. diff --git a/docs/source/device/joint_space/index.rst b/docs/source/device/joint_space/index.rst new file mode 100644 index 0000000000..9a26e76a5b --- /dev/null +++ b/docs/source/device/joint_space/index.rst @@ -0,0 +1,220 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Generic Joint-Space Device +========================== + +A reusable device path for any **joint-encoder source** -- leader arms, exoskeletons, haptic +gloves, or other articulated input devices. A device streams a name-keyed ``JointStateOutput`` +FlatBuffer over the OpenXR tensor transport; one schema, one tracker, one source, and one +retargeter serve them all, so adding a new joint-space device is just a new **plugin** plus a +small **config**. + +The **SO-101 leader arm** (`TheRobotStudio SO-ARM100 `_, +6 Feetech STS3215 bus servos) is the reference instance; the **reBot DevArm leader** +(`Seeed reBot DevArm `_, 7 Damiao DM-series or +RobStride RS-series motors) is a second instance built from the same recipe. + +At a glance +----------- + +.. list-table:: + :header-rows: 1 + :widths: 18 82 + + * - Layer + - Component + * - Schema + - :code-file:`src/core/schema/fbs/joint_state.fbs` -- ``JointState`` (name + position + + optional velocity/effort) and ``JointStateOutput`` (a vector of joints + ``device_id``). + * - Plugin + - :code-dir:`src/plugins/so101_leader` -- pushes ``JointStateOutput`` via ``SchemaPusher``. + Reads the FEETECH STS3215 servos over serial (``FeetechBus``); synthetic fallback when no + device path is given. + * - Tracker + - ``JointStateTracker`` (facade) with live (``LiveJointStateTrackerImpl``) and MCAP-replay + (``ReplayJointStateTrackerImpl``) backends, registered in the live/replay factories. + * - Source + - ``JointStateSource`` (``IDeviceIOSource``) -- converts the FlatBuffer into a name-keyed + group of joint positions for the retargeting graph. + * - Retargeter + - ``JointStateRetargeter`` -- ``joint`` (mirror) or ``ee_pose`` (URDF FK) mode. See + :doc:`/references/retargeting/joint_space`. + +Data schema +----------- + +Joints are modeled as **name -> value** records so consumers read them by name, independent of +wire order: + +.. code-block:: idl + :class: code-100col + + table JointState { + name: string (id: 0, key); // e.g. "shoulder_pan", "gripper" + position: float (id: 1); // [rad] revolute, [m] prismatic + velocity: float (id: 2); // optional (JointStateOutput.has_velocity) + effort: float (id: 3); // optional (JointStateOutput.has_effort) + valid: bool = true (id: 4); + } + + table JointStateOutput { + joints: [JointState] (id: 0); + device_id: string (id: 1); + has_velocity: bool (id: 2); + has_effort: bool (id: 3); + ee_pose: Pose (id: 4); // RESERVED: device-side FK; not consumed yet + ee_pose_valid: bool (id: 5); + } + +The gripper is just another named DOF (conventionally ``"gripper"``). ``velocity``, ``effort``, +and ``ee_pose`` are optional/reserved: the reference plugin and ``JointStateSource`` populate and +surface joint **positions** only. + +.. _so101-leader-plugin: + +The SO-101 leader plugin +------------------------ + +``so101_leader`` reads the six SO-101 servos (``shoulder_pan, shoulder_lift, elbow_flex, +wrist_flex, wrist_roll, gripper``) and pushes them to a tensor collection. With a serial device +path it talks to the FEETECH STS3215 bus servos directly via ``FeetechBus`` -- the same SMS/STS +wire protocol the FEETECH SCServo SDK / LeRobot's ``FeetechMotorsBus`` use, with no SDK dependency: +it disables torque (so the leader can be back-driven) and reads ``Present_Position`` each frame, +converting ticks to radians with per-joint calibration. With no device path it falls back to a +**synthetic** trajectory so the pipeline runs hardware-free (CI and the headless example). + +.. code-block:: bash + + # Synthetic backend (no hardware), default collection id "so101_leader": + ./install/plugins/so101_leader/so101_leader_plugin + + # Real SO-101 leader on a serial port (Linux), optional calibration file: + ./install/plugins/so101_leader/so101_leader_plugin /dev/ttyACM0 so101_leader so101_leader.calib + +See the :code-file:`plugin README ` for hardware setup +(unique servo ids, gear removal, back-driving) and the calibration file format. + +The consumer side creates a ``JointStateSource(name=..., collection_id="so101_leader", +joint_names=[...])`` on the same ``collection_id``; ``TeleopSession`` discovers and polls the +``JointStateTracker`` each frame. + +.. _rebot-devarm-leader-plugin: + +The reBot DevArm leader plugin +------------------------------ + +``rebot_devarm_leader`` reads the seven joints of the Seeed reBot DevArm (``joint1 .. joint6, +gripper``) and pushes them to a tensor collection, mirroring the SO-101 plugin's structure and +CLI shape. The arm ships in **two motor builds**, and the plugin picks the backend from the shape +of the device argument: a serial path containing ``/`` (e.g. ``/dev/ttyACM0``) selects the +**Damiao** build (:ref:`below `), a bare SocketCAN interface name +(e.g. ``can0``) selects the **RobStride** build (:ref:`below `). +With no device argument it falls back to a **synthetic** trajectory, exactly like +``so101_leader``: + +.. code-block:: bash + + # Synthetic backend (no hardware), default collection id "rebot_devarm_leader": + ./install/plugins/rebot_devarm_leader/rebot_devarm_leader_plugin + +Both builds share the calibration file format; see the +:code-file:`plugin README ` for the format +(name, command/feedback CAN ids, motor model, sign, zero offset) and hardware notes. + +.. _rebot-devarm-damiao-build: + +The Damiao build +~~~~~~~~~~~~~~~~ + +The **Damiao build** (7 DM-series MIT-protocol motors: DM4340P on joints 1-3, DM4310 on +joints 4-6 and the gripper) puts the motors on a CAN bus behind a Damiao USB-to-CAN serial adapter +(USB CDC-ACM); ``DamiaoBus`` speaks the adapter's fixed-size binary framing directly -- no SDK +dependency. As a *leader*, the plugin sends the **disable** control frame so the arm can be +back-driven by hand (Damiao motors keep answering feedback requests while disabled), then +requests one feedback frame per motor per cycle (command ``0xCC`` addressed via CAN id ``0x7FF``) +and decodes the fixed-point position/velocity feedback, which lands directly in radians -- no +tick conversion, only an optional per-joint sign and zero offset from a calibration file. + +.. code-block:: bash + + # Real reBot DevArm on the Damiao USB-to-CAN adapter (Linux), optional calibration file: + ./install/plugins/rebot_devarm_leader/rebot_devarm_leader_plugin /dev/ttyACM0 rebot_devarm_leader rebot_devarm.calib + + # Probe wiring, motor ids, and the decode path -- no OpenXR runtime needed: + ./install/plugins/rebot_devarm_leader/rebot_devarm_leader_plugin probe /dev/ttyACM0 + +``probe`` exits ``0`` when every motor replied, and ``3`` when the motors replied but the gripper +reads outside its physical travel: the Damiao multi-turn counter is volatile across power cycles, +so the gripper (whose geared travel exceeds one turn) can wake up reading ``physical + 2*pi*k`` +and must be re-homed (closed against the mechanical stop and re-zeroed) before teleoperating. +While wrapped, the running plugin streams the gripper joint with ``valid = false`` so consumers +hold it instead of executing garbage. + +.. _rebot-devarm-robstride-build: + +The RobStride build +~~~~~~~~~~~~~~~~~~~ + +The **RobStride build** (7 RS-series motors) speaks classic CAN at **1 Mbps** through any +SocketCAN adapter (PCAN, candleLight, ...); this backend is Linux-only. ``RobStrideBus`` +implements the leader subset of the RobStride private 29-bit extended-id CAN protocol directly -- +again no SDK dependency: it sends the **stop** frame (comm type ``0x04``) so the arm can be +back-driven by hand, then alternates single-parameter reads (comm type ``0x11``) of ``mechPos`` +(``0x7019``) and ``mechVel`` (``0x701A``) per motor per cycle. Replies carry exact little-endian +IEEE ``f32`` radians / rad/s, so the decode is **model-independent** -- no fixed-point limit +tables. Each channel refreshes at 45 Hz with the plugin's 90 Hz loop, under 20% load of the +1 Mbps bus for 7 motors. + +.. code-block:: bash + + # Bring up the SocketCAN interface (once per boot): + sudo ip link set can0 up type can bitrate 1000000 + + # Real reBot DevArm (RobStride build) on a SocketCAN interface: + ./install/plugins/rebot_devarm_leader/rebot_devarm_leader_plugin can0 + + # Probe wiring, motor ids, and the decode path -- no OpenXR runtime needed: + ./install/plugins/rebot_devarm_leader/rebot_devarm_leader_plugin probe can0 + +``probe`` uses the same exit codes as the Damiao build (``0`` all motors replied, ``1`` some +missing, ``2`` no device, ``3`` gripper out of travel). The calibration file format is shared; +on the RobStride backend only ``motor_id`` is used (replies are matched by device id, the +``feedback_id`` column is ignored), and ``rs-*`` model names are accepted and ignored since the +decode is model-independent. Factory RobStride device ids are ``1..7``; the host id is ``0xFD``. + +Record and replay +----------------- + +The live tracker records to MCAP, and ``ReplayJointStateTrackerImpl`` replays it back with no +OpenXR runtime, so a recorded session drives the retargeting graph headlessly: + +.. code-block:: python + + from isaaccapture.deviceio import McapRecordingConfig, McapReplayConfig + from isaaccapture.teleop_session_manager import SessionMode, TeleopSession, TeleopSessionConfig + + # Record (live): TeleopSessionConfig(..., mcap_config=McapRecordingConfig("leader.mcap")) + # Replay (headless): TeleopSessionConfig(..., mode=SessionMode.REPLAY, + # mcap_config=McapReplayConfig("leader.mcap")) + +Add another joint-space device +------------------------------ + +Reuse everything above by writing only: + +#. A **plugin** that reads your hardware and fills ``JointStateOutput`` (positions; optionally + velocity/effort), modeled on :code-dir:`src/plugins/so101_leader` (or + :code-dir:`src/plugins/rebot_devarm_leader`, a second instance of the same recipe on a + different motor bus). +#. A **config**: a ``collection_id``, the device joint names, and -- for ``ee_pose`` mode -- a URDF + and end-effector link. + +The schema, ``JointStateTracker``, ``JointStateSource``, and ``JointStateRetargeter`` are unchanged. + +.. seealso:: + + :doc:`../add_device` -- the general four-step device-plugin recipe (foot-pedal reference). + + :doc:`/references/retargeting/joint_space` -- the ``JointStateRetargeter`` (joint / EE modes), + the end-to-end example, and validation. diff --git a/docs/source/device/manus.rst b/docs/source/device/manus.rst index 19d04b42ed..60abb878b2 100644 --- a/docs/source/device/manus.rst +++ b/docs/source/device/manus.rst @@ -1,330 +1,9 @@ -.. SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 +:orphan: + MANUS Gloves ============ -A Linux-only plugin for integrating `MANUS `_ gloves -into the Isaac Teleop framework. It provides full hand-joint tracking via the -Manus SDK and injects the resulting poses into the OpenXR hand-tracking layer so -any downstream retargeter can consume them transparently. Optionally it also -publishes Manus flex-sensor (RawDeviceData) tip poses as ``JointSe3PoseOutput`` -tensors and consumes inbound haptic commands for vibration gloves. - -.. contents:: On this page - :local: - :depth: 2 - -Components ----------- - -- **Core library** (``manus_plugin_core``) — wraps the Manus SDK - (``libIsaacTeleopPluginsManus.so``) and exposes per-joint tracking data. -- **Plugin executable** (``manus_hand_plugin``) — the main plugin binary that - integrates with the Teleop system via CloudXR / OpenXR. -- **CLI tool** (``manus_hand_tracker_printer``) — a standalone diagnostic tool - that prints tracked joint data to the terminal and opens a real-time - **MANUS Data Visualizer** window showing the hand skeleton from two orthographic - views per hand. - -Prerequisites -------------- - -- **Linux** — x86_64 or aarch64 (tested on Ubuntu 22.04 / 24.04). -- **Manus SDK** for Linux — downloaded automatically by the install script. -- **System dependencies** — the install script installs required packages automatically. - -Installation ------------- - -MANUS access has two halves: **device permissions** (kernel/udev, lives on the -host) and **SDK + plugin build** (lives wherever you build, typically a -container). The two scripts below split along that line. - -Step 1: grant the host access to the Manus dongle (one-time) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Run this **on the host machine**, not inside a container. udev rules are -processed by ``systemd-udevd``, which does not run inside Docker — so -installing rules from a container has no effect. - -.. code-block:: bash - - cd src/plugins/manus - ./install_udev_rules.sh - # then unplug + replug the Manus dongle - -If you're using the Isaac ROS dev container (``isaac_ros run_dev``), it -bind-mounts ``/dev/bus/usb`` from the host, so once the host has the rules -applied the container will see the dongle with the right permissions. - -Step 2: build the SDK and plugin -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Run this **inside the build environment** (devcontainer or Isaac ROS container): - -.. code-block:: bash - - cd src/plugins/manus - ./install_manus.sh - -Pass ``--build-dir `` to reuse a non-default top-level CMake build -directory. - -The script will: - -1. Install the required system packages for MANUS Core Integrated. -2. Download MANUS SDK v3.2.1. -3. Extract and place the SDK for the host architecture in the correct location. -4. Build the plugin and the diagnostic tool. - -When run inside a container, ``install_manus.sh`` skips the udev step and -reminds you to run ``install_udev_rules.sh`` on the host. Pass ``--container`` -when the build environment does not expose standard container markers, such as -during a Docker BuildKit build. - -Manual installation (fallback only) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Steps 1 and 2 above are the supported path — most users should never need -this section. Only follow it if ``install_manus.sh`` fails (for example, the -SDK download is unreachable from your network) and you need to place the SDK -by hand. - -1. Download the MANUS Core SDK from - `MANUS Downloads `_. -2. From ``C++/SDKClient/ManusSDK`` in the archive, place ``include/`` and the - library for your architecture (``lib/amd64/libManusSDK-amd64.so`` or - ``lib/aarch64/libManusSDK-aarch64.so``, renamed to ``lib/libManusSDK.so``) - into a ``ManusSDK`` folder inside ``src/plugins/manus/``, or point CMake at a - different path by setting ``MANUS_SDK_ROOT``. -3. Follow the - `MANUS Getting Started guide for Linux `_ - to install the dependencies and configure device permissions. - -Expected directory layout after placing the SDK: - -.. code-block:: text - - src/plugins/manus/ - app/ - main.cpp - core/ - manus_hand_tracking_plugin.cpp - inc/ - manus/ - manus_glove_collection.hpp - manus_hand_tracking_plugin.hpp - tools/ - manus_hand_tracker_printer.cpp - ManusSDK/ <-- placed here - include/ - lib/ - libManusSDK.so - -Then build from the root: - -.. code-block:: bash - - cd ../../.. # navigate to root - cmake -S . -B build - cmake --build build --target manus_hand_plugin manus_hand_tracker_printer -j - cmake --install build --component manus - -Running the Plugin ------------------- - -1. Start the CloudXR runtime and load its environment -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The MANUS plugin connects to the Teleop session through the CloudXR / OpenXR -runtime, so the runtime must be running and its environment sourced in the -shell that launches the plugin. - -Start the CloudXR runtime. It runs as a background service that outlives -this shell, until you stop it with ``python -m isaaccapture.cloudxr.service stop``: - -.. code-block:: bash - - python -m isaaccapture.cloudxr.service start - -In the shell you will use to run the plugin, source the environment file -that the runtime writes on startup. This points the OpenXR loader at CloudXR: - -.. code-block:: bash - - source ~/.cloudxr/run/cloudxr.env - -See :ref:`dedicated-cloudxr-runtime` for the full CloudXR service setup, -including EULA acceptance, and :ref:`whitelist-firewall-ports` in the Quick -Start for firewall configuration. - -2. Verify with the CLI tool -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Verify that the gloves are working using the CLI tool. It runs from the build -tree, while the plugin in the next step runs from the install tree; both come -from the same build: - -.. code-block:: bash - - ./build/bin/manus_hand_tracker_printer - -The tool prints joint positions to the terminal and opens a **MANUS Data -Visualizer** window showing a top-down and side view of each hand. - -3. Run the plugin -~~~~~~~~~~~~~~~~~~ - -The plugin is installed to the ``install`` directory. Ensure the CLI tool is -not running when you launch the plugin — only one process can hold the Manus -SDK connection at a time. - -.. code-block:: bash - - ./install/plugins/manus/manus_hand_plugin - -By default the plugin enables human hand injection, flex-sensor push, and -haptic read. Restrict datasets with ``--datasets=`` (comma-separated): - -.. code-block:: bash - - ./install/plugins/manus/manus_hand_plugin --datasets=human,sensors,haptic - ./install/plugins/manus/manus_hand_plugin --datasets=human # skeleton only - ./install/plugins/manus/manus_hand_plugin --datasets=sensors # flex sensors only - -Data paths ----------- - -Human (OpenXR hand injection) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The Manus raw skeleton stream is mapped to 26 OpenXR hand joints and pushed -through ``HandInjector`` (requires ``XR_NVX1_device_interface_base``). Downstream -hosts consume hands via ``HandsSource`` as with any other hand-tracking plugin. - -Sensors (flex tips via SchemaPusher) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -When gloves report at least five RawDeviceData flex sensors, the plugin pushes -a ``JointSe3PoseOutput`` (tensor id ``joint_se3_pose``) on: - -- ``manus_sensors_left`` -- ``manus_sensors_right`` - -Each frame is stamped ``type = JointType.HAND_RAW`` and carries the five -fingertips keyed by ``JointName`` -(``HAND_RAW_THUMB_TIP`` .. ``HAND_RAW_LITTLE_TIP``), position in meters and -orientation as an xyzw quaternion, in the Manus SDK frame after the plugin's -VUH coordinate setup. A joint absent from a frame is not tracked. Poses are raw -Manus flex transforms; hosts that mask against a re-framed skeleton must apply -their own sensor-pose processing. - -This path requires CloudXR tensor **push** extensions -(``XR_NVX1_push_tensor`` and ``XR_NVX1_tensor_data``). - -Host-side consumption example: - -.. code-block:: python - - from isaaccapture.deviceio_trackers import JointSe3PoseTracker - from isaaccapture.schema import JointName, JointType - - left = JointSe3PoseTracker("manus_sensors_left") - # ... register with a DeviceIOSession, then each tick: - data = left.get_data(session).data - assert data is None or data.type == JointType.HAND_RAW - thumb = data.lookup(JointName.HAND_RAW_THUMB_TIP) if data else None - if thumb is not None: - position, orientation = thumb.pose.position, thumb.pose.orientation - -See :code-file:`examples/mcap_record_replay/python/isaaccapture_examples/mcap_record_replay/record_joint_se3_pose.py` for a -runnable recorder. Run ``python -m isaaccapture_examples.mcap_record_replay.replay_joint_se3_pose`` -to play one back without hardware. - -Haptic (inbound vibration) -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The plugin reads ``HapticCommand`` on collection ``manus_glove_haptic`` -(``XR_NVX1_tensor_data``) and drives five finger motors via the Manus SDK. -See the haptic feedback example and -``isaaccapture.haptic_devices.glove.haptic_glove_device``. - -Wrist Positioning — Controllers vs Optical Hand Tracking ---------------------------------------------------------- - -Two sources are available for positioning the MANUS gloves in 3D space: - -- **Controller adapters** — attach Quest 3 controllers to the MANUS Universal - Mount on the back of the glove. The controller pose drives wrist placement. -- **Optical hand tracking** — use the HMD's built-in optical hand tracking to - position the hands. No physical controller adapter required. - -The plugin selects the source automatically at runtime: optical hand tracking is -preferred when ``XR_MNDX_xdev_space`` is supported and the runtime reports an -actively tracked wrist pose; otherwise it falls back to the controller pose. - -.. note:: - - When using controller adapters it is advisable to disable the HMD's automatic - hand-tracking–to–controller switching to avoid unexpected source changes - mid-session. - -Troubleshooting ---------------- - -.. list-table:: - :widths: 40 60 - :header-rows: 1 - - * - Symptom - - Resolution - * - SDK download fails - - Check your internet connection and re-run the install script. - * - ``uv not found. Please install uv: ...`` - - Install uv and re-run: ``curl -LsSf https://astral.sh/uv/install.sh | sh``. - It installs to ``~/.local/bin``, so also add that to ``PATH`` for the - current shell (and ``~/.bashrc``, to make it stick): - ``export PATH="$HOME/.local/bin:$PATH"``. - * - ``Missing build tools: ... patchelf ...`` - - CMake's dependency preflight lists every missing tool and the exact - install command in the same message, e.g. - ``sudo apt-get install -y patchelf``. Run the printed command and - re-run the install script. - * - Manus SDK not found at build time - - With manual installation, ensure ``ManusSDK`` is inside - ``src/plugins/manus/`` or set ``MANUS_SDK_ROOT`` to your installation path. - * - Manus SDK not found at runtime - - The build configures RPATH automatically. If you moved the SDK after - building, set ``LD_LIBRARY_PATH`` to its ``lib/`` directory. - * - No data received - - Ensure MANUS Core is running and the gloves are connected and calibrated. - The plugin logs ``Left glove connected (ID ); start streaming.`` (or - ``Right ...``) when MANUS Core reports a glove; without that line it has - no glove to stream. - * - CloudXR runtime errors - - Check that a runtime is serving with ``python -m isaaccapture.cloudxr.service status``, - and that ``~/.cloudxr/run/cloudxr.env`` has been sourced in the same - terminal as the plugin. - * - Permission denied for USB devices - - udev rules must be installed on the host. Run ``./install_udev_rules.sh`` - from the host (not inside a container), then unplug and replug the - MANUS dongle. Verify on the host with ``ls -l /dev/hidraw*`` — entries for the - dongle should be mode ``0666``. - * - ``udevadm control --reload-rules`` fails with "No such file or directory" - - You're inside a container. ``systemd-udevd`` doesn't run in containers, - so this command can never succeed there. Run ``install_udev_rules.sh`` - on the host instead. - * - Dongle not visible inside the Isaac ROS container (``lsusb`` doesn't - show vendor ``3325``) - - The container needs ``/dev/bus/usb`` bind-mounted from the host. The - Isaac ROS dev container does this automatically; for a custom - ``docker run``, add ``-v /dev/bus/usb:/dev/bus/usb`` (or - ``--device=/dev/hidraw`` for a specific device). - -License -------- - -Source files are covered by their stated licenses (Apache-2.0). The Manus SDK is -proprietary to MANUS and is subject to its own license; it is **not** redistributed -by this project. +This page has moved to :doc:`/device/hand_tracking/manus`. diff --git a/docs/source/device/oak.rst b/docs/source/device/oak.rst index d7ebd94582..2a74a32dfd 100644 --- a/docs/source/device/oak.rst +++ b/docs/source/device/oak.rst @@ -1,280 +1,9 @@ -.. SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 +:orphan: + OAK Camera Plugin ================= -C++ plugin that captures H.264 video from OAK cameras and saves to raw H.264 -files. Source and plugin README: -:code-file:`src/plugins/oak/README.md`. - -.. contents:: On this page - :local: - :depth: 2 - -Features --------- - -- **Hardware H.264 Encoding** — Uses OAK's built-in video encoder -- **Raw H.264 Recording** — Writes H.264 NAL units directly to file (no - container overhead) -- **OpenXR Integration** — Optional CloudXR runtime integration -- **Self-contained build** — DepthAI built automatically via CMake - -Build ------ - -DepthAI v3.x is fetched and built automatically via FetchContent. Its transitive -dependencies are managed by **vcpkg**: rather than tracking our own ``vcpkg.json``, -the build downloads **DepthAI's own vcpkg manifest** for the pinned tag at configure -time (see ``cmake/DepthAIVcpkgManifest.cmake``) and enables its ``usb`` feature for -libusb. The first build takes ~10–15 minutes (mostly vcpkg deps + DepthAI); -subsequent builds are fast. - -Prerequisites -~~~~~~~~~~~~~ - -.. code-block:: bash - - # Install build tools for vcpkg to build libusb (Linux) - sudo apt install libudev-dev autoconf automake libtool pkg-config - - # Install vcpkg (one-time) - git clone https://github.com/microsoft/vcpkg ~/.vcpkg - ~/.vcpkg/bootstrap-vcpkg.sh - echo 'export VCPKG_ROOT=~/.vcpkg' >> ~/.bashrc && source ~/.bashrc - -Configure and Build -~~~~~~~~~~~~~~~~~~~ - -.. warning:: - - ``CMAKE_TOOLCHAIN_FILE`` is a CMake constraint: it is only read on the - **first** configure of a build tree and is silently ignored on any - re-configure. **If** ``build/`` **already exists, delete it before running - the commands below**, otherwise the vcpkg toolchain will not be picked up and - configure will fail. - -.. code-block:: bash - - cd IsaacTeleop - - rm -rf build/ # required if build/ already exists or was previously configured without CMAKE_TOOLCHAIN_FILE - cmake -B build -DBUILD_PLUGIN_OAK_CAMERA=ON \ - -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake - cmake --build build --target camera_plugin_oak --parallel - -Usage ------ - -.. code-block:: bash - - # Record a single color stream - ./build/src/plugins/oak/camera_plugin_oak --add-stream=camera=Color,output=./color.h264 - - # Record multiple streams - ./build/src/plugins/oak/camera_plugin_oak \ - --add-stream=camera=Color,output=./color.h264 \ - --add-stream=camera=MonoLeft,output=./left.h264 \ - --add-stream=camera=MonoRight,output=./right.h264 - - # Record with a live preview window - ./build/src/plugins/oak/camera_plugin_oak \ - --add-stream=camera=Color,output=./color.h264 --preview - - # Show help - ./build/src/plugins/oak/camera_plugin_oak --help - -Press ``Ctrl+C`` to stop recording. - -Record camera metadata in MCAP -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Each captured frame emits metadata (sequence number, timestamps) as FlatBuffer -``FrameMetadataOak`` messages. There are two ways to record it — -``--collection-prefix`` and ``--mcap-filename`` are **mutually exclusive**. - -Local MCAP file (``--mcap-filename``) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The plugin writes per-stream metadata directly to an MCAP file. No host-side -tracker or TeleopSession is required. - -.. code-block:: bash - - ./build/src/plugins/oak/camera_plugin_oak \ - --add-stream=camera=Color,output=./color.h264 \ - --add-stream=camera=MonoLeft,output=./left.h264 \ - --mcap-filename=./metadata.mcap - -TeleopSession (multi-device recording) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Record camera metadata in the same MCAP file as other teleop data (hands, head, -controllers, etc.): - -1. **Plugin** — launch ``oak_camera`` via PluginManager or - :doc:`TeleopSession <../getting_started/teleop_session>`'s ``PluginConfig``, - passing ``--collection-prefix`` so the plugin pushes metadata via OpenXR - ``SchemaPusher``. -2. **Host trackers** — create one ``FrameMetadataTrackerOak`` **per stream**, - each with collection id ``{collection_prefix}/{StreamName}`` using the - **same** prefix the plugin was given (see :doc:`trackers`). TeleopSession's - DeviceIO layer uses the live tracker implementation to read the pushed - tensors and write MCAP channels. -3. **MCAP config** — add every tracker to both ``TeleopSessionConfig.trackers`` - and ``McapRecordingConfig.tracker_names``, each under its own base name. See - also :doc:`../references/mcap_record_replay`. - -.. code-block:: python - - from pathlib import Path - - from isaaccapture.deviceio import FrameMetadataTrackerOak, McapRecordingConfig - from isaaccapture.teleop_session_manager import PluginConfig, TeleopSession, TeleopSessionConfig - - PLUGIN_ROOT = Path("build/src/plugins") # or your installed plugin search path - COLLECTION_PREFIX = "oak_camera" - STREAM_NAMES = ["Color", "MonoLeft"] - - # One tracker per stream, each on the collection the plugin publishes it under. - oak_trackers = { - name: FrameMetadataTrackerOak(f"{COLLECTION_PREFIX}/{name}") - for name in STREAM_NAMES - } - - config = TeleopSessionConfig( - app_name="OakTeleop", - pipeline=pipeline, # your retargeting pipeline - trackers=list(oak_trackers.values()), - mcap_config=McapRecordingConfig( - "recording.mcap", - [(tracker, f"oak_metadata_{name.lower()}") for name, tracker in oak_trackers.items()], - ), - plugins=[ - PluginConfig( - plugin_name="oak_camera", - plugin_root_id="oak_camera", - search_paths=[PLUGIN_ROOT], - plugin_args=[ - "--add-stream=camera=Color,output=./color.h264", - "--add-stream=camera=MonoLeft,output=./left.h264", - f"--collection-prefix={COLLECTION_PREFIX}", - ], - ), - ], - ) - - with TeleopSession(config) as session: - while True: - session.step() # metadata is recorded each update - -See also :code-file:`examples/oxr/python/test_oak_camera.py` (``--mode -schema-pusher``) for a standalone PluginManager + DeviceIOSession example of -the same SchemaPusher flow. - -Configuration -------------- - -.. list-table:: - :widths: 30 15 55 - :header-rows: 1 - - * - Option - - Default - - Description - * - ``--add-stream=camera=,output=`` - - (at least one required) - - Add a capture stream. ``camera`` is one of ``Color``, ``MonoLeft``, - ``MonoRight``. Repeatable. - * - ``--fps=N`` - - 30 - - Frame rate for all streams - * - ``--bitrate=N`` - - 8000000 - - H.264 bitrate (bps) - * - ``--quality=N`` - - 80 - - H.264 quality (1–100) - * - ``--device-id=ID`` - - first available - - OAK device MxId - * - ``--preview`` - - off - - Open a live SDL2 window showing the color camera feed - * - ``--collection-prefix=PREFIX`` - - - - Push per-frame metadata via OpenXR tensor extensions - * - ``--mcap-filename=PATH`` - - - - Record per-frame metadata to an MCAP file - -Architecture ------------- - -.. code-block:: text - - ┌─────────────────┐ ┌──────────────────┐ ┌───────────────┐ ┌──────────────┐ - │ OakCamera │────>│ FrameSink │────>│ RawDataWriter │────>│ .h264 File │ - │ (H.264 encode) │ │ (write + push) │ │ (file writer) │ │ │ - └─────────────────┘ └──────┬───────────┘ └───────────────┘ └──────────────┘ - core/ │ core/ core/ - v - ┌──────────────────┐ - │ MetadataPusher │ - │ (OpenXR tensor) │ - └──────────────────┘ - -Dependencies ------------- - -Transitive dependencies via **vcpkg**, DepthAI and SDL2 via FetchContent: - -- **DepthAI v3.x** — OAK camera interface (FetchContent) -- **SDL2** — Live preview window (FetchContent, used by ``--preview``) -- **vcpkg** — DepthAI's transitive deps (nlohmann-json, spdlog, libusb, - libarchive, etc.), resolved from DepthAI's own auto-downloaded manifest - (``usb`` feature enabled for libusb) - -Output Format -------------- - -The plugin writes raw H.264 NAL units (Annex B format) to ``.h264`` files. To -play or convert: - -.. code-block:: bash - - # Play with ffplay - ffplay -f h264 recording.h264 - - # Convert to MP4 - ffmpeg -f h264 -i recording.h264 -c copy output.mp4 - - # Convert with specific framerate - ffmpeg -f h264 -framerate 30 -i recording.h264 -c copy output.mp4 - -Troubleshooting ---------------- - -Device USB Connection -~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: bash - - # Check OAK camera connection - lsusb | grep 03e7 - -For more connection troubleshooting, see the -`OAK USB deployment guide `_. - -Inspect the Recording -~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: bash - - # Verify recording (convert to MP4 first) - ffmpeg -f h264 -i recording.h264 -c copy recording.mp4 - ffprobe recording.mp4 - - # Check frame count - ffprobe -show_entries stream=nb_frames recording.mp4 +This page has moved to :doc:`/device/cameras/oak`. diff --git a/docs/source/device/oglo.rst b/docs/source/device/oglo.rst index 8b3bd6938c..bedeedc784 100644 --- a/docs/source/device/oglo.rst +++ b/docs/source/device/oglo.rst @@ -1,144 +1,9 @@ .. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 +:orphan: + OGLO Tactile Glove Plugin ========================= -C++ plugin that streams **OGLO** tactile gloves over BLE into the standard -DeviceIO recording path. Each glove (one BLE device per hand) provides **80 -tactile taxels** (5 fingers × 4×4) plus a **6-axis IMU** at 100 Hz. Source and -plugin README: :code-file:`src/plugins/oglo_tactile/README.md`. - -This plugin follows the :doc:`add_device` pattern (FlatBuffer schema → plugin -``SchemaPusher`` → tracker → MCAP) and is modeled on the OAK camera plugin. - -.. figure:: ../_static/oglo-demo.gif - :align: center - :width: 100% - - Both OGLO gloves driving a live per-finger contact heatmap while manipulating - an object — 80 taxels + a 6-axis IMU per hand, streamed over BLE at 100 Hz and - time-synced with the operator's hand/head tracking. - -.. contents:: On this page - :local: - :depth: 2 - -Components ----------- - -- **Schema** — :code-file:`src/core/schema/fbs/oglo_tactile.fbs` - (``OgloGloveSample`` / ``OgloGloveSampleRecord``). -- **Plugin** — :code-dir:`src/plugins/oglo_tactile` (BLE read → parse → OpenXR push). -- **Tracker** — ``OgloTactileTracker``, with live backend - ``LiveOgloTactileTrackerImpl``. Both are generated at configure time from the - ``oglo_tactile`` entry in - :code-file:`src/core/deviceio_trackers/trackers.toml`. - -Build ------ - -Linux only (BlueZ). The plugin is off by default. - -.. code-block:: bash - - sudo apt install libdbus-1-dev # BlueZ daemon + libdbus for the BLE backend - cmake -B build -DBUILD_PLUGIN_OGLO=ON - cmake --build build --target oglo_tactile_plugin --parallel - -``nlohmann/json`` (MIT) is fetched automatically. - -.. note:: - - **BLE backend.** The plugin talks to BlueZ directly over `libdbus - `_ (AFL-2.1, permissive), so - the build carries **no copyleft dependency** and works out of the box. The - transport is isolated behind the ``OgloBleClient`` interface - (:code-file:`src/plugins/oglo_tactile/ble/oglo_ble_client.hpp`), so an - alternative backend can be dropped in via ``make_ble_client()`` without - touching the parser, schema, or tracker. - -Usage ------ - -The plugin streams one glove (``--side``) and pushes it via OpenXR for a host -``OgloTactileTracker`` to record: - -.. code-block:: bash - - # Push for a host OgloTactileTracker into a shared session MCAP - ./build/src/plugins/oglo_tactile/oglo_tactile_plugin --side right --collection-prefix=oglo - -The packet parser reads the device **Config characteristic** first and branches -on the notify ``flags`` byte (``0x04`` packed12 v5 — primary; ``0x02``/``0x01`` -legacy schema-4 fallback), so payload sizes are never hardcoded. - -Recorded data -------------- - -The host ``OgloTactileTracker`` records per hand into channels -``oglo_/oglo`` and ``oglo_/oglo_tracked``, carrying -``core.OgloGloveSampleRecord``: ``seq``, ``device_time_us``, ``taxels[80]`` (raw -12-bit, ``finger,row,col``), and a 6-axis IMU, each with a ``DeviceDataTimestamp`` -whose ``sample_time_local_common_clock`` is on the shared host monotonic clock — -so OGLO aligns in time with hand/head streams. - -**Naming.** The glove's identity threads through several layers, but all of them -derive from the single ``--collection-prefix`` you pick (``oglo`` below), with the -hand side appended. For the **left** hand: - -.. list-table:: - :header-rows: 1 - :widths: 32 44 24 - - * - Layer - - Where it is set - - Value (left hand) - * - CLI flag - - ``oglo_tactile_plugin --collection-prefix`` - - ``oglo`` - * - OpenXR collection id - - ``OgloTactileTracker("/left")`` - - ``oglo/left`` - * - MCAP recording base name - - ``McapRecordingConfig`` tracker name - - ``oglo_left`` - * - MCAP channels - - ``OgloRecordingTraits`` (``oglo``, ``oglo_tracked``) - - ``oglo_left/oglo``, ``oglo_left/oglo_tracked`` - * - Plugin device path - - :code-file:`src/plugins/oglo_tactile/plugin.yaml` - - ``/glove/oglo_left`` - -The right hand mirrors this (``right`` / ``oglo_right``), so ``mcap info`` on a -recording shows ``oglo_left/oglo`` (+ ``_tracked``) and ``oglo_right/oglo`` -(+ ``_tracked``). - -A complete data-collection demo (MetaQuest hand/head + both gloves + a live -in-headset tactile heatmap) lives at -:code-dir:`examples/oglo_tactile`; see its ``README.md``. - -IMU frame ---------- - -Each sample includes a 6-axis IMU reading — a 3-axis accelerometer and 3-axis -gyroscope — as raw ``int16`` sensor counts. Scale: accel ±8 g -(``≈0.000244 g/LSB``), gyro ±2000 dps (``≈0.061 dps/LSB``). The IMU is -wrist-mounted, so its axes form a single **wrist-fixed frame** — they track the -wrist, not individual fingers, and are the sensor's own raw axes (not aligned to -an anatomical hand frame). For absolute orientation, apply the IMU-to-hand -extrinsic supplied with the hardware, or calibrate against gravity for just "up". - -Tests ------ - -``test_oglo_packet_parser`` validates the wire decode against the firmware's own -12-bit packing reference, plus the schema-4 fallback and malformed-packet -rejection: - -The plugin target above does not build the test, so build it first: - -.. code-block:: bash - - cmake --build build --target test_oglo_packet_parser --parallel - ctest --test-dir build -R oglo_packet_parser --output-on-failure +This page has moved to :doc:`/device/tactile/oglo`. diff --git a/docs/source/device/sharpa_avatar.rst b/docs/source/device/sharpa_avatar.rst index 1e3f7ca37b..a8c6687dc6 100644 --- a/docs/source/device/sharpa_avatar.rst +++ b/docs/source/device/sharpa_avatar.rst @@ -1,235 +1,9 @@ -.. SPDX-FileCopyrightText: Copyright (c) 2025-2026 Avatar SDK contributors. All rights reserved. +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 +:orphan: + Sharpa AvatarExo Glove ====================== -A Linux-only plugin for integrating Sharpa AvatarExo (AE) Gloves into the -Isaac Capture framework. The gloves stream IMU and encoder data through the -official Avatar SDK over USB. The plugin maps that stream into Isaac Capture's -usual surfaces: OpenXR hand tracking for the HUMAN skeleton, -``JointStateOutput`` tensors for the RAW and ROBOT 22-DoF joint sets, and -inbound per-finger vibration on a haptic collection. - -Any downstream consumer that already reads OpenXR hands (``HandsSource``) or -schema joint-state collections can use Sharpa AE Gloves through the same DeviceIO -interfaces. The plugin talks to the gloves directly, so do not run -``avatar-backend`` or Avatar Desktop at the same time. - -Plugin sources and the operator README live under -:code-dir:`src/plugins/sharpa_avatar`. A TeleopSession visualization example -is :code-dir:`examples/sharpa_avatar`. - -.. contents:: On this page - :local: - :depth: 2 - -Data flow ---------- - -The glove SDK exposes three datasets plus haptics. HUMAN landmarks are placed -at a fused wrist pose and injected as OpenXR hands; RAW and ROBOT stay as -named joint-state collections: - -.. code-block:: text - - Sharpa AvatarExo Gloves (USB) - │ - ▼ - avatar_hand_plugin (Avatar SDK, production package) - │ - ├── HUMAN ──► OpenXR /hand/left, /hand/right (26 joints) - ├── RAW ──► avatar_raw_left / avatar_raw_right - ├── ROBOT ──► avatar_robot_left / avatar_robot_right - └── haptic ◄── avatar_glove_haptic (five finger vibration motors) - -.. seealso:: - - :doc:`/references/retargeting/sharpa` maps OpenXR hand tracking onto a - Sharpa robot hand. Avatar HUMAN output is the same OpenXR layer that - retargeter consumes. - -Prerequisites -------------- - -- **Linux x86_64** (tested on Ubuntu 22.04). -- **Pinned production Avatar SDK** (``avatar-sdk`` 1.7.3-17) from Sharpa's - signed APT repository. ``install_avatar_sdk.sh`` retrieves it; do not substitute - ``avatar-sdk-dev`` or ``avatar-sdk-beta``. -- **Sharpa AvatarExo Gloves** powered and connected over USB. Transport settings - are selected in ``/share/sdk_config.json`` (default root - ``/opt/avatar-sdk``). -- **CloudXR runtime** on the plugin host, unless a ``TeleopSession`` example - launches it for you. -- A built Isaac Capture checkout. CMake 3.24 or newer is required. - -The SDK stays external: headers, libraries, and runtime assets are not copied -into this repository or the plugin install prefix. - -Installation ------------- - -Install the pinned production SDK first if it is not already present: - -.. code-block:: bash - - ./src/plugins/sharpa_avatar/install_avatar_sdk.sh - -Then use the canonical top-level CMake workflow from the Isaac Capture root. -``-DAVATAR_SDK_ROOT`` selects a non-default SDK tree; that tree must already -be complete: - -.. code-block:: bash - - cmake -B build \ - -DBUILD_PLUGIN_SHARPA_AVATAR=ON \ - -DAVATAR_SDK_ROOT=/opt/avatar-sdk - cmake --build build --parallel - cmake --install build - -``-DBUILD_PLUGIN_SHARPA_AVATAR=ON`` without a usable SDK under the selected -root skips this plugin and the rest of Isaac Capture still configures. See -:doc:`/getting_started/build_from_source/index`. - -USB access (one-time, on the host) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -udev rules must run **on the host**, not inside a container: - -.. code-block:: bash - - ./src/plugins/sharpa_avatar/install_udev_rules.sh - # then unplug and reconnect the glove - -Running the plugin ------------------- - -The plugin publishes through the CloudXR / OpenXR runtime. Start the runtime -and source its environment in the shell that launches the plugin: - -.. code-block:: bash - - python -m isaaccapture.cloudxr.service start # runs in the background - source ~/.cloudxr/run/cloudxr.env - ./install/plugins/sharpa_avatar/avatar_hand_plugin - -See :ref:`dedicated-cloudxr-runtime` and -:ref:`load-cloudxr-environment-variables` for the full service setup. - -``TeleopSession`` can launch the same binary through ``plugin.yaml``. The -visualization example does that for you (CloudXR and the plugin start unless -you opt out): - -.. code-block:: bash - - uv pip install -e ./examples/sharpa_avatar - python -m isaaccapture_examples.sharpa_avatar - -Gloves reconnect from ``update()`` if they drop offline. Only one process -may hold the Avatar SDK at a time. - -Datasets --------- - -By default every path is on. Restrict them with ``--datasets=`` -(comma-separated: ``human``, ``raw``, ``robot``, ``haptic``): - -.. code-block:: bash - - ./install/plugins/sharpa_avatar/avatar_hand_plugin --datasets=human,raw,robot,haptic - ./install/plugins/sharpa_avatar/avatar_hand_plugin --datasets=human - -.. list-table:: - :widths: 38 62 - :header-rows: 1 - - * - Device or collection - - Contents - * - ``/hand/left``, ``/hand/right`` - - OpenXR 26-joint poses from Avatar HUMAN landmarks - * - ``avatar_raw_left``, ``avatar_raw_right`` - - RAW 22-DoF encoder joint state - * - ``avatar_robot_left``, ``avatar_robot_right`` - - ROBOT 22-DoF retargeted joint state - * - ``avatar_glove_haptic`` - - Inbound per-finger vibration commands - -HUMAN (OpenXR hands) -~~~~~~~~~~~~~~~~~~~~ - -HUMAN landmarks are mapped onto the 26 OpenXR hand slots (unsupported slots -stay invalid) and pushed with ``HandInjector``. Hosts read them through -``HandsSource`` like any other hand-tracking plugin. - -RAW and ROBOT (joint-state tensors) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Each side pushes a ``JointStateOutput`` (tensor id ``joint_state``) with the -22 named DoF from the pinned SDK. Consume them with ``JointStateSource`` -and the collection ids above. - -Haptic -~~~~~~ - -The plugin reads ``HapticCommand`` on ``avatar_glove_haptic`` and drives -five finger vibration motors. The visualization example uses the same -per-finger pinch path as the generic glove haptic sample. - -Wrist pose ----------- - -Sharpa AE Glove data is wrist-relative, so the plugin anchors HUMAN landmarks -with a pose from ``WristPoseSource`` before publishing OpenXR hands. It prefers -optical hand tracking and falls back to the controller aim pose when needed. -If no wrist pose is available yet, the plugin still publishes a local skeleton -with VALID-only flags; TRACKED is set once the selected wrist source is actively -tracked. - -The example places that local skeleton into a stable left/right display -frame. Pass ``--world-frame`` to leave the OpenXR poses unmodified. - -Troubleshooting ---------------- - -.. list-table:: - :widths: 40 60 - :header-rows: 1 - - * - Symptom - - Fix - * - Avatar SDK installation fails - - Confirm access to Sharpa's production APT endpoint and re-run - ``install_avatar_sdk.sh``. - * - CMake skipped the plugin - - The SDK is missing under the selected root. Run - ``install_avatar_sdk.sh`` and reconfigure with - ``-DBUILD_PLUGIN_SHARPA_AVATAR=ON``. - * - CMake rejected the Avatar SDK - - Install the pinned production package, or point ``AVATAR_SDK_ROOT`` - at a complete SDK tree. A ``-dev`` / ``-beta`` package at - ``/opt/avatar-sdk`` is not the pin. - * - USB glove is not detected - - Run ``install_udev_rules.sh`` on the host, then unplug and reconnect - the AE Gloves. - * - Plugin binary is not found - - Build and install through the top-level CMake workflow. The example looks under - ``install/plugins/sharpa_avatar/``. - * - HUMAN / RAW / ROBOT stay offline - - Power the gloves, and stop Avatar Desktop, ``avatar-backend``, or - any other process using the Avatar SDK. - * - Viser shows an empty grid - - HUMAN must be enabled (default). RAW-only is joint angles, not a - 3D skeleton. Confirm the example is not launched with - ``--datasets`` that omit ``human``. - * - No vibration - - Keep ``haptic`` in ``--datasets`` and close a fingertip toward the - thumb (example pinch path). - * - CloudXR runtime errors - - Check ``python -m isaaccapture.cloudxr.service status``, and source - ``~/.cloudxr/run/cloudxr.env`` in the same terminal as the plugin. - -License -------- - -Plugin sources are Apache-2.0. The Avatar SDK is proprietary to Sharpa and -is subject to its own license; this project does not redistribute it. +This page has moved to :doc:`/device/hand_tracking/sharpa_avatar`. diff --git a/docs/source/device/soma.rst b/docs/source/device/soma.rst index 30fd524333..69c1e95422 100644 --- a/docs/source/device/soma.rst +++ b/docs/source/device/soma.rst @@ -1,245 +1,9 @@ .. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 +:orphan: + SOMA Pose Transport =================== -Isaac Teleop can transport SOMA body and hand data through FlatBuffers and -``SchemaPusher``. Producers publish either joint rotations or evaluated joint -poses. Consumer-side sources normalize both representations to global joint -poses for retargeting or visualization. - -The schemas and examples are defined and verified against the SOMA-X v0.3.1 -pose interface. See the `SOMA-X repository `_ -for the upstream model and assets. - -.. contents:: On this page - :local: - :depth: 2 - -Data flow ---------- - -Body and hand data use separate schemas and collections. Applications can -select their body and hand sources independently. - -.. code-block:: text - - vendor body SDK -> SOMA body FlatBuffer -> SchemaPusher -> SOMA body tracker - vendor hand SDK -> SOMA hand FlatBuffer -> SchemaPusher -> SOMA hand tracker - -The body schema contains SOMA's complete body control set, including its hand -controls. The standalone hand schemas provide independent left and right hand -streams, identified by ``hand_side`` (``SomaHandSide.LEFT`` or -``SomaHandSide.RIGHT``). Isaac Teleop does not automatically fuse standalone -hands with the hand controls in a body payload. - -Choose a representation ------------------------ - -Use the representation closest to the producer's native data. - -.. list-table:: - :header-rows: 1 - :widths: 20 30 50 - - * - Representation - - FlatBuffer types - - Behavior - * - ``joint-rotations`` - - ``SomaBodyJointRotations`` and ``SomaHandJointRotations`` - - Publishes SOMA joint rotations and the global Hips or Wrist translation. - The consumer evaluates forward kinematics against a prepared SOMA - identity. - * - ``joint-poses`` - - ``SomaBodyJointPoses`` and ``SomaHandJointPoses`` - - Publishes global position and orientation for each provided joint. The - consumer maps these poses directly and does not run per-frame forward - kinematics. - -``joint-rotations`` most directly matches the SOMA-X pose-control interface. -``joint-poses`` is an Isaac Teleop transport profile for producers that already -have evaluated global joint transforms. - -Both representations use positions in meters and unit XYZW quaternions. They -use a right-handed reference frame with positive Y up and positive Z forward. -The publisher must convert its native skeleton and reference frame before -serialization. See :ref:`tracker-reference` for the complete field and -validity contracts. - -Consumers normalize finite quaternions whose norm is greater than ``1e-12``. A -zero, degenerate, or non-finite quaternion, position, or global translation is -unavailable. Quaternion signs are not canonicalized, so ``q`` and ``-q`` remain -equivalent. - -Rotation consumers resolve the SOMA v0.3.1 reference through SOMA-X and pass it -explicitly during forward kinematics. The prepared identity and subject scale -remain caller-selected calibration and are not constrained by this check. - -Body and hand joint values are sorted, unique keyed vectors. Every entry carries -an explicit ``SomaBodyJoint`` or ``SomaHandJoint`` identifier. A missing joint is -unavailable for that frame; producers do not serialize placeholder values for -joints they cannot provide. Consumers expand the keyed values into SOMA-X's -canonical dense order when invoking its body or hand helpers. - -Build the examples ------------------- - -Start with a configured Isaac Teleop checkout. See -:doc:`../getting_started/build_from_source/index` for the complete build -prerequisites. Install the live viewer and both deterministic publishers: - -.. code-block:: bash - - uv pip install -e ./examples/deviceio_live_view - uv pip install -e ./examples/soma_body_publisher - uv pip install -e ./examples/soma_hand_publisher - - .venv/bin/cmake -S . -B build - .venv/bin/cmake --build build \ - --target isaacteleop_python \ - -j4 - -The example packages install ``py-soma-x==0.3.1``. -The publishers use the Python-accessible ``TensorPushTracker`` and its native -``SchemaPusher`` transport; no separate sender executable is needed. Demo samples -use the session's monotonic clock for both timestamps. - -Run the body example --------------------- - -Start the viewer first. It launches the CloudXR runtime and prints the viewer -URL. Open that URL before starting the publisher. On a remote host, replace -``localhost`` in the URL with the host name or IP address. - -.. code-block:: bash - - uv run --no-sync python -m isaaccapture_examples.deviceio_live_view \ - --accept-eula \ - --body-schema soma - -In another terminal, load the runtime environment and start the body -publisher: - -.. code-block:: bash - - source ~/.cloudxr/run/cloudxr.env - uv run --no-sync python -m isaaccapture_examples.soma_body_publisher \ - --loop - -These commands use ``joint-rotations``. To publish evaluated joint poses, add -``--body-representation joint-poses`` to the publisher and -``--soma-body-representation joint-poses`` to the viewer. - -Run the hand example --------------------- - -Start the viewer with SOMA hands: - -.. code-block:: bash - - uv run --no-sync python -m isaaccapture_examples.deviceio_live_view \ - --accept-eula \ - --hand-schema soma - -In another terminal, load the runtime environment and start the hand -publisher: - -.. code-block:: bash - - source ~/.cloudxr/run/cloudxr.env - uv run --no-sync python -m isaaccapture_examples.soma_hand_publisher \ - --loop - -These commands use ``joint-rotations``. To publish evaluated joint poses, add -``--hand-representation joint-poses`` to the publisher and -``--soma-hand-representation joint-poses`` to the viewer. - -Collection IDs and schema identifiers -------------------------------------- - -A collection ID pairs a producer with a consumer and is not encoded in the -FlatBuffer payload. The tensor identifier selects the representation carried by -that collection. The demo publishers use these fixed collection IDs; another -producer can choose different IDs as long as its consumers use the same values. - -.. list-table:: - :header-rows: 1 - :widths: 30 30 40 - - * - Data - - Collection ID - - Viewer override - * - Body - - ``soma_body_demo`` - - ``--soma-body-collection-id`` - * - Left hand - - ``soma_hand_left_demo`` - - ``--soma-left-hand-collection-id`` - * - Right hand - - ``soma_hand_right_demo`` - - ``--soma-right-hand-collection-id`` - -The representation selected by the publisher and consumer must match. The -transport identifiers are ``soma_body_joint_rotations``, -``soma_body_joint_poses``, ``soma_hand_joint_rotations``, and -``soma_hand_joint_poses``. - -Vendor integration ------------------- - -A vendor integration converts its native skeleton to one of the checked-in -SOMA contracts and publishes it through ``SchemaPusher``. Use these schemas as -the serialization source: - -- :code-file:`src/core/schema/fbs/soma_common.fbs` -- :code-file:`src/core/schema/fbs/soma_body_common.fbs` -- :code-file:`src/core/schema/fbs/soma_body_joint_rotations.fbs` -- :code-file:`src/core/schema/fbs/soma_body_joint_poses.fbs` -- :code-file:`src/core/schema/fbs/soma_hand_joint_rotations.fbs` -- :code-file:`src/core/schema/fbs/soma_hand_joint_poses.fbs` - -The FlatBuffer transport and generated trackers do not depend on SOMA-X or -PyTorch. A rotation consumer uses SOMA-X and its resolved neutral identity to -evaluate forward kinematics. A joint-pose consumer does not need SOMA-X for -per-frame evaluation. - -Current limitations -------------------- - -- The examples evaluate rotations against a neutral identity. They do not - perform subject calibration. -- Body and standalone hand streams are not fused automatically. -- A producer and consumer must select the same representation. -- The live viewer selects sources at launch. It does not switch schemas while - running. - -Troubleshooting ---------------- - -No module named ``isaaccapture_examples`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Install the corresponding example with ``uv pip install -e`` and launch it -with ``uv run --no-sync`` from the repository root. - -The publisher cannot connect -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Start the viewer, open its client, and then source -``~/.cloudxr/run/cloudxr.env`` in the publisher terminal. The environment file -identifies the active CloudXR OpenXR runtime. - -The viewer receives no skeleton -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Confirm that the publisher and viewer use the same representation and -collection names. - -See also --------- - -- :code-file:`examples/soma_body_publisher/README.md` -- :code-file:`examples/soma_hand_publisher/README.md` -- :code-file:`examples/deviceio_live_view/README.md` -- :doc:`trackers` +This page has moved to :doc:`/device/body_tracking/soma`. diff --git a/docs/source/device/spacemouse.rst b/docs/source/device/spacemouse.rst index bb6f6c8c08..35108d771b 100644 --- a/docs/source/device/spacemouse.rst +++ b/docs/source/device/spacemouse.rst @@ -1,149 +1,9 @@ .. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 +:orphan: + SpaceMouse Plugin ================= -C++ plugin that streams a **3Dconnexion SpaceMouse** (a 6-DoF cap plus buttons) into the -standard DeviceIO path. It reads the device's raw HID node (``/dev/hidraw*``), so it needs no -3Dconnexion driver, and pushes six axes and the button state at 90 Hz. Source and plugin README: -:code-file:`src/plugins/spacemouse/README.md`. - -Validated devices: SpaceMouse Compact, SpaceMouse Wireless, SpaceNavigator, SpaceNavigator for -Notebooks, and the 3Dconnexion Universal Receiver. - -This plugin follows the :doc:`add_device` pattern (FlatBuffer schema → plugin ``SchemaPusher`` → -tracker → MCAP). - -.. contents:: On this page - :local: - :depth: 2 - -Components ----------- - -- **Schema** — :code-file:`src/core/schema/fbs/spacemouse.fbs` - (``SpaceMouseOutput`` / ``SpaceMouseOutputRecord``). -- **Plugin** — :code-dir:`src/plugins/spacemouse`: ``core/`` parses HID reports, discovers the - device and reconnects after an unplug; ``app/`` is the OpenXR push loop. -- **Tracker** — ``SpaceMouseTracker``, generated at configure time from the ``spacemouse`` entry - in :code-file:`src/core/deviceio_trackers/trackers.toml`. -- **Source node** — ``SpaceMouseSource`` turns the tracker's data into - ``spacemouse_translation``, ``spacemouse_rotation`` and ``spacemouse_buttons`` for retargeters. - -Device access -------------- - -``/dev/hidraw*`` nodes are readable by root only by default. Install the plugin's udev rule once -**on the host** (udev does not run in containers), then unplug and replug the SpaceMouse: - -.. code-block:: bash - - ./src/plugins/spacemouse/install_udev_rules.sh - -From a wheel install, run the copy bundled with the plugin: - -.. code-block:: bash - - "$(python -c 'from isaaccapture.teleop_session_manager import BUNDLED_PLUGINS_DIR; print(BUNDLED_PLUGINS_DIR)')"/spacemouse/install_udev_rules.sh - -The rule (:code-file:`src/plugins/spacemouse/70-spacemouse.rules`) makes SpaceMouse HID nodes -read-only for all users. No group membership is needed. - -Build ------ - -Linux only. The plugin builds with the default plugin set: - -.. code-block:: bash - - cmake -B build - cmake --build build --target spacemouse_plugin --parallel - -Usage ------ - -Start the plugin with the session through ``TeleopSessionConfig.plugins``, from the copy bundled -with ``isaaccapture`` (wheel installs included), and read it with ``SpaceMouseSource``, as -:code-file:`examples/teleop/python/spacemouse_printer_example.py` does: - -.. code-block:: python - - from isaaccapture.retargeting_engine.deviceio_source_nodes import SpaceMouseSource - from isaaccapture.teleop_session_manager import ( - BUNDLED_PLUGINS_DIR, - PluginConfig, - TeleopSessionConfig, - ) - - config = TeleopSessionConfig( - app_name="MyApp", - pipeline=pipeline_reading(SpaceMouseSource(name="spacemouse")), - plugins=[ - PluginConfig( - plugin_name="spacemouse", - plugin_root_id="spacemouse", - search_paths=[BUNDLED_PLUGINS_DIR], - plugin_args=[], # e.g. ["--device=/dev/hidraw3"] - ) - ], - ) - -Without ``--device`` the plugin opens the first validated model under ``/sys/class/hidraw`` and -finds it again after it is unplugged. For a Universal Receiver at an explicit path, add -``--combined-report``. - -The plugin can also run on its own, for example on another machine pushing to the same runtime, -next to any reader of the same collection: - -.. code-block:: bash - - ./install/plugins/spacemouse/spacemouse_plugin - ./install/plugins/spacemouse/spacemouse_plugin --device=/dev/hidraw3 - -Published data --------------- - -The plugin pushes ``SpaceMouseOutput`` to collection ``spacemouse`` (``--collection-id`` changes -it; match it in ``SpaceMouseSource`` / ``SpaceMouseTracker``): - -- ``translation``, ``rotation`` — ``[x, y, z]`` in ``[-1, 1]``, in the device's axis order. -- ``buttons`` — one entry per button, 1 while held. -- ``connected`` — whether the plugin has the device open. - -While no device is open the plugin keeps pushing with ``connected`` false and zero axes, so a -reader never holds the last deflection of an unplugged device; ``SpaceMouseSource`` then reports -its outputs as absent. An open device that stops reporting for 250 ms while deflected has its -axes zeroed. Recordings use MCAP channels ``spacemouse`` and ``spacemouse_tracked``. - -Retargeters ------------ - -``SpaceMouseToSe3RelRetargeter`` (end-effector delta), ``SpaceMouseToSe2Retargeter`` (base -velocity) and ``SpaceMouseGripperRetargeter`` (toggle on the left button) consume -``SpaceMouseSource``. - -Device status -------------- - -When ``TeleopSession`` starts the plugin, it reports device ``/spacemouse`` -(``spacemouse/spacemouse`` by default) through -:doc:`../references/device_provider_monitoring`: ``CONNECTED``, ``DISCONNECTED`` when no SpaceMouse -is found, or ``FAILED`` when one is found but cannot be opened. - -Troubleshooting ---------------- - -- **Status FAILED, or "Cannot open /dev/hidrawN (Permission denied)"** — install the udev rule - and replug the device. -- **Status DISCONNECTED with the device plugged in** — check ``cat - /sys/class/hidraw/*/device/uevent | grep HID_NAME``; an unlisted model can be read with - ``--device``. -- **Axes move but buttons never change** — some receivers report buttons on another interface; - try ``--device`` on the device's other ``hidraw`` node. - -Tests ------ - -``spacemouse_plugin_tests`` covers HID report parsing, discovery against a fake sysfs tree, and -reconnecting through a FIFO standing in for the device node. +This page has moved to :doc:`/device/spatial_controllers/spacemouse`. diff --git a/docs/source/device/spatial_controllers/index.rst b/docs/source/device/spatial_controllers/index.rst new file mode 100644 index 0000000000..808c2fe7ee --- /dev/null +++ b/docs/source/device/spatial_controllers/index.rst @@ -0,0 +1,15 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Spatial controllers +=================== + +Spatial controllers provide motion input for teleoperation. The SpaceMouse +guide covers its six-axis input and buttons. XR controller tracking is covered +in :doc:`/device/trackers`. + +.. toctree:: + :maxdepth: 1 + :titlesonly: + + spacemouse diff --git a/docs/source/device/spatial_controllers/spacemouse.rst b/docs/source/device/spatial_controllers/spacemouse.rst new file mode 100644 index 0000000000..07000e4e0e --- /dev/null +++ b/docs/source/device/spatial_controllers/spacemouse.rst @@ -0,0 +1,149 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +SpaceMouse Plugin +================= + +C++ plugin that streams a **3Dconnexion SpaceMouse** (a 6-DoF cap plus buttons) into the +standard DeviceIO path. It reads the device's raw HID node (``/dev/hidraw*``), so it needs no +3Dconnexion driver, and pushes six axes and the button state at 90 Hz. Source and plugin README: +:code-file:`src/plugins/spacemouse/README.md`. + +Validated devices: SpaceMouse Compact, SpaceMouse Wireless, SpaceNavigator, SpaceNavigator for +Notebooks, and the 3Dconnexion Universal Receiver. + +This plugin follows the :doc:`../add_device` pattern (FlatBuffer schema → plugin ``SchemaPusher`` → +tracker → MCAP). + +.. contents:: On this page + :local: + :depth: 2 + +Components +---------- + +- **Schema** — :code-file:`src/core/schema/fbs/spacemouse.fbs` + (``SpaceMouseOutput`` / ``SpaceMouseOutputRecord``). +- **Plugin** — :code-dir:`src/plugins/spacemouse`: ``core/`` parses HID reports, discovers the + device and reconnects after an unplug; ``app/`` is the OpenXR push loop. +- **Tracker** — ``SpaceMouseTracker``, generated at configure time from the ``spacemouse`` entry + in :code-file:`src/core/deviceio_trackers/trackers.toml`. +- **Source node** — ``SpaceMouseSource`` turns the tracker's data into + ``spacemouse_translation``, ``spacemouse_rotation`` and ``spacemouse_buttons`` for retargeters. + +Device access +------------- + +``/dev/hidraw*`` nodes are readable by root only by default. Install the plugin's udev rule once +**on the host** (udev does not run in containers), then unplug and replug the SpaceMouse: + +.. code-block:: bash + + ./src/plugins/spacemouse/install_udev_rules.sh + +From a wheel install, run the copy bundled with the plugin: + +.. code-block:: bash + + "$(python -c 'from isaaccapture.teleop_session_manager import BUNDLED_PLUGINS_DIR; print(BUNDLED_PLUGINS_DIR)')"/spacemouse/install_udev_rules.sh + +The rule (:code-file:`src/plugins/spacemouse/70-spacemouse.rules`) makes SpaceMouse HID nodes +read-only for all users. No group membership is needed. + +Build +----- + +Linux only. The plugin builds with the default plugin set: + +.. code-block:: bash + + cmake -B build + cmake --build build --target spacemouse_plugin --parallel + +Usage +----- + +Start the plugin with the session through ``TeleopSessionConfig.plugins``, from the copy bundled +with ``isaaccapture`` (wheel installs included), and read it with ``SpaceMouseSource``, as +:code-file:`examples/teleop/python/spacemouse_printer_example.py` does: + +.. code-block:: python + + from isaaccapture.retargeting_engine.deviceio_source_nodes import SpaceMouseSource + from isaaccapture.teleop_session_manager import ( + BUNDLED_PLUGINS_DIR, + PluginConfig, + TeleopSessionConfig, + ) + + config = TeleopSessionConfig( + app_name="MyApp", + pipeline=pipeline_reading(SpaceMouseSource(name="spacemouse")), + plugins=[ + PluginConfig( + plugin_name="spacemouse", + plugin_root_id="spacemouse", + search_paths=[BUNDLED_PLUGINS_DIR], + plugin_args=[], # e.g. ["--device=/dev/hidraw3"] + ) + ], + ) + +Without ``--device`` the plugin opens the first validated model under ``/sys/class/hidraw`` and +finds it again after it is unplugged. For a Universal Receiver at an explicit path, add +``--combined-report``. + +The plugin can also run on its own, for example on another machine pushing to the same runtime, +next to any reader of the same collection: + +.. code-block:: bash + + ./install/plugins/spacemouse/spacemouse_plugin + ./install/plugins/spacemouse/spacemouse_plugin --device=/dev/hidraw3 + +Published data +-------------- + +The plugin pushes ``SpaceMouseOutput`` to collection ``spacemouse`` (``--collection-id`` changes +it; match it in ``SpaceMouseSource`` / ``SpaceMouseTracker``): + +- ``translation``, ``rotation`` — ``[x, y, z]`` in ``[-1, 1]``, in the device's axis order. +- ``buttons`` — one entry per button, 1 while held. +- ``connected`` — whether the plugin has the device open. + +While no device is open the plugin keeps pushing with ``connected`` false and zero axes, so a +reader never holds the last deflection of an unplugged device; ``SpaceMouseSource`` then reports +its outputs as absent. An open device that stops reporting for 250 ms while deflected has its +axes zeroed. Recordings use MCAP channels ``spacemouse`` and ``spacemouse_tracked``. + +Retargeters +----------- + +``SpaceMouseToSe3RelRetargeter`` (end-effector delta), ``SpaceMouseToSe2Retargeter`` (base +velocity) and ``SpaceMouseGripperRetargeter`` (toggle on the left button) consume +``SpaceMouseSource``. + +Device status +------------- + +When ``TeleopSession`` starts the plugin, it reports device ``/spacemouse`` +(``spacemouse/spacemouse`` by default) through +:doc:`../../references/device_provider_monitoring`: ``CONNECTED``, ``DISCONNECTED`` when no SpaceMouse +is found, or ``FAILED`` when one is found but cannot be opened. + +Troubleshooting +--------------- + +- **Status FAILED, or "Cannot open /dev/hidrawN (Permission denied)"** — install the udev rule + and replug the device. +- **Status DISCONNECTED with the device plugged in** — check ``cat + /sys/class/hidraw/*/device/uevent | grep HID_NAME``; an unlisted model can be read with + ``--device``. +- **Axes move but buttons never change** — some receivers report buttons on another interface; + try ``--device`` on the device's other ``hidraw`` node. + +Tests +----- + +``spacemouse_plugin_tests`` covers HID report parsing, discovery against a fake sysfs tree, and +reconnecting through a FIFO standing in for the device node. diff --git a/docs/source/device/tactile/index.rst b/docs/source/device/tactile/index.rst new file mode 100644 index 0000000000..57b8522573 --- /dev/null +++ b/docs/source/device/tactile/index.rst @@ -0,0 +1,14 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Tactile devices +=============== + +Tactile devices capture contact measurements. For actuator output, see +:doc:`/references/haptic_feedback`. + +.. toctree:: + :maxdepth: 1 + :titlesonly: + + oglo diff --git a/docs/source/device/tactile/oglo.rst b/docs/source/device/tactile/oglo.rst new file mode 100644 index 0000000000..000a25f4dc --- /dev/null +++ b/docs/source/device/tactile/oglo.rst @@ -0,0 +1,144 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +OGLO Tactile Glove Plugin +========================= + +C++ plugin that streams **OGLO** tactile gloves over BLE into the standard +DeviceIO recording path. Each glove (one BLE device per hand) provides **80 +tactile taxels** (5 fingers × 4×4) plus a **6-axis IMU** at 100 Hz. Source and +plugin README: :code-file:`src/plugins/oglo_tactile/README.md`. + +This plugin follows the :doc:`../add_device` pattern (FlatBuffer schema → plugin +``SchemaPusher`` → tracker → MCAP) and is modeled on the OAK camera plugin. + +.. figure:: ../../_static/oglo-demo.gif + :align: center + :width: 100% + + Both OGLO gloves driving a live per-finger contact heatmap while manipulating + an object — 80 taxels + a 6-axis IMU per hand, streamed over BLE at 100 Hz and + time-synced with the operator's hand/head tracking. + +.. contents:: On this page + :local: + :depth: 2 + +Components +---------- + +- **Schema** — :code-file:`src/core/schema/fbs/oglo_tactile.fbs` + (``OgloGloveSample`` / ``OgloGloveSampleRecord``). +- **Plugin** — :code-dir:`src/plugins/oglo_tactile` (BLE read → parse → OpenXR push). +- **Tracker** — ``OgloTactileTracker``, with live backend + ``LiveOgloTactileTrackerImpl``. Both are generated at configure time from the + ``oglo_tactile`` entry in + :code-file:`src/core/deviceio_trackers/trackers.toml`. + +Build +----- + +Linux only (BlueZ). The plugin is off by default. + +.. code-block:: bash + + sudo apt install libdbus-1-dev # BlueZ daemon + libdbus for the BLE backend + cmake -B build -DBUILD_PLUGIN_OGLO=ON + cmake --build build --target oglo_tactile_plugin --parallel + +``nlohmann/json`` (MIT) is fetched automatically. + +.. note:: + + **BLE backend.** The plugin talks to BlueZ directly over `libdbus + `_ (AFL-2.1, permissive), so + the build carries **no copyleft dependency** and works out of the box. The + transport is isolated behind the ``OgloBleClient`` interface + (:code-file:`src/plugins/oglo_tactile/ble/oglo_ble_client.hpp`), so an + alternative backend can be dropped in via ``make_ble_client()`` without + touching the parser, schema, or tracker. + +Usage +----- + +The plugin streams one glove (``--side``) and pushes it via OpenXR for a host +``OgloTactileTracker`` to record: + +.. code-block:: bash + + # Push for a host OgloTactileTracker into a shared session MCAP + ./build/src/plugins/oglo_tactile/oglo_tactile_plugin --side right --collection-prefix=oglo + +The packet parser reads the device **Config characteristic** first and branches +on the notify ``flags`` byte (``0x04`` packed12 v5 — primary; ``0x02``/``0x01`` +legacy schema-4 fallback), so payload sizes are never hardcoded. + +Recorded data +------------- + +The host ``OgloTactileTracker`` records per hand into channels +``oglo_/oglo`` and ``oglo_/oglo_tracked``, carrying +``core.OgloGloveSampleRecord``: ``seq``, ``device_time_us``, ``taxels[80]`` (raw +12-bit, ``finger,row,col``), and a 6-axis IMU, each with a ``DeviceDataTimestamp`` +whose ``sample_time_local_common_clock`` is on the shared host monotonic clock — +so OGLO aligns in time with hand/head streams. + +**Naming.** The glove's identity threads through several layers, but all of them +derive from the single ``--collection-prefix`` you pick (``oglo`` below), with the +hand side appended. For the **left** hand: + +.. list-table:: + :header-rows: 1 + :widths: 32 44 24 + + * - Layer + - Where it is set + - Value (left hand) + * - CLI flag + - ``oglo_tactile_plugin --collection-prefix`` + - ``oglo`` + * - OpenXR collection id + - ``OgloTactileTracker("/left")`` + - ``oglo/left`` + * - MCAP recording base name + - ``McapRecordingConfig`` tracker name + - ``oglo_left`` + * - MCAP channels + - ``OgloRecordingTraits`` (``oglo``, ``oglo_tracked``) + - ``oglo_left/oglo``, ``oglo_left/oglo_tracked`` + * - Plugin device path + - :code-file:`src/plugins/oglo_tactile/plugin.yaml` + - ``/glove/oglo_left`` + +The right hand mirrors this (``right`` / ``oglo_right``), so ``mcap info`` on a +recording shows ``oglo_left/oglo`` (+ ``_tracked``) and ``oglo_right/oglo`` +(+ ``_tracked``). + +A complete data-collection demo (MetaQuest hand/head + both gloves + a live +in-headset tactile heatmap) lives at +:code-dir:`examples/oglo_tactile`; see its ``README.md``. + +IMU frame +--------- + +Each sample includes a 6-axis IMU reading — a 3-axis accelerometer and 3-axis +gyroscope — as raw ``int16`` sensor counts. Scale: accel ±8 g +(``≈0.000244 g/LSB``), gyro ±2000 dps (``≈0.061 dps/LSB``). The IMU is +wrist-mounted, so its axes form a single **wrist-fixed frame** — they track the +wrist, not individual fingers, and are the sensor's own raw axes (not aligned to +an anatomical hand frame). For absolute orientation, apply the IMU-to-hand +extrinsic supplied with the hardware, or calibrate against gravity for just "up". + +Tests +----- + +``test_oglo_packet_parser`` validates the wire decode against the firmware's own +12-bit packing reference, plus the schema-4 fallback and malformed-packet +rejection: + +The plugin target above does not build the test, so build it first: + +.. code-block:: bash + + cmake --build build --target test_oglo_packet_parser --parallel + ctest --test-dir build -R oglo_packet_parser --output-on-failure diff --git a/docs/source/device/trackers.rst b/docs/source/device/trackers.rst index 0cbc4dfc47..3313ce6cf3 100644 --- a/docs/source/device/trackers.rst +++ b/docs/source/device/trackers.rst @@ -1,5 +1,14 @@ -Device Trackers -=============== +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Spatial Trackers and Device Data +================================ + +Spatial tracking includes headset and controller poses, articulated hands and +bodies, and generic SE3 pose sources such as tracker pucks and mocap rigid bodies. +This guide is the common tracker reference; device-specific setup belongs in the +individual device guides. The same interfaces also carry camera metadata and +joint-space inputs. Trackers (defined in :code-dir:`src/core/deviceio_trackers`) are the consumer-side API for reading device data from an active :code-file:`DeviceIOSession `. @@ -39,7 +48,7 @@ All trackers follow the same lifecycle: .. note:: The ``DeviceIOSession`` is considered a low-level API. In practice, it is recommended to - use the :doc:`../getting_started/teleop_session` to manage a teleop session with multiple + use the :doc:`../teleop/teleop_session` to manage a teleop session with multiple device trackers and retargeters to work together. .. _data-schema-convention: @@ -475,7 +484,7 @@ When driving devices through the higher-level teleop session manager, vendor selection is carried on the DeviceIO source itself via its ``vendor`` argument (e.g. ``FullBodySource(name="full_body", vendor=deviceio.TrackerVendor("body.pico-xr"))``), so it travels with the pipeline into both extension discovery and session -construction; see :doc:`../getting_started/teleop_session`. +construction; see :doc:`../teleop/teleop_session`. .. _tracker-usage-example: diff --git a/docs/source/device/wuji_glove.rst b/docs/source/device/wuji_glove.rst index 038924d813..7503535ab6 100644 --- a/docs/source/device/wuji_glove.rst +++ b/docs/source/device/wuji_glove.rst @@ -1,133 +1,9 @@ -.. SPDX-FileCopyrightText: Copyright (c) 2026 Wuji Technology. All rights reserved. +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 +:orphan: + Wuji Glove ========== -A Linux-only plugin for integrating `Wuji `_ -data gloves into the Isaac Teleop framework. It reads the glove's 21-joint hand -skeleton over the network via the wuji_sdk C API and injects the resulting -poses into the OpenXR hand-tracking layer, so any downstream consumer can read -them as standard hand tracking. - -.. contents:: On this page - :local: - :depth: 2 - -Data flow ---------- - -The plugin combines the glove's wrist-relative skeleton with an absolute wrist -pose and publishes the result through OpenXR: - -.. code-block:: text - - Wuji glove (21-joint shape) ─┐ - ├─► wuji_glove plugin ──► CloudXR OpenXR runtime - Optical hand / controller ───┘ (wrist-pose fusion) (spatial hand joints) - -.. seealso:: - - :doc:`/references/retargeting/wuji` explains how to map OpenXR hand tracking - to the 20 joint commands of a Wuji Hand or Wuji Hand 2. - -Prerequisites -------------- - -- **Linux** — x86_64 or aarch64 (tested on Ubuntu 22.04 / 24.04). -- **wuji_sdk C SDK v2026.7.14** — the - ``wuji-sdk-c-2026.7.14--linux-gnu.tar.gz`` GitHub release asset, which - extracts to ``include/wuji_sdk.h`` plus ``lib/libwuji_sdk_c.so`` and its - companion ``lib/libwujihandcpp.so``. This is the C FFI, *not* the - ``wuji-sdk`` pip package; the install script downloads it automatically. -- **Wuji glove(s)** reachable on the same LAN as the plugin host. -- **CloudXR runtime** running on the plugin host. - -Installation ------------- - -``install.sh`` downloads the pinned C SDK release, verifies it against a -per-architecture SHA-256, extracts it next to the script (re-runs reuse the -extracted copy), and configures, builds, and installs the plugin: - -.. code-block:: bash - - ./src/plugins/wuji_glove/install.sh - -Pass ``--build-dir `` to reuse a non-default top-level CMake build -directory. - -To build offline, point ``WUJI_SDK_C_DIR`` at an already-extracted SDK copy — -or drive CMake directly: - -.. code-block:: bash - - cmake -B build -DBUILD_PLUGINS=ON -DBUILD_PLUGIN_WUJI_GLOVE=ON \ - -DWUJI_SDK_INCLUDE_DIR=/path/to/wuji-sdk-c/include \ - -DWUJI_SDK_LIB=/path/to/wuji-sdk-c/lib/libwuji_sdk_c.so - cmake --build build --target wuji_glove_plugin - cmake --install build --component wuji_glove - -Running the Plugin ------------------- - -The plugin reaches the Teleop session through the CloudXR / OpenXR runtime, so -start the runtime first and source its environment in the shell that launches -the plugin: - -.. code-block:: bash - - python -m isaaccapture.cloudxr.service start # runs in the background - source ~/.cloudxr/run/cloudxr.env - ./install/plugins/wuji_glove/wuji_glove_plugin - -See :ref:`dedicated-cloudxr-runtime` and -:ref:`load-cloudxr-environment-variables` for the full service setup. The plugin resolves each glove's side from device -metadata and reconnects automatically after a disconnect; applications can also -launch it through its ``plugin.yaml`` using ``TeleopSession``. - -Wrist pose ----------- - -The glove stream contains wrist-relative joints but no absolute 6DoF wrist -pose. Before injection, the plugin places that skeleton at a wrist pose from -optical hand tracking or from a controller aim pose combined with a calibrated -per-hand rigid offset. - -Set ``WUJI_GLOVE_WRIST_SOURCE`` to ``auto`` (the default), ``hand_tracking``, or -``controller``. In ``auto`` mode, optical tracking is preferred; the controller -is used when optical tracking provides no valid pose. Override the built-in -controller offsets when needed: - -.. code-block:: bash - - export WUJI_GLOVE_AIM_TO_WRIST_LEFT="px,py,pz,qx,qy,qz,qw" - export WUJI_GLOVE_AIM_TO_WRIST_RIGHT="px,py,pz,qx,qy,qz,qw" - -Position values are in meters and quaternion values use ``(x, y, z, w)``. -The built-in offsets assume mirrored left/right mounts. - -Troubleshooting ---------------- - -.. list-table:: - :widths: 40 60 - :header-rows: 1 - - * - Symptom - - Fix - * - ``install.sh`` fails with a SHA-256 mismatch - - The downloaded tarball does not match the pinned release. Re-run to - rule out a corrupted download; if it persists, do not bypass the check - — verify the release assets upstream. - * - No glove is discovered - - Confirm the glove is powered and on the same LAN/subnet as the plugin - host, then check the plugin log for ``wuji_scan`` errors. - * - Log says a second same-side glove is ignored - - The plugin binds one glove per side (first discovered wins) and ignores - later same-side gloves until restart. Power off the spare, or restart - the plugin to rebind. - * - Plugin runs but a downstream client sees no hand data - - The plugin and the client must talk to the same CloudXR runtime; check - that the runtime was up before both, and that hands are streaming - (plugin prints per-glove connect lines). +This page has moved to :doc:`/device/hand_tracking/wuji_glove`. diff --git a/docs/source/device/xr/index.rst b/docs/source/device/xr/index.rst new file mode 100644 index 0000000000..f4ddd6f18b --- /dev/null +++ b/docs/source/device/xr/index.rst @@ -0,0 +1,12 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +XR headsets +=========== + +XR headsets provide head poses and controller or hand input. + +The :doc:`/teleop/quick_start` covers setup and connection for Meta Quest, +PICO, and Apple Vision Pro. Quest and PICO use the WebXR client; Apple Vision +Pro uses the native sample client. See :doc:`/device/trackers` for the +tracking interfaces and :doc:`/device/body_tracking/index` for body tracking. diff --git a/docs/source/getting_started/build_from_source/index.rst b/docs/source/getting_started/build_from_source/index.rst index 08d4c0340a..589aaede4f 100644 --- a/docs/source/getting_started/build_from_source/index.rst +++ b/docs/source/getting_started/build_from_source/index.rst @@ -1,458 +1,9 @@ .. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 +:orphan: + Build from Source ================= -This page describes how to build Isaac Teleop from source, including core libraries, plugins, and -examples. The instructions align with the project's CMake configuration and the CI workflow -(:code-file:`build-ubuntu.yml <.github/workflows/build-ubuntu.yml>` in the GitHub repository). - -.. contents:: Steps - :local: - :depth: 1 - -.. admonition:: Next Steps - - - To build and serve the **WebXR client** locally, see :doc:`webxr`. - -Prerequisites -------------- - -- **CMake** 3.24 or higher (Ubuntu 22.04's apt ``cmake`` is 3.22 — install a newer one - from `Kitware's APT repository `_ or with ``pip install cmake``) -- **C++20** compatible compiler -- **Python** 3.11, 3.12, or 3.13 (default 3.11; see ``ISAAC_TELEOP_PYTHON_VERSION`` in root ``CMakeLists.txt``) -- **uv** for Python dependency management and managed Python -- **Internet connection** for downloading dependencies via CMake FetchContent - -.. note:: - **Optional — only needed to build the Televiz visualization module,** ``BUILD_VIZ``. - ``BUILD_VIZ`` is auto-detected from the GPU stack: it defaults to ``ON`` when both of the - following are found at configure time and to ``OFF`` otherwise, so a core-only source - build still configures on a machine without them. Watch the - ``-- BUILD_VIZ: (Vulkan=... CUDAToolkit=...)`` configure line to see which one - is missing. - - - **Vulkan headers + loader** — ``libvulkan-dev`` on Linux, the LunarG SDK on Windows. - - **CUDA Toolkit** (cudart at link time) — ``nvidia-cuda-toolkit`` or the official NVIDIA - installer. - - ``BUILD_VIZ=ON`` additionally requires **glslangValidator** to compile shaders to SPIR-V - (``glslang-tools`` on Linux, ``brew install glslang`` on macOS; ships with the Vulkan SDK - on Windows), and on Linux **wayland-scanner** (``libwayland-dev``) for GLFW's Wayland - backend — pass ``-DGLFW_BUILD_WAYLAND=OFF`` to build Televiz against X11 only instead. - Neither gates the auto-default: if one is missing, the configure fails naming it rather - than quietly dropping the module. Most users do not need any of this: - ``pip install isaaccapture`` already ships the compiled ``isaaccapture.viz`` module. See - `Other Build options`_ for the full option table. - -.. _one-time-setup: - -One time setup --------------- - -Install build tools and dependencies, such as CMake, clang-format. See :code-file:`build-ubuntu.yml <.github/workflows/build-ubuntu.yml>` in the GitHub repository for -the list of dependencies. On **Ubuntu**, install build tools and clang-format: - -.. code-block:: bash - - sudo apt-get update - sudo apt-get install -y build-essential cmake libegl-dev libx11-dev libwayland-dev clang-format-14 ccache patchelf pkg-config glslang-tools - -Runtime-only dependencies (needed to actually run teleop, not to build): - -.. code-block:: bash - - # adb — required for OOB teleop (``--setup-oob``) to talk to the headset over USB. - # coturn — required for USB-local mode (``--usb-local``); runs a local TURN server - # so WebRTC ICE can relay traffic from the headset to the CloudXR backend - # over the USB cable. - sudo apt-get install -y android-tools-adb coturn - -Our build system uses `uv`_ for Python version and dependency management. Install `uv`_ if not already installed: - -.. code-block:: bash - - curl -LsSf https://astral.sh/uv/install.sh | sh - -The installer drops ``uv`` in ``~/.local/bin``, which is not on ``PATH`` in most shells. Add it for -the current shell — and to your ``~/.bashrc`` to make it stick — or every ``uv`` command below -fails with ``uv: command not found``: - -.. code-block:: bash - - export PATH="$HOME/.local/bin:$PATH" - -.. note:: - While the build system uses `uv`_, the final Python packages can be installed via any Python package manager - such as `pip `_ or `conda `_. - -1. Clone the repository ------------------------ - -.. code-block:: bash - - git clone https://github.com/NVIDIA/IsaacTeleop.git - cd IsaacTeleop - -.. note:: - Dependencies (OpenXR SDK, pybind11, yaml-cpp) are automatically downloaded - during CMake configuration using FetchContent. No manual dependency installation or - git submodule initialization is required. - -Pre-download CloudXR SDK (Optional) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. note:: - - If you are using the default flow, skip this step. The - :code-file:`CMakeLists.txt ` - will automatically download the CloudXR SDK by calling the - :code-file:`download_cloudxr_runtime_sdk.sh ` - script. - -Sometimes NVIDIA might share early access CloudXR SDKs with you. In that case, you may get -tarballs such as: - -- ``CloudXR--Linux--sdk.tar.gz`` (CloudXR Runtime SDK) -- ``CloudXR-exp--Linux--sdk.tar.gz`` (experimental - runtime). Packaged by default as ``isaaccapture.cloudxr_exp``; needed for Jetson Orin - support (for example :doc:`/getting_started/televiz`) until the default runtime covers - those platforms. -- ``nvidia-cloudxr-.tgz`` (CloudXR Web SDK) - -You can place them in the :code-dir:`deps/cloudxr/` directory and update the ``deps/cloudxr/.env`` -file to locally override the default version defined in :code-file:`deps/cloudxr/.env.default`, -like this: - -.. code-block:: bash - - CXR_RUNTIME_SDK_VERSION= - CXR_WEB_SDK_VERSION= - -The experimental runtime is packaged into the wheel as ``isaaccapture.cloudxr_exp`` by default -(``ENABLE_CLOUDXR_EXP_BUNDLE=ON``). Pass ``-DENABLE_CLOUDXR_EXP_BUNDLE=OFF`` to skip it. -Select it at runtime with ``ISAAC_TELEOP_CLOUDXR_EXP``. -See :ref:`dedicated-cloudxr-runtime`. - -2. CMake: Configure and build ------------------------------ - -From the project root: - -.. code-block:: bash - - cmake -B build # configure - cmake --build build --parallel # build - cmake --install build # install - -Add any other options as ``-D`` flags on the configure line, for example -``cmake -B build -DCMAKE_BUILD_TYPE=Debug``. - -.. important:: - - The Python version is baked into a build directory's CMake cache and its build - venv, so ``ISAAC_TELEOP_PYTHON_VERSION`` cannot be changed on an existing tree — - configuring again with a different value fails with an explanatory error. Select - it on the first configure, and delete the tree to switch later: - - .. code-block:: bash - - cmake -B build -DISAAC_TELEOP_PYTHON_VERSION=3.12 - cmake --build build --parallel - -This will: - -1. Fetch dependencies (OpenXR SDK, yaml-cpp, pybind11, FlatBuffers, MCAP, and optionally Catch2 for tests) via FetchContent in ``deps/third_party/CMakeLists.txt`` -2. Build core C++ libraries (schema, oxr_utils, plugin_manager, oxr, pusherio, deviceio, mcap, etc.) and Python bindings -3. Build the Python wheel -4. Build examples (if enabled) -5. Install to ``./install`` (default prefix set in root ``CMakeLists.txt``) - - -C++ Formatting Enforcement (Linux) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -On Linux, **clang-format** is enforced by default; the build fails if formatting changes would be applied. The project -uses **clang-format-14** for consistent results across distributions (see ``cmake/ClangFormat.cmake``). - -To disable enforcement, set ``ENABLE_CLANG_FORMAT_CHECK`` to ``OFF``: - -.. code-block:: bash - - cmake -B build -DENABLE_CLANG_FORMAT_CHECK=OFF - -Useful targets: - -- ``clang_format_check`` — verifies formatting (part of ``ALL`` on Linux) -- ``clang_format_fix`` — applies formatting in place - -.. code-block:: bash - - cmake --build build --target clang_format_check - cmake --build build --target clang_format_fix - -Other Build options -~~~~~~~~~~~~~~~~~~~ - -The CMake options (defined in root :code-file:`CMakeLists.txt` and :code-file:`cmake/SetupPython.cmake`): - -.. list-table:: Common CMake Options - :widths: 20 36 44 - :header-rows: 1 - - * - Option - - CMake flag - - Default / Notes - * - **Build type** - - ``CMAKE_BUILD_TYPE`` - - ``Release`` or ``Debug`` - * - **Install prefix** - - ``CMAKE_INSTALL_PREFIX`` - - ``./install`` - -.. list-table:: Common Isaac Teleop Options - :widths: 24 36 40 - :header-rows: 1 - - * - Option - - CMake flag - - Default / Notes - * - **Examples** - - ``BUILD_EXAMPLES`` - - ``ON`` - * - **Python bindings** - - ``BUILD_PYTHON_BINDINGS`` - - ``ON`` - * - **Python version** - - ``ISAAC_TELEOP_PYTHON_VERSION`` - - ``3.11`` (3.11, 3.12, or 3.13) - * - **Testing** - - ``BUILD_TESTING`` - - ``ON``; enables CTest and Catch2 - * - **Clang-format check** - - ``ENABLE_CLANG_FORMAT_CHECK`` - - ``ON`` on Linux - * - **Televiz visualization** - - ``BUILD_VIZ`` - - Auto: ``ON`` when Vulkan and the CUDA Toolkit are detected, else ``OFF``. Force with ``-DBUILD_VIZ=ON`` / ``-DBUILD_VIZ=OFF``. (Most users don't need this — ``pip install isaaccapture`` already ships the compiled ``isaaccapture.viz`` module.) - -.. list-table:: Plugin Specific Options - :widths: 26 34 40 - :header-rows: 1 - - * - Option - - CMake flag - - Default / Notes - * - **Plugins master switch** - - ``BUILD_PLUGINS`` - - ``ON`` - * - **OAK camera plugin** - - ``BUILD_PLUGIN_OAK_CAMERA`` - - ``OFF``; when ``ON``, builds DepthAI v3.x and pulls its dependencies through - vcpkg, so it also needs ``CMAKE_TOOLCHAIN_FILE`` on a fresh build directory. - See :doc:`/device/oak`. - * - **Teleop ROS2 example only** - - ``BUILD_EXAMPLE_TELEOP_ROS2`` - - ``OFF``; when ``ON``, only ``examples/teleop_ros2`` (e.g. Docker) - -Examples -~~~~~~~~ - -Build for a different Python version (``3.11``, ``3.12``, ``3.13`` are supported; -delete ``build/`` first if it is already configured for another one): - -.. code-block:: bash - - cmake -B build -DISAAC_TELEOP_PYTHON_VERSION=3.12 - cmake --build build --parallel - -Debug build: - -.. code-block:: bash - - cmake -B build -DCMAKE_BUILD_TYPE=Debug - cmake --build build - -Build without examples: - -.. code-block:: bash - - cmake -B build -DBUILD_EXAMPLES=OFF - cmake --build build - -Build without Python bindings: - -.. code-block:: bash - - cmake -B build -DBUILD_PYTHON_BINDINGS=OFF - cmake --build build - -Build with the OAK camera plugin. It needs the vcpkg toolchain, and CMake only -reads ``CMAKE_TOOLCHAIN_FILE`` on a build tree's **first** configure, so delete -``build/`` first if it already exists (see :doc:`/device/oak`): - -.. code-block:: bash - - rm -rf build - cmake -B build -DBUILD_PLUGIN_OAK_CAMERA=ON \ - -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake - cmake --build build --target camera_plugin_oak --parallel - -Build only the teleop_ros2 example (e.g. for Docker, as in :code-file:`build-ubuntu.yml <.github/workflows/build-ubuntu.yml>` teleop-ros2-docker job): - -.. code-block:: bash - - cmake -B build -DBUILD_EXAMPLES=OFF -DBUILD_EXAMPLE_TELEOP_ROS2=ON - cmake --build build - -Clean rebuild (``--fresh`` wipes the CMake cache and reconfigures): - -.. code-block:: bash - - cmake -B build --fresh - cmake --build build - -3. Running tests ----------------- - -When ``BUILD_TESTING`` is ``ON``, CTest is enabled at the top level. Run all tests either via the CMake ``test`` target or with ``ctest``: - -.. code-block:: bash - - cmake --build build --target test - - # Or with ctest (e.g. parallel, output on failure) - ctest --test-dir build --output-on-failure --parallel - -The CI uses ``ctest`` (see :code-file:`build-ubuntu.yml <.github/workflows/build-ubuntu.yml>`). - -4. Install the ``isaaccapture`` pip package -------------------------------------------- - -The wheels are built in the ``./install/wheels/`` directory. Install the package from the wheels. -Using ``pip``, you need to pass the ``--no-index`` option to automatically find the right wheel -based on the Python version. Note that ``pip`` and ``uv pip`` has slightly different options. - -.. code-block:: bash - - # Pass --no-index to use only wheels in ./install/wheels/; - # Pass --force-reinstall to replace an existing install. - pip install "isaaccapture[retargeters,cloudxr,ui]" --find-links=./install/wheels/ --no-index --force-reinstall - -With ``uv pip``, ``--find-links`` still consults the package index and installs the highest -version it finds, so a newer published ``isaaccapture`` there silently wins over your local build. -``--no-index`` is not a fix either: the wheel's dependencies are not in ``./install/wheels/``, so -the install fails. Install the local wheel by path instead. That always selects your build, while -the index stays available for its dependencies: - -.. code-block:: bash - - # Pick the wheel for your Python version (cp312 = Python 3.12; use cp311 or cp313 otherwise). - # -t sorts by modification time so head -1 selects the most recent build if older wheels remain. - WHEEL=$(ls -t ./install/wheels/isaaccapture-*-cp312-*.whl | head -1) - - # --reinstall-package replaces an already-installed isaaccapture, e.g. after a rebuild. - # --find-links exposes the sibling isaacteleop transition wheel built alongside isaaccapture. - uv pip install "${WHEEL}[retargeters,cloudxr,ui]" --find-links=./install/wheels/ --reinstall-package isaaccapture - -Check that the installed version ends in ``+local`` with ``uv pip list | grep isaaccapture``. - -The wheel carries the ``isaacteleop`` compatibility alias, which will be removed -in 1.9, so -``import isaacteleop`` resolves without installing anything else. See -:doc:`/references/migration`. - -Alternative: install directly from source with pip -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The repository also ships a root :code-file:`pyproject.toml` using the -`scikit-build-core `_ backend, which -drives the same top-level ``CMakeLists.txt``. This lets you build and install -with standard Python tooling, without running the ``cmake -B build`` steps -yourself; the classic flow above (which CI uses to produce ``install/wheels/`` and -the released wheels) is unchanged, so the two coexist. - -.. code-block:: bash - - pip install . # build + install the compiled wheel from source - pip install -e . # editable / developer install - -.. note:: - - The CMake build tree is kept under ``build-wheel//`` (e.g. - ``build-wheel/cpython-311/``) — one per Python version, so different - interpreters never share a configured CMake cache — instead of a temporary - directory. Re-installs are therefore incremental. It sits outside ``build/``, so - it never collides with a classic ``cmake -B build`` tree. Both are gitignored. - -What this path does and how it differs from the classic flow: - -- **Full build, no overrides.** The backend passes no ``cmake.define`` overrides, - so ``BUILD_EXAMPLES``, ``BUILD_TESTING``, and ``BUILD_PLUGINS`` stay at their - CMake defaults: a ``pip install`` builds the full default target — extensions, - examples, C++ tests, and plugins (a plugin without its SDK skips gracefully). It - installs only the ``isaacteleop_wheel`` and ``isaacteleop_binaries`` components, - so C++ library/header install rules do not leak into the wheel. -- **clang-format is enforced.** Because the gate is left on, the build **fails** if - ``clang-format-14`` is not installed or any C++ file is unformatted — a - ``pip install`` therefore requires ``clang-format-14`` on Linux. -- **All executables on PATH.** Every executable built from ``src/`` and - ``examples/`` — example programs, C++ test binaries, plugin tools - (``oxr_simple_api_demo``, ``se3_printer``, ``schema_tests``, …) — is installed - into the venv's ``bin/`` (the wheel's scripts scheme; see the executable-install - loop in the root ``CMakeLists.txt``), so they are on ``PATH`` once the venv is - active. They statically link the teleop core libraries, so they are - self-contained. Shared libraries (Python extension modules, plugin ``.so``) are - not executables and are **not** placed in ``bin/``. -- **ABI.** Extensions are compiled against the interpreter that runs the build - (so the wheel's ABI tag matches) and against NumPy 2.x, so a single wheel works - with both NumPy 1.x and 2.x at runtime. -- **No type stubs.** The pip-built wheel omits the ``.pyi`` stubs that the classic - and released wheels ship (stub generation shells out to ``uv``, which is not - guaranteed inside pip's build isolation). Imports and runtime behavior are - unaffected. -- **Version.** The pip-facing version is derived from the ``VERSION`` file as - ``MAJOR.MINOR+local`` — the same value the classic flow produces for a - non-CI/local build. The pip path is a from-source/dev build, so it is always - tagged ``+local``; the git-aware release versioning (tags, ``a1`` / ``rc1`` / - ``.devN`` labels) stays with the classic flow via - :code-file:`cmake/IsaacTeleopVersion.cmake`. - -.. admonition:: ``import isaacteleop`` under this path - - Both ``pip install .`` and ``pip install -e .`` serve the alias: the shim's - source (:code-file:`src/compat/isaacteleop.py`) is mapped beside ``isaaccapture``'s - and ships in the same wheel (:doc:`/references/migration`). - -.. admonition:: Editable installs and iterating on pure-Python subpackages - - An editable install (``pip install -e .``) never recompiles on import. Pure-Python - subpackages resolve straight to ``src/python/isaaccapture/``, so edits **take effect - live** — a fresh interpreter is enough. - - Compiled extensions are the exception: ``.so``/``.pyd`` still come from the CMake - build tree, so C++ changes need a ``cmake --build`` (or a re-run of - ``pip install -e .``) first. - -See :doc:`/references/build` for the full build-system reference. - -.. admonition:: ``retargeters`` extra on aarch64 (Jetson / DGX) - - The full ``retargeters`` extra does **not** resolve on ``aarch64`` (some of its - pinned dependencies have no aarch64 wheels). On Jetson/DGX-class robotics - targets, install the ``retargeters-lite`` extra instead: - - .. code-block:: bash - - pip install "isaaccapture[retargeters-lite]" - -.. toctree:: - :hidden: - - webxr - -.. - References -.. _`uv`: https://docs.astral.sh/uv/ +This page has moved to :doc:`/development/build_from_source/index`. diff --git a/docs/source/getting_started/build_from_source/webxr.rst b/docs/source/getting_started/build_from_source/webxr.rst index 184fbf062f..c08e1e7a67 100644 --- a/docs/source/getting_started/build_from_source/webxr.rst +++ b/docs/source/getting_started/build_from_source/webxr.rst @@ -1,101 +1,9 @@ .. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 +:orphan: + Build WebXR Client ================== -.. tip:: - - A prebuilt CloudXR web client is available at |web_client_url|. - You only need to build from source if you want to customize the client. - -The Isaac Teleop WebXR client is built with React, Three.js, and WebXR. It streams VR/AR content -from a CloudXR Runtime server to the browser. For more details about CloudXR.js SDK and its advanced features, see the -`CloudXR.js documentation`_. - -The source is located at :code-dir:`deps/cloudxr/webxr_client/`. - -.. contents:: Steps - :local: - :depth: 1 - -Prerequisites -------------- - -- Node.js (v20 or higher). Ubuntu's ``nodejs`` apt package is older (18.x on Ubuntu 24.04), so - install a current release from `Node.js downloads `_ (for - example with nvm) and check it with ``node --version``. ``npm`` ships with it. - -1. Download CloudXR.js SDK --------------------------- - -From the project root: - -.. code-block:: bash - - source scripts/setup_cloudxr_env.sh - scripts/download_cloudxr_sdk.sh - -This will automatically download the CloudXR.js SDK and place it in ``deps/cloudxr/``. The -`package.json` is configured to install the SDK from the downloaded local package. - -2. Install dependencies ------------------------ - -From the ``deps/cloudxr/webxr_client/`` directory: - -.. code-block:: bash - - npm install - -3. Build & Run --------------- - -Production build -~~~~~~~~~~~~~~~~ - -From ``deps/cloudxr/webxr_client/``: - -.. code-block:: bash - - npm run build - -Output in ``build/``: - -- ``index.html`` — entry page -- ``bundle.js`` — main application (UIKit, MSDF text, icons; loaded on first paint) -- ``bundle.emulator.js`` — desktop XR / IWER stack (DevUI, synthetic environments, - and related deps); loaded only when emulation is used - - -Development build (one-shot) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Outputs to ``./build/`` with source maps for debugging: - -.. code-block:: bash - - npm run dev - -Development server (with hot reload) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Starts a webpack dev server on ``http://localhost:8080`` with hot module replacement: - -.. code-block:: bash - - npm run dev-server - -For **HTTPS** (required by WebXR on most browsers): - -.. code-block:: bash - - npm run dev-server:https - -This starts the dev server with a self-signed certificate on ``https://localhost:8080``. You can follow the same -instructions to use the WebXR client as documented in the :ref:`Quick Start Guide `. - - -.. - References -.. _`CloudXR.js documentation`: https://docs.nvidia.com/cloudxr-sdk/latest/usr_guide/cloudxr_js/index.html +This page has moved to :doc:`/development/build_from_source/webxr`. diff --git a/docs/source/getting_started/contributing.rst b/docs/source/getting_started/contributing.rst index 796cf8e7e5..a4b8b4a638 100644 --- a/docs/source/getting_started/contributing.rst +++ b/docs/source/getting_started/contributing.rst @@ -1,77 +1,9 @@ .. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 +:orphan: + Contributing ============ -We welcome contributions. Please see the repository's `CONTRIBUTING.md `_ for: - -- Code of conduct and how to contribute -- Development setup and coding standards -- Pull request process - -Previewing documentation changes --------------------------------- - -Local build -~~~~~~~~~~~ - -Build the docs locally to catch broken links and rendering issues before opening -a pull request: - -.. code-block:: bash - - cd docs - pip install -r requirements.txt - make current-docs - -The output is written to ``docs/build/current/``. Open ``index.html`` in a -browser to inspect it. Sphinx is run with ``-W --keep-going``, so warnings are -treated as errors — fix them locally before pushing. - -PR preview on GitHub Pages -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Every PR preview is published to a single canonical location: - -.. code-block:: text - - https://nvidia.github.io/IsaacCapture/preview/pr-/ - -How the preview gets built depends on where the PR's branch lives. - -PRs from a branch on ``NVIDIA/IsaacTeleop`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The ``Build & deploy docs`` workflow runs automatically on every push to the -PR branch and publishes the preview. The preview URL is added to the workflow -run's *Summary* tab and refreshed on every push. No extra action required. - -PRs from a fork -^^^^^^^^^^^^^^^ - -GitHub Actions on PRs from forks run with a read-only token, so the workflow -cannot push to ``gh-pages`` automatically. Instead: - -1. ``Build & deploy docs`` still runs and uploads the built artifacts; only - the deploy step is skipped. -2. When a fork PR is opened, a bot comments with instructions. -3. A maintainer (anyone with write access to ``NVIDIA/IsaacTeleop``) deploys - the preview by commenting on the PR: - - .. code-block:: text - - /preview-docs - -4. The maintainer-triggered workflow downloads the artifacts that - ``Build & deploy docs`` already produced for that commit and pushes them - to ``preview/pr-/``, reacting to the comment with 👀 while deploying - and 👍 once published. A follow-up comment posts the preview URL. - -Because the deploy reuses the same artifacts that the build job validated, -``/preview-docs`` does not run any PR code itself. Re-comment -``/preview-docs`` after new commits land to redeploy from the latest build. - -Previews are not auto-cleaned; maintainers can run the -``Cleanup docs PR previews`` workflow from the *Actions* tab to clear the -``preview/`` tree. +This page has moved to :doc:`/development/contributing`. diff --git a/docs/source/getting_started/lerobot/data_collection_real.rst b/docs/source/getting_started/lerobot/data_collection_real.rst index da8360cd1b..20f6a782ab 100644 --- a/docs/source/getting_started/lerobot/data_collection_real.rst +++ b/docs/source/getting_started/lerobot/data_collection_real.rst @@ -1,244 +1,9 @@ .. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 +:orphan: + Data Collection in Real ======================= -Record demonstrations on a physical SO-101 into a `LeRobot dataset -`_, driving the follower with either teleop device -(see :doc:`devices`). The example scripts live in ``examples/isaac_teleop_to_so101/`` in the -`LeRobot `_ repository — they ship there only, and are -part of neither the ``lerobot`` pip package nor the Isaac Teleop repository, so everything on this -page runs from a LeRobot source checkout. ``teleoperate.py`` drives the arm live, and -``record.py`` does the same while saving a dataset. Both take the same -``--robot.*`` / ``--teleop.*`` flags; ``--teleop.type`` selects the device -(``xr_controller`` | ``so101_leader``). - -Before you start ----------------- - -Follow the necessary one-time steps to set up your environment and hardware: - -#. A **LeRobot source checkout**, since that is where the example scripts live. Clone it and - change into its root — the installs below and every later command run from there, except where - a step is explicitly scoped to another checkout (the ``so101_leader`` plugin is built from an - Isaac Teleop checkout): - - .. code-block:: bash - - git clone https://github.com/huggingface/lerobot.git - cd lerobot - -#. A working **SO-101 follower** — assembled, motors set up, and calibrated. See - `SO-101 support in LeRobot`_. - -#. The example dependencies, installed into an environment created at that checkout root. The - LeRobot extras cover the SO-101 motor bus (``feetech``), the IK solver for the XR path - (``kinematics``), and dataset recording (``dataset``). For Isaac Teleop, ``cloudxr`` brings the - CloudXR runtime bindings and ``retargeters-lite`` is the default retargeter path on both - x86_64 and aarch64; the full ``retargeters`` extra is optional. The ``isaaccapture`` pin follows - the release series this page documents — see :ref:`install-isaaccapture-pip-package` for the - other install options: - - .. parsed-literal:: - - uv venv --python 3.12 .venv - source .venv/bin/activate - uv pip install -e ".[feetech,kinematics,dataset]" "huggingface_hub>=1.5" - uv pip install "isaaccapture[cloudxr,retargeters-lite]\ |pip_version_pin|\ " "scipy>=1.14" \\ - --extra-index-url https://pypi.nvidia.com --prerelease=allow - - The upstream ``isaac_teleop_to_so101`` scripts still import ``isaacteleop``. - Installing ``isaaccapture`` includes that compatibility alias automatically - -- see :doc:`/references/migration`. - -#. Log in to the Hugging Face Hub — recorded datasets are pushed to the Hub by default (pass - ``--dataset.push_to_hub=false`` to keep them local): - - .. code-block:: bash - - hf auth login - -#. Accept the CloudXR EULA once and start the runtime. The EULA prompts on stdin, which would hang - a headless run, so do both ahead of time; the service outlives each script you run below: - - .. code-block:: bash - - python -m isaaccapture.cloudxr.service start --accept-eula - -Teleop and data recording -------------------------- - -Run the scripts as modules from the **LeRobot repository root**, with the environment from -`Before you start`_ activated (they use relative imports, so ``python -m`` is required). - -.. note:: - - The ``examples`` package resolves only from that root. Running these commands from an Isaac - Teleop checkout or any other directory fails with:: - - ModuleNotFoundError: No module named 'examples.isaac_teleop_to_so101' - - If you hit this, ``cd`` into your LeRobot clone and re-run. - -Then follow the steps for your teleop device: - -.. tab-set:: - - .. tab-item:: XR controller - - The controller pose drives the follower's end-effector through the clutch + IK pipeline, - streamed over CloudXR. - - #. **Connect a headset.** On ``connect()`` the script auto-launches the CloudXR runtime - (~30 s) — you do **not** need a separate terminal or to source ``cloudxr.env``. Set - ``LEROBOT_CLOUDXR_SKIP_AUTOLAUNCH=1`` to opt out when running CloudXR yourself. For headset - pairing and firewall setup, follow the :doc:`/getting_started/quick_start`. - - .. note:: - - The XR path solves inverse kinematics, so it needs the SO-101 URDF and meshes. These are - fetched automatically from the ``lerobot/robot-urdfs`` Hugging Face bucket into the - LeRobot cache on first run — no manual download step. - - #. **(Optional) Try teleoperation without recording.** A good way to check the setup before - committing to a dataset: - - .. code-block:: bash - - python -m examples.isaac_teleop_to_so101.teleoperate \ - --robot.type=so101_follower \ - --robot.port=/dev/ttyACM0 \ - --robot.id=so101_follower_arm \ - --teleop.type=xr_controller - - Squeeze and hold the grip to engage the clutch and move the arm; the trigger controls the - gripper. Release the grip to pause. - - #. **Record a dataset.** Add cameras and the dataset parameters: - - .. code-block:: bash - - python -m examples.isaac_teleop_to_so101.record \ - --robot.type=so101_follower \ - --robot.port=/dev/ttyACM0 \ - --robot.id=so101_follower_arm \ - --teleop.type=xr_controller \ - --robot.cameras="{ front: {type: opencv, index_or_path: 0, width: 640, height: 480, fps: 30}}" \ - --dataset.repo_id=$(hf auth whoami --format json | jq -r '.user')/my_test_dataset \ - --dataset.single_task="Pick up vial from rack on the left side" \ - --dataset.num_episodes=3 \ - --dataset.episode_time_s=20 \ - --dataset.reset_time_s=5 - - .. note:: - - **Customizing the reset pose.** On startup the XR path slews the arm to a built-in default - reset pose (a comfortable mid-range pose) before handing control to the clutch — you do - **not** need to record anything. To tailor it to your setup, back-drive the arm to the - pose you want and run: - - .. code-block:: bash - - python -m examples.isaac_teleop_to_so101.override_reset_pose \ - --port /dev/ttyACM0 \ - --id so101_follower_arm - - This writes ``$HF_LEROBOT_HOME/reset_poses//.json`` (```` - is the follower type — ``so_follower`` for SO-100/SO-101 arms — and ```` is the - same arm identifier you pass as ``--robot.id``, given here as ``--id``). The pose is keyed - per arm by ``--robot.id``, so later runs with the same ``--robot.id`` pick it up - automatically and slew to it instead of the default. Pass ``--reset_to_origin=false`` to - skip the slew and keep the arm where it is. - - .. tab-item:: SO-101 Leader - - A back-drivable SO-101 leader arm mirrored 1:1 to the follower. Its joints are streamed by - Isaac Teleop's ``so101_leader`` plugin, which the script launches for you. - - #. **Build the so101_leader plugin.** It is part of Isaac Teleop's C++ source, not the - ``isaaccapture`` pip package, so build it from an Isaac Teleop checkout: - - .. code-block:: bash - - cmake -B build && cmake --build build --parallel && cmake --install build - - The binary lands at ``install/plugins/so101_leader/so101_leader_plugin``. For details see - :ref:`so101-leader-plugin` and :doc:`/getting_started/build_from_source/index`. - - #. **Calibrate the leader** so the leader and follower agree on each joint's zero and range. - This reuses the serial SO-101 leader's calibration (stored under ``so_leader/.json`` - and reused on every run): - - .. code-block:: bash - - lerobot-calibrate \ - --teleop.type=so101_leader \ - --teleop.port=/dev/ttyACM1 \ - --teleop.id=so101_leader_arm - - #. **(Optional) Try teleoperation without recording.** ``--launch_plugin`` spawns the plugin - after CloudXR is up; ``--teleop.port`` is the leader's serial port: - - .. code-block:: bash - - python -m examples.isaac_teleop_to_so101.teleoperate \ - --robot.type=so101_follower \ - --robot.port=/dev/ttyACM0 \ - --robot.id=so101_follower_arm \ - --teleop.type=so101_leader \ - --teleop.port=/dev/ttyACM1 \ - --teleop.id=so101_leader_arm \ - --launch_plugin=/path/to/IsaacTeleop/install/plugins/so101_leader/so101_leader_plugin - - Back-drive the leader arm by hand to move the follower. - - #. **Record a dataset.** Same flags as teleoperation, plus the cameras and dataset parameters - (keep ``--launch_plugin`` so the plugin is started): - - .. code-block:: bash - - python -m examples.isaac_teleop_to_so101.record \ - --robot.type=so101_follower \ - --robot.port=/dev/ttyACM0 \ - --robot.id=so101_follower_arm \ - --teleop.type=so101_leader \ - --teleop.port=/dev/ttyACM1 \ - --teleop.id=so101_leader_arm \ - --launch_plugin=/path/to/IsaacTeleop/install/plugins/so101_leader/so101_leader_plugin \ - --robot.cameras="{ front: {type: opencv, index_or_path: 0, width: 640, height: 480, fps: 30}}" \ - --dataset.repo_id=$(hf auth whoami --format json | jq -r '.user')/my_test_dataset \ - --dataset.single_task="Pick up vial from rack on the left side" \ - --dataset.num_episodes=3 \ - --dataset.episode_time_s=20 \ - --dataset.reset_time_s=5 - -Recording controls ------------------- - -``record.py`` records ``--dataset.num_episodes`` episodes of ``--dataset.episode_time_s`` seconds -each, with a ``--dataset.reset_time_s`` window between episodes to reposition the scene. While -it is running, **press** these keys in the terminal where ``record.py`` is running — the example -reads them from that terminal, so they work over SSH and in a plain terminal (Linux/macOS), with -no desktop session required: - -.. list-table:: - :header-rows: 1 - :widths: 22 78 - - * - Key - - Action - * - Right arrow → (or ``n``) - - End the current episode early and save it. - * - Left arrow ← (or ``r``) - - Discard the current take and re-record it. - * - Escape (or ``q``) - - Stop after the current episode (already-saved episodes are kept). - -Keys are read from the terminal when stdin is a TTY (so they work over SSH); with no TTY the -example falls back to LeRobot's default keyboard listener. No configuration is needed. -The dataset is written under ``$HF_LEROBOT_HOME/`` and pushed to the Hub when recording -finishes (unless ``--dataset.push_to_hub=false``). Next, train a policy on it: -:doc:`training_groot`. - -.. _SO-101 support in LeRobot: https://huggingface.co/docs/lerobot/en/so101 +This page has moved to :doc:`/integrations/lerobot/data_collection_real`. diff --git a/docs/source/getting_started/lerobot/data_collection_sim.rst b/docs/source/getting_started/lerobot/data_collection_sim.rst index 42b3b7ea7a..b3a1d3bbca 100644 --- a/docs/source/getting_started/lerobot/data_collection_sim.rst +++ b/docs/source/getting_started/lerobot/data_collection_sim.rst @@ -1,303 +1,9 @@ .. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. SPDX-License-Identifier: Apache-2.0 +:orphan: + Data Collection in Sim ====================== -Collect SO-101 demonstrations in simulation with `NVIDIA Isaac Lab -`_, on the cube-stacking task. You drive the simulated -follower through Isaac Teleop (see :doc:`devices`) and record episodes to an HDF5 dataset. - -Three SO-101 stack tasks are registered in Isaac Lab — pick the one that matches your teleop -device: - -.. list-table:: - :header-rows: 1 - :widths: 45 55 - - * - Task id - - Use - * - ``IsaacContrib-Stack-Cube-SO101-IK-Abs-v0`` - - Absolute-pose IK + Isaac Teleop teleoperation, driven by an **XR controller**. - * - ``IsaacContrib-Stack-Cube-SO101-Joint-Teleop-v0`` - - Joint-space mirror, driven by a physical **SO-101 leader arm** (no headset or IK). - * - ``IsaacContrib-Stack-Cube-SO101-v0`` - - Joint-position control baseline (no teleop). - -Before you start ----------------- - -.. important:: - - The steps below are **required** — complete them first. The teleoperation and recording - commands later on will not work until you have. - -**Step 1 — Install Isaac Lab.** Follow the `Isaac Lab installation guide`_ to set up the ``Lab`` -repository, then run every script through its launcher: ``./isaaclab.sh -p