diff --git a/docs/source/_data/devices.yaml b/docs/source/_data/devices.yaml index 3c9485de6c..b057eb5838 100644 --- a/docs/source/_data/devices.yaml +++ b/docs/source/_data/devices.yaml @@ -317,6 +317,22 @@ devices: - label: 3dconnexion.com url: https://3dconnexion.com/ + - id: gamepad + name: Gamepads + url: https://github.com/mdqinc/SDL_GameControllerDB + group: peripheral + modes: 6-DoF end effector, base velocity + since: main + details: + setup: + - label: Gamepad Plugin + url: https://github.com/NVIDIA/IsaacCapture/tree/main/src/plugins/gamepad + - label: Gamepad docs + doc: /device/gamepad + requirements: + - Linux workstation with the pad on USB or Bluetooth + - Xbox, PlayStation, Switch Pro, Steam or any pad SDL_GameControllerDB maps + - id: oak-d-camera name: OAK-D Camera url: https://shop.luxonis.com/products/oak-d diff --git a/docs/source/device/gamepad.rst b/docs/source/device/gamepad.rst new file mode 100644 index 0000000000..e3a1a1c945 --- /dev/null +++ b/docs/source/device/gamepad.rst @@ -0,0 +1,196 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Gamepad Plugin +============== + +C++ plugin that streams a **gamepad** (USB or Bluetooth) into the standard DeviceIO path in one +layout for every pad: buttons named by position and the two sticks and triggers. It reads the pad +through `SDL3 `_, whose drivers and +`SDL_GameControllerDB `_ mappings resolve each pad's +own numbering, and pushes the state at 90 Hz. Source and plugin README: +:code-file:`src/plugins/gamepad/README.md`. + +Supported pads: Xbox (360, One, Series, USB and Bluetooth), PlayStation (DualShock 4, DualSense), +Nintendo Switch Pro, Steam Controller, and any pad with an SDL_GameControllerDB mapping. Validated: +Xbox 360 Controller. + +This plugin follows the :doc:`add_device` pattern (FlatBuffer schema → plugin ``SchemaPusher`` → +tracker → MCAP). + +.. contents:: On this page + :local: + :depth: 2 + +Components +---------- + +- **Schema** — :code-file:`src/core/schema/fbs/gamepad.fbs`: ``GamepadInputState``, the standard + layout as named fields, inside ``GamepadSnapshot`` / ``GamepadSnapshotRecord``. +- **Plugin** — :code-dir:`src/plugins/gamepad`: ``core/`` reads the pad through SDL3 and reopens it + after an unplug; ``app/`` is the OpenXR push loop. +- **Tracker** — ``GamepadTracker``, generated at configure time from the ``gamepad`` entry in + :code-file:`src/core/deviceio_trackers/trackers.toml`. +- **Source node** — ``GamepadSource`` turns the tracker's data into a ``gamepad`` tensor group + (``GamepadInput``, indexed by ``GamepadInputIndex``) for retargeters. + +Standard layout +--------------- + +``GamepadInputState`` is a generic controller split in two, with the same naming as +``ControllerInputState``. Face buttons are named by position, so a binding means the same on every +pad: + +.. list-table:: + :header-rows: 1 + :widths: 25 25 25 25 + + * - Field + - Xbox + - PlayStation + - Nintendo + * - ``south_click`` + - A + - Cross + - B + * - ``east_click`` + - B + - Circle + - A + * - ``west_click`` + - X + - Square + - Y + * - ``north_click`` + - Y + - Triangle + - X + * - ``view_click`` / ``menu_click`` + - View / Menu + - Create / Options + - − / + + +The other buttons are ``shoulder_[left|right]_click``, ``thumbstick_[left|right]_click`` and +``dpad_[up|down|left|right]_click``; the axes are ``thumbstick_[left|right]_[x|y]`` in ``[-1, 1]`` +(right and up positive, as in OpenXR) and ``trigger_[left|right]_value`` in ``[0, 1]``. These are the +inputs every standard pad has, the same set as OpenXR's ``xbox_controller`` profile; pad-specific +extras such as the guide button, paddles and touchpad are not reported. + +Device access +------------- + +On a desktop session udev grants the logged-in user access to gamepad nodes, so no setup is +needed. Without a local seat (SSH, a service, a container), add the user to the ``input`` group, +and pass the pad's ``/dev/input/event*`` node into a container. + +Build +----- + +Linux only. The plugin builds with the default plugin set and fetches SDL3 (zlib license) at +configure time, linking it statically with only its joystick and HIDAPI subsystems: + +.. code-block:: bash + + cmake -B build + cmake --build build --target gamepad_plugin --parallel + +Usage +----- + +Start the plugin with the session through ``TeleopSessionConfig.plugins``, from the copy bundled +with ``isaaccapture`` (wheel installs included), and read it with ``GamepadSource``, as +:code-file:`examples/teleop/python/gamepad_printer_example.py` does: + +.. code-block:: python + + from isaaccapture.retargeting_engine.deviceio_source_nodes import GamepadSource + from isaaccapture.teleop_session_manager import ( + BUNDLED_PLUGINS_DIR, + PluginConfig, + TeleopSessionConfig, + ) + + config = TeleopSessionConfig( + app_name="MyApp", + pipeline=pipeline_reading(GamepadSource(name="gamepad")), + plugins=[ + PluginConfig( + plugin_name="gamepad", + plugin_root_id="gamepad", + search_paths=[BUNDLED_PLUGINS_DIR], + ) + ], + ) + +The plugin reads the first connected pad, keeps it until it disconnects, and then takes the first +one again. Its name, type and USB IDs are logged when it is opened. + +The plugin can also run on its own, for example on another machine pushing to the same runtime, +next to any reader of the same collection: + +.. code-block:: bash + + ./install/plugins/gamepad/gamepad_plugin + +Multiple pads +~~~~~~~~~~~~~ + +Run one plugin per pad. ``--player=N`` reads the Nth connected pad, and each plugin pushes to its +own collection, read by a ``GamepadSource`` with the same ``collection_id``: + +.. code-block:: python + + plugins = [ + PluginConfig( + plugin_name="gamepad", + plugin_root_id=f"gamepad_{n}", + search_paths=[BUNDLED_PLUGINS_DIR], + plugin_args=[f"--player={n}", f"--collection-id=gamepad_{n}"], + ) + for n in (1, 2) + ] + sources = [GamepadSource(name=f"gamepad_{n}", collection_id=f"gamepad_{n}") for n in (1, 2)] + +Published data +-------------- + +The plugin pushes ``GamepadSnapshot`` to collection ``gamepad`` (``--collection-id`` changes it; +match it in ``GamepadSource`` / ``GamepadTracker``): + +- ``inputs`` — the ``GamepadInputState`` above. + +While no pad is open the plugin keeps pushing snapshots without ``inputs``, so a +reader never holds the last stick deflection of an unplugged pad; ``GamepadSource`` then reports +its outputs as absent. Recordings use MCAP channels ``gamepad`` and ``gamepad_tracked``. + +Retargeters +----------- + +``GamepadToSe3RelRetargeter`` (end-effector delta from the sticks and D-pad), +``GamepadToSe2Retargeter`` (base velocity from the sticks) and ``GamepadGripperRetargeter`` +(toggle on ``WEST``) consume ``GamepadSource``. Their dead zone defaults to 0.15: an idle pad's +sticks rest up to 0.14 off center. + +Device status +------------- + +When ``TeleopSession`` starts the plugin, it reports device ``/gamepad`` +(``gamepad/gamepad`` by default) through :doc:`../references/device_provider_monitoring`: +``CONNECTED``, ``DISCONNECTED`` when no pad is connected for its player, or ``FAILED`` when SDL +finds the pad but cannot open it. + +Troubleshooting +--------------- + +- **Status DISCONNECTED with the pad connected** — SDL lists a device as a gamepad only when it + has a mapping. Find the pad's line in SDL_GameControllerDB (or write one) and set it in the + ``SDL_GAMECONTROLLERCONFIG`` environment variable of the process that starts the plugin. +- **Status DISCONNECTED over SSH or in a container** — the user cannot read the pad's + ``/dev/input/event*`` node; see `Device access`_. +- **The robot drifts with the pad at rest** — raise the retargeter's ``dead_zone``. + +Tests +----- + +``gamepad_plugin_tests`` reads SDL virtual gamepads: the standard layout, stick and trigger +ranges, a pad plugged in late, an unplug, and player selection. diff --git a/docs/source/index.rst b/docs/source/index.rst index e817825e86..94ab5e5a25 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -73,6 +73,7 @@ Table of Contents device/sharpa_avatar device/haptikos device/spacemouse + device/gamepad .. toctree:: :maxdepth: 2 diff --git a/docs/source/references/generated_trackers.rst b/docs/source/references/generated_trackers.rst index 0ccd8e0249..a2d13bec6a 100644 --- a/docs/source/references/generated_trackers.rst +++ b/docs/source/references/generated_trackers.rst @@ -116,6 +116,9 @@ Generated today * - ``spacemouse`` - ``SpaceMouseTracker`` - Overrides ``class`` and ``traits``; the defaults would spell ``Spacemouse`` + * - ``gamepad`` + - ``GamepadTracker`` + - Gamepad state pushed by the gamepad plugin (SDL3); only ``python_accessor`` is overridden Still hand-written ------------------ diff --git a/examples/teleop/python/gamepad_printer_example.py b/examples/teleop/python/gamepad_printer_example.py new file mode 100644 index 0000000000..5842cbf3a2 --- /dev/null +++ b/examples/teleop/python/gamepad_printer_example.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Gamepad Printer Example. + +Prints the gamepad's sticks, triggers and held buttons each frame, via GamepadSource's "gamepad" +output, and what the gamepad retargeters make of them: the end-effector delta +(GamepadToSe3RelRetargeter), the base velocity (GamepadToSe2Retargeter) and the gripper command +(GamepadGripperRetargeter, toggled by the west face button: X on Xbox, Square on PlayStation). +Buttons print by their field name in the standard layout, so the names are the same on every pad. + +Like a downstream app driving a robot, it also integrates the end-effector delta into an absolute +pose and the base velocity into a base pose, treating both as rates per second. Menu resets them. +Stick drift above --dead-zone shows up as a pose that creeps with the pad at rest. + +The session starts the gamepad plugin bundled with isaaccapture; nothing else needs to run. Like +every TeleopSession it opens an OpenXR session; the CloudXR launcher arguments start a runtime, and +no headset needs to connect. GamepadToSe3RelRetargeter needs scipy (the retargeters-lite extra). +""" + +import argparse +import math +import sys +import time + +import numpy as np +from scipy.spatial.transform import Rotation + +from isaaccapture.cloudxr import CloudXRLauncher +from isaaccapture.retargeters import ( + GamepadGripperRetargeter, + GamepadToSe2Retargeter, + GamepadToSe2RetargeterConfig, + GamepadToSe3RelRetargeter, + GamepadToSe3RelRetargeterConfig, +) +from isaaccapture.retargeting_engine.deviceio_source_nodes import GamepadSource +from isaaccapture.retargeting_engine.interface import OutputCombiner +from isaaccapture.retargeting_engine.tensor_types import GamepadInputIndex +from isaaccapture.teleop_session_manager import ( + BUNDLED_PLUGINS_DIR, + PluginConfig, + TeleopSession, + TeleopSessionConfig, +) + +STATUS_LINES = 3 + + +def build_pipeline(dead_zone: float) -> OutputCombiner: + """The source's raw outputs next to the SE(3), SE(2) and gripper retargeters' outputs.""" + source = GamepadSource(name="gamepad") + gamepad = {"gamepad": source.output("gamepad")} + se3 = GamepadToSe3RelRetargeter( + GamepadToSe3RelRetargeterConfig(dead_zone=dead_zone), name="se3" + ) + se2 = GamepadToSe2Retargeter( + GamepadToSe2RetargeterConfig(dead_zone=dead_zone), name="se2" + ) + gripper = GamepadGripperRetargeter(name="gripper") + return OutputCombiner( + { + "gamepad": source.output("gamepad"), + "ee_delta": se3.connect(gamepad).output("ee_delta"), + "base_command": se2.connect(gamepad).output("base_command"), + "gripper_command": gripper.connect(gamepad).output("gripper_command"), + } + ) + + +class IntegratedPoses: + """End-effector and base poses accumulated from the retargeters' rate outputs.""" + + def __init__(self) -> None: + self.reset() + + def reset(self) -> None: + self.ee_position = np.zeros(3) + self.ee_rotation = Rotation.identity() + self.base = np.zeros(3) # x, y, yaw + + def step(self, ee_delta, base_command, dt: float) -> None: + delta = np.asarray(ee_delta, dtype=np.float64) + self.ee_position += delta[:3] * dt + # Left-multiplied: the delta rotates about the base frame's axes, as a relative IK + # controller applies it. + self.ee_rotation = Rotation.from_rotvec(delta[3:] * dt) * self.ee_rotation + + v_x, v_y, omega_z = np.asarray(base_command, dtype=np.float64) + yaw = self.base[2] + self.base[0] += (v_x * math.cos(yaw) - v_y * math.sin(yaw)) * dt + self.base[1] += (v_x * math.sin(yaw) + v_y * math.cos(yaw)) * dt + self.base[2] = math.remainder(yaw + omega_z * dt, math.tau) + + def describe(self) -> str: + rpy = self.ee_rotation.as_euler("XYZ", degrees=True) + return ( + f"EE pose: pos [{_fmt(self.ee_position)}] rpy [{_fmt(rpy, 1)}] deg " + f"Base pose: [{_fmt(self.base[:2])}] yaw {math.degrees(self.base[2]):+.1f} deg" + ) + + +# GamepadInput's buttons, printed without the "_click" suffix, and its sticks and triggers. +BUTTONS = [i for i in GamepadInputIndex if i.name.endswith("_CLICK")] +AXES = [i for i in GamepadInputIndex if not i.name.endswith("_CLICK")] + + +def _name(button) -> str: + return button.name.removesuffix("_CLICK").lower() + + +def _fmt(values, decimals: int = 2) -> str: + return " ".join(f"{v:+.{decimals}f}" for v in values) + + +def _show(lines: list[str]) -> None: + """Rewrite the status block in place: move up to its first line, then print each line.""" + print( + f"\033[{STATUS_LINES - 1}F" + "\n".join(f"{line}\033[K" for line in lines), + end="", + flush=True, + ) + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--player", + type=int, + default=1, + help="Read the Nth connected gamepad (default: the first)", + ) + parser.add_argument( + "--dead-zone", + type=float, + default=GamepadToSe3RelRetargeterConfig().dead_zone, + help="Stick deflection below which the retargeters read zero", + ) + parser.add_argument("--duration", type=float, default=30.0, help="Seconds to run") + CloudXRLauncher.add_launcher_arguments(parser) + args = parser.parse_args() + + session_config = TeleopSessionConfig( + app_name="GamepadPrinterExample", + pipeline=build_pipeline(args.dead_zone), + plugins=[ + PluginConfig( + plugin_name="gamepad", + plugin_root_id="gamepad", + search_paths=[BUNDLED_PLUGINS_DIR], + plugin_args=[f"--player={args.player}"], + required=True, + ) + ], + ) + + print( + "Move the sticks or D-pad, or press a button. west toggles the gripper, menu resets the poses." + ) + print("\n" * (STATUS_LINES - 1), end="") + poses = IntegratedPoses() + with CloudXRLauncher.launch_context(args), TeleopSession(session_config) as session: + start_time = time.time() + last_step = time.monotonic() + prev_pressed: set = set() + + while time.time() - start_time < args.duration: + result = session.step() + now = time.monotonic() + poses.step( + result["ee_delta"][0], result["base_command"][0], now - last_step + ) + last_step = now + + elapsed = session.get_elapsed_time() + gripper = "closed" if result["gripper_command"][0] < 0 else "open" + retargeted = ( + f"EE delta: [{_fmt(result['ee_delta'][0])}] " + f"Base cmd: [{_fmt(result['base_command'][0])}] Gripper: {gripper}" + ) + if result["gamepad"].is_none: + _show([f"[{elapsed:5.1f}s] (no gamepad)", retargeted, poses.describe()]) + time.sleep(0.01) + continue + + gamepad = result["gamepad"] + pressed = {b for b in BUTTONS if float(gamepad[b]) > 0.5} + if GamepadInputIndex.MENU_CLICK in pressed - prev_pressed: + poses.reset() + held = " ".join(_name(i) for i in sorted(pressed)) or "-" + _show( + [ + f"[{elapsed:5.1f}s] Axes: [{_fmt(float(gamepad[a]) for a in AXES)}] Held: {held}", + retargeted, + poses.describe(), + ] + ) + # Log every transition above the status block: a quick tap can flash by in it. + events = [ + f"[{elapsed:5.1f}s] {_name(i)} down" + for i in sorted(pressed - prev_pressed) + ] + events += [ + f"[{elapsed:5.1f}s] {_name(i)} up" + for i in sorted(prev_pressed - pressed) + ] + if events: + print("\n" + "\n".join(events) + "\n" * STATUS_LINES, end="") + prev_pressed = pressed + + time.sleep(0.01) + + print("\nDone.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/core/deviceio_trackers/trackers.toml b/src/core/deviceio_trackers/trackers.toml index 549561a285..d27518d43c 100644 --- a/src/core/deviceio_trackers/trackers.toml +++ b/src/core/deviceio_trackers/trackers.toml @@ -106,3 +106,8 @@ class = "SpaceMouseTracker" traits = "SpaceMouseRecordingTraits" max_flatbuffer_size = 256 python_accessor = "get_spacemouse_data" + +[[tracker]] +name = "gamepad" +table = "GamepadSnapshot" +python_accessor = "get_gamepad_data" diff --git a/src/core/schema/fbs/gamepad.fbs b/src/core/schema/fbs/gamepad.fbs new file mode 100644 index 0000000000..d83a4b6a79 --- /dev/null +++ b/src/core/schema/fbs/gamepad.fbs @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +include "timestamp.fbs"; + +namespace core; + +// Inputs every standard gamepad has, the same set as OpenXR's xbox_controller profile: a generic +// controller split in two (see ControllerInputState). Face buttons are named by position, so they +// mean the same on every pad: south is A on an Xbox pad, Cross on a PlayStation pad and B on a +// Nintendo pad. Pad-specific extras (guide, paddles, touchpad) are left out so every source can +// fill every field. +struct GamepadInputState { + // Button states. `*_click` is OpenXR's input component name for a button's boolean state, as in + // ControllerInputState: true for as long as the button is held, not a press-and-release event. + south_click: bool; + east_click: bool; + west_click: bool; + north_click: bool; + view_click: bool; + menu_click: bool; + shoulder_left_click: bool; + shoulder_right_click: bool; + thumbstick_left_click: bool; + thumbstick_right_click: bool; + dpad_up_click: bool; + dpad_down_click: bool; + dpad_left_click: bool; + dpad_right_click: bool; + + // Thumbsticks in [-1, 1] with right and up positive, as OpenXR's thumbstick x and y are. + thumbstick_left_x: float32; + thumbstick_left_y: float32; + thumbstick_right_x: float32; + thumbstick_right_y: float32; + + // Triggers in [0, 1]. + trigger_left_value: float32; + trigger_right_value: float32; +} + +// A gamepad, pushed by the gamepad plugin in the layout above, whatever the pad's own numbering. +table GamepadSnapshot { + // Absent while the plugin has no pad open, so a reader never keeps the last input of a pad that + // went away. + inputs: GamepadInputState (id: 0); +} + +// MCAP recording wrapper for GamepadSnapshot. +table GamepadSnapshotRecord { + data: GamepadSnapshot (id: 0); + timestamp: DeviceDataTimestamp (id: 1); +} + +root_type GamepadSnapshotRecord; diff --git a/src/core/schema/golden/gamepad.bfbs b/src/core/schema/golden/gamepad.bfbs new file mode 100644 index 0000000000..2b8f1874e5 Binary files /dev/null and b/src/core/schema/golden/gamepad.bfbs differ diff --git a/src/core/schema/python/CMakeLists.txt b/src/core/schema/python/CMakeLists.txt index d8fc43bfd8..cee33a811b 100644 --- a/src/core/schema/python/CMakeLists.txt +++ b/src/core/schema/python/CMakeLists.txt @@ -5,6 +5,7 @@ pybind11_add_module(schema_py oak_bindings.h controller_bindings.h full_body_bindings.h + gamepad_bindings.h hand_bindings.h haptic_command_bindings.h head_bindings.h diff --git a/src/core/schema/python/gamepad_bindings.h b/src/core/schema/python/gamepad_bindings.h new file mode 100644 index 0000000000..418b5ceea0 --- /dev/null +++ b/src/core/schema/python/gamepad_bindings.h @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Python bindings for the Gamepad FlatBuffer schema. +// GamepadInputState is a struct; GamepadSnapshot is a table, exposed as an encoded view. + +#pragma once + +#include "schema_serialized.h" + +#include +#include +#include + +#include +#include +#include + +namespace py = pybind11; + +namespace core +{ + +inline void bind_gamepad(py::module& m) +{ + py::class_(m, "GamepadInputState") + .def(py::init(), + py::arg("south_click") = false, py::arg("east_click") = false, py::arg("west_click") = false, + py::arg("north_click") = false, py::arg("view_click") = false, py::arg("menu_click") = false, + py::arg("shoulder_left_click") = false, py::arg("shoulder_right_click") = false, + py::arg("thumbstick_left_click") = false, py::arg("thumbstick_right_click") = false, + py::arg("dpad_up_click") = false, py::arg("dpad_down_click") = false, py::arg("dpad_left_click") = false, + py::arg("dpad_right_click") = false, py::arg("thumbstick_left_x") = 0.0f, + py::arg("thumbstick_left_y") = 0.0f, py::arg("thumbstick_right_x") = 0.0f, + py::arg("thumbstick_right_y") = 0.0f, py::arg("trigger_left_value") = 0.0f, + py::arg("trigger_right_value") = 0.0f, + "Gamepad inputs; omitted buttons are released and omitted axes centered.") + .def_property_readonly("south_click", &GamepadInputState::south_click) + .def_property_readonly("east_click", &GamepadInputState::east_click) + .def_property_readonly("west_click", &GamepadInputState::west_click) + .def_property_readonly("north_click", &GamepadInputState::north_click) + .def_property_readonly("view_click", &GamepadInputState::view_click) + .def_property_readonly("menu_click", &GamepadInputState::menu_click) + .def_property_readonly("shoulder_left_click", &GamepadInputState::shoulder_left_click) + .def_property_readonly("shoulder_right_click", &GamepadInputState::shoulder_right_click) + .def_property_readonly("thumbstick_left_click", &GamepadInputState::thumbstick_left_click) + .def_property_readonly("thumbstick_right_click", &GamepadInputState::thumbstick_right_click) + .def_property_readonly("dpad_up_click", &GamepadInputState::dpad_up_click) + .def_property_readonly("dpad_down_click", &GamepadInputState::dpad_down_click) + .def_property_readonly("dpad_left_click", &GamepadInputState::dpad_left_click) + .def_property_readonly("dpad_right_click", &GamepadInputState::dpad_right_click) + .def_property_readonly("thumbstick_left_x", &GamepadInputState::thumbstick_left_x) + .def_property_readonly("thumbstick_left_y", &GamepadInputState::thumbstick_left_y) + .def_property_readonly("thumbstick_right_x", &GamepadInputState::thumbstick_right_x) + .def_property_readonly("thumbstick_right_y", &GamepadInputState::thumbstick_right_y) + .def_property_readonly("trigger_left_value", &GamepadInputState::trigger_left_value) + .def_property_readonly("trigger_right_value", &GamepadInputState::trigger_right_value) + .def("__repr__", + [](const GamepadInputState& self) + { + std::string held; + const auto add_held = [&held](const char* name, bool pressed) + { + if (pressed) + held += (held.empty() ? "" : ", ") + std::string(name); + }; + add_held("south", self.south_click()); + add_held("east", self.east_click()); + add_held("west", self.west_click()); + add_held("north", self.north_click()); + add_held("view", self.view_click()); + add_held("menu", self.menu_click()); + add_held("shoulder_left", self.shoulder_left_click()); + add_held("shoulder_right", self.shoulder_right_click()); + add_held("thumbstick_left", self.thumbstick_left_click()); + add_held("thumbstick_right", self.thumbstick_right_click()); + add_held("dpad_up", self.dpad_up_click()); + add_held("dpad_down", self.dpad_down_click()); + add_held("dpad_left", self.dpad_left_click()); + add_held("dpad_right", self.dpad_right_click()); + return "GamepadInputState(held=[" + held + "], left=(" + std::to_string(self.thumbstick_left_x()) + + ", " + std::to_string(self.thumbstick_left_y()) + "), right=(" + + std::to_string(self.thumbstick_right_x()) + ", " + std::to_string(self.thumbstick_right_y()) + + "), triggers=(" + std::to_string(self.trigger_left_value()) + ", " + + std::to_string(self.trigger_right_value()) + "))"; + }); + + serialized_class( + m, "GamepadSnapshot", "Encoded gamepad snapshot: the input state, absent while no pad is open.") + .def(py::init( + [](std::optional inputs) + { + GamepadSnapshotT native; + if (inputs) + native.inputs = std::make_shared(*inputs); + return pack(native); + }), + py::arg("inputs") = std::nullopt, "Encode a gamepad snapshot. Without inputs it reads as no pad open.") + .def_property_readonly("inputs", field(&GamepadSnapshot::inputs), py::return_value_policy::reference_internal) + .def("__repr__", [](const Serialized& self) + { return "GamepadSnapshot(inputs=" + std::string(self->inputs() != nullptr ? "..." : "None") + ")"; }); + + bind_record(m, "GamepadSnapshotRecord", "GamepadSnapshot"); +} + +} // namespace core diff --git a/src/core/schema/python/schema_module.cpp b/src/core/schema/python/schema_module.cpp index c76db6cc8a..80d32c9fca 100644 --- a/src/core/schema/python/schema_module.cpp +++ b/src/core/schema/python/schema_module.cpp @@ -8,6 +8,7 @@ // Include binding definitions. #include "controller_bindings.h" #include "full_body_bindings.h" +#include "gamepad_bindings.h" #include "hand_bindings.h" #include "haptic_command_bindings.h" #include "head_bindings.h" @@ -76,6 +77,9 @@ PYBIND11_MODULE(_schema, m) // Bind SpaceMouse types (SpaceMouseOutput table). core::bind_spacemouse(m); + // Bind gamepad types (GamepadInputState struct, GamepadSnapshot table). + core::bind_gamepad(m); + // Bind message channel types (MessageChannelMessages table). core::bind_message_channel(m); diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index 25a5cb98ea..4e5fa231f1 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -19,6 +19,7 @@ add_subdirectory(controller_synthetic_hands) add_subdirectory(generic_3axis_pedal) add_subdirectory(steering_wheel) add_subdirectory(spacemouse) +add_subdirectory(gamepad) add_subdirectory(xsens_full_body) add_subdirectory(so101_leader) add_subdirectory(rebot_devarm_leader) diff --git a/src/plugins/gamepad/CMakeLists.txt b/src/plugins/gamepad/CMakeLists.txt new file mode 100644 index 0000000000..566525bc9a --- /dev/null +++ b/src/plugins/gamepad/CMakeLists.txt @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Reads a gamepad through SDL3 and pushes GamepadSnapshot, in the standard gamepad layout, via OpenXR. +if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(STATUS "Skipping gamepad plugin (Linux only)") + add_custom_target(gamepad_plugin + COMMAND ${CMAKE_COMMAND} -E echo "Skipping gamepad: Linux only") + return() +endif() + +# ------------------------------------------------------------------------------ +# SDL3 (zlib): device support and the SDL_GameControllerDB mappings that turn each pad's own +# numbering into the standard layout. Built static with only the joystick and HIDAPI subsystems, so +# the bundled plugin still needs nothing beyond libc/libstdc++; libudev is loaded at runtime. +# ------------------------------------------------------------------------------ +include(FetchContent) +FetchContent_Declare( + SDL3 + GIT_REPOSITORY https://github.com/libsdl-org/SDL.git + GIT_TAG 829a65d769d935c4852f8159e964312c0957260a # release-3.4.18 +) +foreach(_sdl_option SDL_SHARED SDL_TEST_LIBRARY SDL_TESTS SDL_EXAMPLES SDL_INSTALL + SDL_AUDIO SDL_VIDEO SDL_HAPTIC SDL_POWER SDL_SENSOR SDL_DIALOG SDL_TRAY + SDL_DBUS SDL_IBUS SDL_HIDAPI_LIBUSB) + set(${_sdl_option} OFF CACHE BOOL "" FORCE) +endforeach() +# SDL_UNIX_CONSOLE_BUILD: no windows, so no X11 or Wayland development packages are needed. +foreach(_sdl_option SDL_STATIC SDL_JOYSTICK SDL_HIDAPI SDL_UNIX_CONSOLE_BUILD) + set(${_sdl_option} ON CACHE BOOL "" FORCE) +endforeach() +isaac_teleop_third_party_scope_begin() +FetchContent_MakeAvailable(SDL3) +isaac_teleop_third_party_scope_end() + +# app/ also stages the plugin with its manifest into the Python package. +set(GAMEPAD_PLUGIN_MANIFEST "${CMAKE_CURRENT_SOURCE_DIR}/plugin.yaml") + +add_subdirectory(core) +add_subdirectory(app) + +install(FILES plugin.yaml README.md DESTINATION plugins/gamepad) diff --git a/src/plugins/gamepad/README.md b/src/plugins/gamepad/README.md new file mode 100644 index 0000000000..634c4abb11 --- /dev/null +++ b/src/plugins/gamepad/README.md @@ -0,0 +1,60 @@ + + +# Gamepad Plugin + +Reads a gamepad through [SDL3](https://github.com/libsdl-org/SDL) and pushes `GamepadSnapshot` +via OpenXR. The inputs are `GamepadInputState` from `gamepad.fbs`: the buttons +and axes every standard pad has, the same set as OpenXR's `xbox_controller` profile, as named fields +like `ControllerInputState`'s, with face buttons named by position (`south_click`, `west_click`, +...). SDL resolves each pad's own numbering through its drivers and +[SDL_GameControllerDB](https://github.com/mdqinc/SDL_GameControllerDB) mappings, so `west_click` is X +on an Xbox pad and Square on a PlayStation pad. Read it with `GamepadTracker`, `GamepadSource`, or +any reader using the same `collection_id`. Linux only. + +The plugin is also bundled in the `isaaccapture` package: a `TeleopSession` starts it with +`PluginConfig(plugin_name="gamepad", plugin_root_id="gamepad", search_paths=[BUNDLED_PLUGINS_DIR])`. + +Supported pads: Xbox (360, One, Series, USB and Bluetooth), PlayStation (DualShock 4, DualSense), +Nintendo Switch Pro, Steam Controller, and any pad with an SDL_GameControllerDB mapping. Validated: +Xbox 360 Controller. + +SDL3 (zlib license) is fetched at configure time and linked statically with only its joystick and +HIDAPI subsystems, so the binary needs nothing beyond libc/libstdc++. + +## Device access + +On a desktop session udev grants the logged-in user access to gamepad nodes, so no setup is +needed. Without a local seat (SSH, a service, a container) add the user to the `input` group, and +pass the pad's `/dev/input/event*` node into a container. + +## Usage + +```bash +./gamepad_plugin [--player=N] [--collection-id=ID] +``` + +- **--player**: Read the Nth connected gamepad, in connection order (default 1). The pad is kept + until it disconnects; for several pads, run one plugin per player with its own collection. +- **--collection-id**: Default `gamepad`. Match this when creating `GamepadTracker`. +- **--plugin-root-id**: Set by the plugin launcher; enables device status reporting. + +The plugin pushes at 90 Hz. While no pad is open it keeps pushing snapshots without inputs, and +nothing held, so a reader never holds the last stick deflection of a pad that went away. The pad's +name, type and USB IDs are logged when it is opened. + +A pad without a mapping is not listed as a gamepad. Give SDL one through the +`SDL_GAMECONTROLLERCONFIG` environment variable (an SDL_GameControllerDB line) in the plugin's +environment. + +## Data + +- `inputs`: `GamepadInputState`. Sticks are in `[-1, 1]` with right and up positive, triggers in + `[0, 1]`. + +## Device status + +With `--plugin-root-id`, the plugin reports device `/gamepad` as `CONNECTED`, `DISCONNECTED` (no +pad for this player), or `FAILED` (SDL found the pad but could not open it). diff --git a/src/plugins/gamepad/app/CMakeLists.txt b/src/plugins/gamepad/app/CMakeLists.txt new file mode 100644 index 0000000000..fbd4799f31 --- /dev/null +++ b/src/plugins/gamepad/app/CMakeLists.txt @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +add_executable(gamepad_plugin + main.cpp + gamepad_plugin.cpp + gamepad_plugin.hpp +) + +target_link_libraries(gamepad_plugin PRIVATE + gamepad::gamepad_core + Teleop::plugin_utils + pusherio::pusherio + oxr::oxr_core + isaacteleop_schema + log_bridge::log_bridge_core +) + +install(TARGETS gamepad_plugin RUNTIME DESTINATION plugins/gamepad) + +# Ship the plugin inside the isaaccapture package, so a wheel install can start it (PluginConfig with +# search path BUNDLED_PLUGINS_DIR). The binary links nothing beyond libc/libstdc++. +if(TARGET python_package) + set(_bundle_dir "${CMAKE_BINARY_DIR}/python_package/$/isaaccapture/plugins/gamepad") + add_custom_command(TARGET gamepad_plugin POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory "${_bundle_dir}" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$" "${GAMEPAD_PLUGIN_MANIFEST}" "${_bundle_dir}/" + COMMENT "Bundling gamepad_plugin into the isaaccapture package") + # Files staged into the package tree must exist before the wheel is built from it. + add_dependencies(python_package gamepad_plugin) +endif() diff --git a/src/plugins/gamepad/app/gamepad_plugin.cpp b/src/plugins/gamepad/app/gamepad_plugin.cpp new file mode 100644 index 0000000000..6af089e09b --- /dev/null +++ b/src/plugins/gamepad/app/gamepad_plugin.cpp @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "gamepad_plugin.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace gamepad +{ + +namespace +{ + +// Matches max_flatbuffer_size in trackers.toml (the default), which the host's GamepadTracker reads +// with; a sample is under 100 bytes. +constexpr size_t kMaxFlatbufferSize = 512; + +std::vector required_extensions(bool monitoring) +{ + std::vector extensions = core::SchemaPusher::get_required_extensions(); + if (monitoring) + { + for (const auto& extension : plugin_utils::PluginDeviceStatusPublisher::get_required_extensions()) + { + if (std::find(extensions.begin(), extensions.end(), extension) == extensions.end()) + extensions.push_back(extension); + } + } + return extensions; +} + +std::shared_ptr make_session(const GamepadPluginConfig& config) +{ + if (config.plugin_root_id && config.plugin_root_id->empty()) + throw std::invalid_argument("GamepadPlugin: --plugin-root-id must not be empty"); + return std::make_shared("GamepadPlugin", required_extensions(config.plugin_root_id.has_value())); +} + +} // namespace + +GamepadPlugin::GamepadPlugin(GamepadPluginConfig config) + : connection_(std::move(config.connection)), + session_(make_session(config)), + pusher_(session_->get_handles(), + core::SchemaPusherConfig{ .collection_id = config.collection_id, + .max_flatbuffer_size = kMaxFlatbufferSize, + .tensor_identifier = "gamepad", + .localized_name = "Gamepad", + .app_name = "GamepadPlugin" }) +{ + if (config.plugin_root_id) + status_ = + std::make_unique(session_->get_handles(), *config.plugin_root_id); +} + +GamepadPlugin::~GamepadPlugin() = default; + +void GamepadPlugin::update() +{ + const int64_t now_ns = core::os_monotonic_now_ns(); + connection_.poll(); + push_state(now_ns); + publish_status(now_ns); +} + +void GamepadPlugin::push_state(int64_t now_ns) +{ + core::GamepadSnapshotT out; + // No inputs while no pad is open, so readers drop the last input of a pad that went away. + if (connection_.status() == ConnectionStatus::Connected) + out.inputs = std::make_shared(connection_.state().inputs); + + flatbuffers::FlatBufferBuilder builder(kMaxFlatbufferSize); + builder.Finish(core::GamepadSnapshot::Pack(builder, &out)); + pusher_.push_buffer(builder.GetBufferPointer(), builder.GetSize(), now_ns, now_ns); +} + +void GamepadPlugin::publish_status(int64_t now_ns) +{ + if (!status_) + return; + + plugin_utils::PluginDeviceStatusEntry entry{ .path = DEVICE_PATH }; + switch (connection_.status()) + { + case ConnectionStatus::Connected: + entry.state = core::PluginDeviceState_CONNECTED; + entry.reason = core::PluginDeviceReason_NONE; + break; + case ConnectionStatus::NoDevice: + entry.state = core::PluginDeviceState_DISCONNECTED; + entry.reason = core::PluginDeviceReason_NO_HARDWARE_SIGNAL; + break; + case ConnectionStatus::OpenFailed: + entry.state = core::PluginDeviceState_FAILED; + entry.reason = core::PluginDeviceReason_DEVICE_ERROR; + entry.error = connection_.error(); + break; + } + status_->publish_if_changed({ entry }, now_ns); +} + +} // namespace gamepad +} // namespace plugins diff --git a/src/plugins/gamepad/app/gamepad_plugin.hpp b/src/plugins/gamepad/app/gamepad_plugin.hpp new file mode 100644 index 0000000000..b7c33a6dec --- /dev/null +++ b/src/plugins/gamepad/app/gamepad_plugin.hpp @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include + +#include +#include +#include + +namespace core +{ +class OpenXRSession; +} + +namespace plugins +{ +namespace gamepad +{ + +struct GamepadPluginConfig +{ + GamepadConnectionConfig connection; + std::string collection_id = "gamepad"; + //! Set by the plugin launcher; enables device status reporting for monitoring. + std::optional plugin_root_id; +}; + +/*! + * @brief Reads a gamepad and pushes GamepadSnapshot via OpenXR SchemaPusher. + * + * A sample is pushed on every update, also while no device is open (no inputs, nothing + * held), so the host never keeps the last stick deflection of a pad that went away. + */ +class GamepadPlugin +{ +public: + //! Device path reported to plugin monitoring; matches plugin.yaml. + static constexpr const char* DEVICE_PATH = "/gamepad"; + + explicit GamepadPlugin(GamepadPluginConfig config); + ~GamepadPlugin(); + + void update(); + +private: + void push_state(int64_t now_ns); + void publish_status(int64_t now_ns); + + GamepadConnection connection_; + std::shared_ptr session_; + core::SchemaPusher pusher_; + std::unique_ptr status_; +}; + +} // namespace gamepad +} // namespace plugins diff --git a/src/plugins/gamepad/app/main.cpp b/src/plugins/gamepad/app/main.cpp new file mode 100644 index 0000000000..d6a0bb0f28 --- /dev/null +++ b/src/plugins/gamepad/app/main.cpp @@ -0,0 +1,88 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "gamepad_plugin.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace plugins::gamepad; + +namespace +{ + +constexpr std::string_view kUsage = "Usage: gamepad_plugin [--player=N] [--collection-id=ID] [--plugin-root-id=ID]"; + +// The value of `--name=value` in @p arg, or nullopt when @p arg is another flag. +std::optional flag_value(std::string_view arg, std::string_view name) +{ + if (arg.starts_with(name) && arg.size() > name.size() && arg[name.size()] == '=') + return std::string(arg.substr(name.size() + 1)); + return std::nullopt; +} + +} // namespace + +int main(int argc, char** argv) +try +{ + auto logger = isaaccapture::Logger::get("isaaccapture.plugins.gamepad.main"); + + GamepadPluginConfig config; + for (int i = 1; i < argc; ++i) + { + const std::string_view arg = argv[i]; + if (arg == "--help" || arg == "-h") + { + // Usage text is terminal UX, not a diagnostic. + std::cout << kUsage << std::endl; + return 0; + } + if (auto value = flag_value(arg, "--player")) + config.connection.player = std::stoi(*value); + else if (auto value = flag_value(arg, "--collection-id")) + config.collection_id = *value; + else if (auto value = flag_value(arg, "--plugin-root-id")) + config.plugin_root_id = *value; + else + logger->warn("ignoring unknown argument '{}'", arg); + } + + logger->info("Gamepad (player: {}, collection: {})", config.connection.player, config.collection_id); + + GamepadPlugin plugin(std::move(config)); + + // Push data at 90 Hz + const auto frame_duration = std::chrono::nanoseconds(1000000000 / 90); + const auto program_start = std::chrono::steady_clock::now(); + std::size_t frame_count = 0; + + while (true) + { + plugin.update(); + frame_count++; + std::this_thread::sleep_until(program_start + frame_duration * frame_count); + } + + return 0; +} +catch (const std::exception& e) +{ + auto logger = isaaccapture::Logger::get("isaaccapture.plugins.gamepad.main"); + logger->error("{}: {}", argv[0], e.what()); + return 1; +} +catch (...) +{ + auto logger = isaaccapture::Logger::get("isaaccapture.plugins.gamepad.main"); + logger->error("{}: Unknown error", argv[0]); + return 1; +} diff --git a/src/plugins/gamepad/core/CMakeLists.txt b/src/plugins/gamepad/core/CMakeLists.txt new file mode 100644 index 0000000000..5ac86928fe --- /dev/null +++ b/src/plugins/gamepad/core/CMakeLists.txt @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The SDL3 reader, kept apart from the OpenXR push loop so it can be unit-tested against SDL's +# virtual gamepads without a runtime or a device. +add_library(gamepad_core STATIC + gamepad_connection.cpp + inc/gamepad/gamepad_connection.hpp +) + +target_include_directories(gamepad_core + INTERFACE + $ +) + +target_link_libraries(gamepad_core + PUBLIC + log_bridge::log_bridge_core + isaacteleop_schema + PRIVATE + SDL3::SDL3-static +) + +add_library(gamepad::gamepad_core ALIAS gamepad_core) diff --git a/src/plugins/gamepad/core/gamepad_connection.cpp b/src/plugins/gamepad/core/gamepad_connection.cpp new file mode 100644 index 0000000000..1bf60c1156 --- /dev/null +++ b/src/plugins/gamepad/core/gamepad_connection.cpp @@ -0,0 +1,159 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/gamepad/gamepad_connection.hpp" + +#include + +#include +#include +#include +#include + +namespace plugins +{ +namespace gamepad +{ + +namespace +{ + +// SDL reports [-32768, 32767]; sticks map to [-1, 1] and triggers to [0, 1]. +float stick(SDL_Gamepad* gamepad, SDL_GamepadAxis axis, bool flip = false) +{ + const float value = static_cast(SDL_GetGamepadAxis(gamepad, axis)) / 32767.0f; + return std::clamp(flip ? -value : value, -1.0f, 1.0f); +} + +float trigger(SDL_Gamepad* gamepad, SDL_GamepadAxis axis) +{ + return std::clamp(static_cast(SDL_GetGamepadAxis(gamepad, axis)) / 32767.0f, 0.0f, 1.0f); +} + +} // namespace + +void GamepadState::clear() +{ + inputs = core::GamepadInputState(); +} + +GamepadConnection::GamepadConnection(GamepadConnectionConfig config) : config_(config) +{ + if (config_.player < 1) + throw std::invalid_argument("GamepadConnection: player must be 1 or more"); + + // The plugin launcher stops the plugin with a signal; SDL must not intercept it. + SDL_SetHint(SDL_HINT_NO_SIGNAL_HANDLERS, "1"); + // There is no window, so the process never has focus: read the pad regardless. + SDL_SetHint(SDL_HINT_JOYSTICK_ALLOW_BACKGROUND_EVENTS, "1"); + if (!SDL_InitSubSystem(SDL_INIT_GAMEPAD)) + throw std::runtime_error(std::string("GamepadConnection: SDL_Init failed: ") + SDL_GetError()); + // poll() reads state directly, so nothing drains an event queue. + SDL_SetJoystickEventsEnabled(false); + SDL_SetGamepadEventsEnabled(false); +} + +GamepadConnection::~GamepadConnection() +{ + if (gamepad_ != nullptr) + SDL_CloseGamepad(gamepad_); + SDL_QuitSubSystem(SDL_INIT_GAMEPAD); +} + +void GamepadConnection::poll() +{ + // Refreshes every pad's state and detects pads added or removed. + SDL_UpdateGamepads(); + if (gamepad_ != nullptr && !SDL_GamepadConnected(gamepad_)) + close_device(); + if (gamepad_ == nullptr) + open_device(); + if (gamepad_ != nullptr) + read_state(); +} + +void GamepadConnection::open_device() +{ + int count = 0; + SDL_JoystickID* ids = SDL_GetGamepads(&count); + if (ids == nullptr) + { + set_status(ConnectionStatus::NoDevice); + return; + } + // Instance IDs only grow, so ascending order is connection order. + std::sort(ids, ids + count); + const SDL_JoystickID id = count >= config_.player ? ids[config_.player - 1] : 0; + SDL_free(ids); + if (id == 0) + { + set_status(ConnectionStatus::NoDevice); + return; + } + + gamepad_ = SDL_OpenGamepad(id); + if (gamepad_ == nullptr) + { + set_status(ConnectionStatus::OpenFailed, SDL_GetError()); + return; + } + + char usb_ids[16]; + std::snprintf(usb_ids, sizeof(usb_ids), "%04x:%04x", SDL_GetGamepadVendor(gamepad_), SDL_GetGamepadProduct(gamepad_)); + const char* name = SDL_GetGamepadName(gamepad_); + const char* type = SDL_GetGamepadStringForType(SDL_GetGamepadType(gamepad_)); + description_ = std::string(name != nullptr ? name : "Unknown gamepad") + " (" + + (type != nullptr ? type : "unknown") + ", " + usb_ids + ")"; + state_.clear(); + set_status(ConnectionStatus::Connected); +} + +void GamepadConnection::close_device() +{ + SDL_CloseGamepad(gamepad_); + gamepad_ = nullptr; + logger_->info("Gamepad {} disconnected", description_); + description_.clear(); + state_.clear(); +} + +void GamepadConnection::read_state() +{ + const auto pressed = [this](SDL_GamepadButton button) { return SDL_GetGamepadButton(gamepad_, button); }; + // SDL reports stick Y down-positive; flip it to up-positive, as OpenXR's thumbstick y is. + state_.inputs = core::GamepadInputState( + pressed(SDL_GAMEPAD_BUTTON_SOUTH), pressed(SDL_GAMEPAD_BUTTON_EAST), pressed(SDL_GAMEPAD_BUTTON_WEST), + pressed(SDL_GAMEPAD_BUTTON_NORTH), pressed(SDL_GAMEPAD_BUTTON_BACK), pressed(SDL_GAMEPAD_BUTTON_START), + pressed(SDL_GAMEPAD_BUTTON_LEFT_SHOULDER), pressed(SDL_GAMEPAD_BUTTON_RIGHT_SHOULDER), + pressed(SDL_GAMEPAD_BUTTON_LEFT_STICK), pressed(SDL_GAMEPAD_BUTTON_RIGHT_STICK), + pressed(SDL_GAMEPAD_BUTTON_DPAD_UP), pressed(SDL_GAMEPAD_BUTTON_DPAD_DOWN), + pressed(SDL_GAMEPAD_BUTTON_DPAD_LEFT), pressed(SDL_GAMEPAD_BUTTON_DPAD_RIGHT), + stick(gamepad_, SDL_GAMEPAD_AXIS_LEFTX), stick(gamepad_, SDL_GAMEPAD_AXIS_LEFTY, true), + stick(gamepad_, SDL_GAMEPAD_AXIS_RIGHTX), stick(gamepad_, SDL_GAMEPAD_AXIS_RIGHTY, true), + trigger(gamepad_, SDL_GAMEPAD_AXIS_LEFT_TRIGGER), trigger(gamepad_, SDL_GAMEPAD_AXIS_RIGHT_TRIGGER)); +} + +void GamepadConnection::set_status(ConnectionStatus status, std::string error) +{ + const bool report = !status_logged_ || status != status_; + status_ = status; + error_ = std::move(error); + status_logged_ = true; + if (!report) + return; + switch (status) + { + case ConnectionStatus::Connected: + logger_->info("Opened gamepad {} as player {}", description_, config_.player); + break; + case ConnectionStatus::NoDevice: + logger_->info("No gamepad for player {}; waiting for one to connect", config_.player); + break; + case ConnectionStatus::OpenFailed: + logger_->warn("Cannot open gamepad for player {}: {}", config_.player, error_); + break; + } +} + +} // namespace gamepad +} // namespace plugins diff --git a/src/plugins/gamepad/core/inc/gamepad/gamepad_connection.hpp b/src/plugins/gamepad/core/inc/gamepad/gamepad_connection.hpp new file mode 100644 index 0000000000..aede7a8d84 --- /dev/null +++ b/src/plugins/gamepad/core/inc/gamepad/gamepad_connection.hpp @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +#include +#include + +struct SDL_Gamepad; + +namespace plugins +{ +namespace gamepad +{ + +//! Why the pad is or is not open. +enum class ConnectionStatus +{ + Connected, //!< The pad is open and being read. + NoDevice, //!< Fewer gamepads are connected than the configured player number. + OpenFailed, //!< The pad is connected but SDL could not open it. +}; + +struct GamepadConnectionConfig +{ + //! 1-based: read the Nth connected gamepad, in connection order. The pad is kept until it + //! disconnects, then the Nth one is taken again. + int player = 1; +}; + +struct GamepadState +{ + //! Sticks in [-1, 1] with right and up positive, triggers in [0, 1] (see gamepad.fbs). + core::GamepadInputState inputs; + + //! The state of a pad that is not open: no buttons held, axes at rest. + void clear(); +}; + +/*! + * @brief Reads one gamepad through SDL3 in the standard layout, reopening after a disconnect. + * + * SDL resolves each pad's own button and axis numbering through its built-in drivers and + * SDL_GameControllerDB mappings. poll() refreshes SDL, opens the configured player's pad when one + * is connected, and reads its state; a closed pad's state is cleared. + */ +class GamepadConnection +{ +public: + //! Initializes SDL's gamepad subsystem; throws std::runtime_error when it cannot. + explicit GamepadConnection(GamepadConnectionConfig config); + ~GamepadConnection(); + + GamepadConnection(const GamepadConnection&) = delete; + GamepadConnection& operator=(const GamepadConnection&) = delete; + + void poll(); + + ConnectionStatus status() const + { + return status_; + } + //! Why the last open failed; empty unless status() is OpenFailed. + const std::string& error() const + { + return error_; + } + //! The open pad's name, type and USB IDs; empty while none is open. + const std::string& description() const + { + return description_; + } + const GamepadState& state() const + { + return state_; + } + +private: + void open_device(); + void close_device(); + void read_state(); + void set_status(ConnectionStatus status, std::string error = {}); + + GamepadConnectionConfig config_; + SDL_Gamepad* gamepad_ = nullptr; + ConnectionStatus status_ = ConnectionStatus::NoDevice; + bool status_logged_ = false; //!< The current status() has been logged once. + std::string error_; + std::string description_; + GamepadState state_; + std::shared_ptr logger_ = isaaccapture::Logger::get("isaaccapture.plugins.gamepad.GamepadConnection"); +}; + +} // namespace gamepad +} // namespace plugins diff --git a/src/plugins/gamepad/plugin.yaml b/src/plugins/gamepad/plugin.yaml new file mode 100644 index 0000000000..df083a4f08 --- /dev/null +++ b/src/plugins/gamepad/plugin.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: gamepad +description: "Gamepad (Xbox, PlayStation, Switch Pro, ...) via SDL3" +command: "./gamepad_plugin" +version: "1.0.0" +devices: + - path: "/gamepad" + type: "gamepad" + description: "Gamepad in the standard layout" diff --git a/src/python/isaaccapture/retargeters/__init__.py b/src/python/isaaccapture/retargeters/__init__.py index f5e5ee71eb..518fb9b2bf 100644 --- a/src/python/isaaccapture/retargeters/__init__.py +++ b/src/python/isaaccapture/retargeters/__init__.py @@ -23,6 +23,9 @@ - SpaceMouseToSe3RelRetargeter: SpaceMouse axes -> relative EE delta control - SpaceMouseGripperRetargeter: SpaceMouse left-button toggle -> gripper open/closed - SpaceMouseToSe2Retargeter: SpaceMouse axes -> base velocity command (v_x, v_y, omega_z) + - GamepadToSe3RelRetargeter: Gamepad stick/dpad axes -> relative EE delta control + - GamepadGripperRetargeter: Gamepad X-button toggle -> gripper open/closed + - GamepadToSe2Retargeter: Gamepad stick axes -> base velocity command (v_x, v_y, omega_z) - SO101ClutchRetargeter: Clutch-rebased absolute EE pose for the SO-101 5-DOF arm -- re-latches BOTH home position and orientation on every engage, base-frame left-composed, no fixed offset @@ -136,6 +139,11 @@ "SpaceMouseGripperRetargeter", None, ), + "GamepadGripperRetargeter": ( + ".gripper_retargeter", + "GamepadGripperRetargeter", + None, + ), "KeyboardGripperRetargeter": ( ".gripper_retargeter", "KeyboardGripperRetargeter", @@ -174,6 +182,28 @@ "SpaceMouseToSe2RetargeterConfig", None, ), + # .gamepad_se3_retargeter (requires retargeters-lite extra: scipy) + "GamepadToSe3RelRetargeter": ( + ".gamepad_se3_retargeter", + "GamepadToSe3RelRetargeter", + "retargeters-lite", + ), + "GamepadToSe3RelRetargeterConfig": ( + ".gamepad_se3_retargeter", + "GamepadToSe3RelRetargeterConfig", + "retargeters-lite", + ), + # .gamepad_se2_retargeter + "GamepadToSe2Retargeter": ( + ".gamepad_se2_retargeter", + "GamepadToSe2Retargeter", + None, + ), + "GamepadToSe2RetargeterConfig": ( + ".gamepad_se2_retargeter", + "GamepadToSe2RetargeterConfig", + None, + ), # .keyboard_se2_retargeter "KeyboardToSe2Retargeter": ( ".keyboard_se2_retargeter", @@ -369,6 +399,11 @@ def __getattr__(name: str): "SpaceMouseGripperRetargeter", "SpaceMouseToSe2Retargeter", "SpaceMouseToSe2RetargeterConfig", + "GamepadToSe3RelRetargeter", + "GamepadToSe3RelRetargeterConfig", + "GamepadGripperRetargeter", + "GamepadToSe2Retargeter", + "GamepadToSe2RetargeterConfig", # SO-101 5-DOF arm retargeters "SO101ClutchRetargeter", "SO101GripperRetargeter", diff --git a/src/python/isaaccapture/retargeters/gamepad_se2_retargeter.py b/src/python/isaaccapture/retargeters/gamepad_se2_retargeter.py new file mode 100644 index 0000000000..02b061cafa --- /dev/null +++ b/src/python/isaaccapture/retargeters/gamepad_se2_retargeter.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Gamepad SE2 Retargeter Module. + +Maps raw gamepad axis state to a base velocity command (v_x, v_y, omega_z). +""" + +from dataclasses import dataclass + +import numpy as np + +from isaaccapture.retargeting_engine.interface import ( + BaseRetargeter, + RetargeterIOType, +) +from isaaccapture.retargeting_engine.interface.retargeter_core_types import RetargeterIO +from isaaccapture.retargeting_engine.interface.tensor_group_type import ( + OptionalType, + TensorGroupType, +) +from isaaccapture.retargeting_engine.tensor_types import ( + DLDataType, + GamepadInput, + GamepadInputIndex, + NDArrayType, +) + + +@dataclass +class GamepadToSe2RetargeterConfig: + """Configuration for the gamepad-to-SE2 base-velocity retargeter.""" + + v_x_sensitivity: float = 1.0 + v_y_sensitivity: float = 1.0 + omega_z_sensitivity: float = 1.0 + # An idle Xbox 360 pad's stick drift reaches 0.14. + dead_zone: float = 0.15 + + +class GamepadToSe2Retargeter(BaseRetargeter): + """ + Maps gamepad stick state to a 3D base velocity command (v_x, v_y, omega_z). + + Stick bindings: + Left stick up/down: +/-v_x Left stick right/left: +/-v_y + Right stick right/left: +/-omega_z + + Output is the instantaneous command implied by the current stick deflection + (scaled by sensitivity), not an integrated velocity -- matching a continuous-axis + input device. + """ + + def __init__(self, config: GamepadToSe2RetargeterConfig, name: str) -> None: + self._config = config + super().__init__(name=name) + + def input_spec(self) -> RetargeterIOType: + return {"gamepad": OptionalType(GamepadInput())} + + def output_spec(self) -> RetargeterIOType: + return { + "base_command": TensorGroupType( + "base_command", + [ + NDArrayType( + "velocity", shape=(3,), dtype=DLDataType.FLOAT, dtype_bits=32 + ) + ], + ) + } + + def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> None: + base_command = outputs["base_command"] + gamepad = inputs["gamepad"] + if gamepad.is_none: + base_command[0] = np.zeros(3, dtype=np.float32) + return + + dead_zone = self._config.dead_zone + + def deadzoned(value: float) -> float: + return 0.0 if abs(value) < dead_zone else value + + v_x = ( + deadzoned(float(gamepad[GamepadInputIndex.THUMBSTICK_LEFT_Y])) + * self._config.v_x_sensitivity + ) + v_y = ( + deadzoned(float(gamepad[GamepadInputIndex.THUMBSTICK_LEFT_X])) + * self._config.v_y_sensitivity + ) + omega_z = ( + deadzoned(float(gamepad[GamepadInputIndex.THUMBSTICK_RIGHT_X])) + * self._config.omega_z_sensitivity + ) + + base_command[0] = np.array([v_x, v_y, omega_z], dtype=np.float32) diff --git a/src/python/isaaccapture/retargeters/gamepad_se3_retargeter.py b/src/python/isaaccapture/retargeters/gamepad_se3_retargeter.py new file mode 100644 index 0000000000..127b362db0 --- /dev/null +++ b/src/python/isaaccapture/retargeters/gamepad_se3_retargeter.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Gamepad SE3 Retargeter Module. + +Maps raw gamepad axis state to end-effector delta commands. +""" + +from dataclasses import dataclass + +import numpy as np +from scipy.spatial.transform import Rotation + +from isaaccapture.retargeting_engine.interface import ( + BaseRetargeter, + RetargeterIOType, +) +from isaaccapture.retargeting_engine.interface.retargeter_core_types import RetargeterIO +from isaaccapture.retargeting_engine.interface.tensor_group_type import ( + OptionalType, + TensorGroupType, +) +from isaaccapture.retargeting_engine.tensor_types import ( + DLDataType, + GamepadInput, + GamepadInputIndex, + NDArrayType, +) + + +@dataclass +class GamepadToSe3RelRetargeterConfig: + """Configuration for the gamepad-to-SE3-relative retargeter.""" + + pos_sensitivity: float = 0.4 + rot_sensitivity: float = 0.8 + # An idle Xbox 360 pad's stick drift reaches 0.14. + dead_zone: float = 0.15 + + +class GamepadToSe3RelRetargeter(BaseRetargeter): + """ + Maps gamepad stick/dpad state to a 6D end-effector delta command. + + Stick/D-pad bindings: + Left stick up/down: +/-X, Left stick right/left: +/-Y, + Right stick up/down: +/-Z (position) + D-pad left/right: +/-roll, D-pad down/up: +/-pitch, + Right stick right/left: +/-yaw (rotation) + + Output is the instantaneous command implied by the current stick/dpad deflection + (scaled by sensitivity), not an integrated delta -- matching a continuous-axis + input device. + """ + + def __init__(self, config: GamepadToSe3RelRetargeterConfig, name: str) -> None: + self._config = config + super().__init__(name=name) + + def input_spec(self) -> RetargeterIOType: + return {"gamepad": OptionalType(GamepadInput())} + + def output_spec(self) -> RetargeterIOType: + return { + "ee_delta": TensorGroupType( + "ee_delta", + [ + NDArrayType( + "delta", shape=(6,), dtype=DLDataType.FLOAT, dtype_bits=32 + ) + ], + ) + } + + def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> None: + ee_delta = outputs["ee_delta"] + gamepad = inputs["gamepad"] + if gamepad.is_none: + ee_delta[0] = np.zeros(6, dtype=np.float32) + return + + def value(field) -> float: + return float(gamepad[field]) + + i = GamepadInputIndex + pos_sens = self._config.pos_sensitivity + rot_sens = self._config.rot_sensitivity + dead_zone = self._config.dead_zone + + def deadzoned(value: float) -> float: + return 0.0 if abs(value) < dead_zone else value + + delta_pos = np.zeros(3) + delta_pos[0] = deadzoned(value(i.THUMBSTICK_LEFT_Y)) * pos_sens + delta_pos[1] = deadzoned(value(i.THUMBSTICK_LEFT_X)) * pos_sens + delta_pos[2] = deadzoned(value(i.THUMBSTICK_RIGHT_Y)) * pos_sens + + delta_euler = np.zeros(3) + delta_euler[0] = ( + (value(i.DPAD_LEFT_CLICK) - value(i.DPAD_RIGHT_CLICK)) * rot_sens * 0.8 + ) + delta_euler[1] = ( + (value(i.DPAD_DOWN_CLICK) - value(i.DPAD_UP_CLICK)) * rot_sens * 0.8 + ) + delta_euler[2] = deadzoned(value(i.THUMBSTICK_RIGHT_X)) * rot_sens + + delta_rot = Rotation.from_euler("XYZ", delta_euler).as_rotvec() + + ee_delta[0] = np.concatenate([delta_pos, delta_rot]).astype(np.float32) diff --git a/src/python/isaaccapture/retargeters/gripper_retargeter.py b/src/python/isaaccapture/retargeters/gripper_retargeter.py index 4844ab8997..9dbf5bf20d 100644 --- a/src/python/isaaccapture/retargeters/gripper_retargeter.py +++ b/src/python/isaaccapture/retargeters/gripper_retargeter.py @@ -5,7 +5,8 @@ Gripper Retargeter Module. Gripper control from hand tracking or controller input (GripperRetargeter), and toggles from a -keyboard key (KeyboardGripperRetargeter) or a SpaceMouse button (SpaceMouseGripperRetargeter). +keyboard key (KeyboardGripperRetargeter), a SpaceMouse button (SpaceMouseGripperRetargeter) or a +gamepad button (GamepadGripperRetargeter). """ import numpy as np @@ -32,6 +33,8 @@ HandInputIndex, HandJointIndex, ControllerInputIndex, + GamepadInput, + GamepadInputIndex, ) @@ -225,3 +228,50 @@ def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> N self._closed = False gripper_out[0] = -1.0 if self._closed else 1.0 + + +class GamepadGripperRetargeter(BaseRetargeter): + """ + Toggles a gripper open/closed state on each press of the gamepad's west face button (X on an + Xbox pad, Square on a PlayStation pad). + + Buttons are sampled, so a press is its rising edge between frames. A reset reopens the + gripper, and a button held through the reset does not toggle again. + + Output matches GripperRetargeter's convention: -1.0 when closed, 1.0 when open. + """ + + TOGGLE_BUTTON = GamepadInputIndex.WEST_CLICK + + def __init__(self, name: str) -> None: + super().__init__(name=name) + self._closed = False + self._was_pressed = False + + def input_spec(self) -> RetargeterIOType: + return {"gamepad": OptionalType(GamepadInput())} + + def output_spec(self) -> RetargeterIOType: + return { + "gripper_command": TensorGroupType( + "gripper_command", [FloatType("command")] + ) + } + + def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> None: + gripper_out = outputs["gripper_command"] + gamepad = inputs["gamepad"] + + # Without a device the last edge state is kept, so a button still held when the + # device returns is not read as a new press. + if not gamepad.is_none: + pressed = float(gamepad[self.TOGGLE_BUTTON]) > 0.5 + if context.execution_events.reset: + self._closed = False + elif pressed and not self._was_pressed: + self._closed = not self._closed + self._was_pressed = pressed + elif context.execution_events.reset: + self._closed = False + + gripper_out[0] = -1.0 if self._closed else 1.0 diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py index e0c0deabd3..338327a763 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py @@ -17,6 +17,7 @@ SpaceMouseSource, SpaceMouseTranslationType, ) +from .gamepad_source import GamepadSource from .keyboard_source import ( KeyboardHeldType, @@ -48,6 +49,7 @@ Generic3AxisPedalOutputTrackedType, KeyboardOutputTrackedType, SpaceMouseOutputTrackedType, + GamepadSnapshotTrackedType, JointStateOutputTrackedType, FullBodyPoseTrackedType, SomaBodyJointRotationsTrackedType, @@ -60,6 +62,7 @@ DeviceIOGeneric3AxisPedalOutputTracked, DeviceIOKeyboardOutputTracked, DeviceIOSpaceMouseOutputTracked, + DeviceIOGamepadSnapshotTracked, DeviceIOJointStateOutputTracked, DeviceIOFullBodyPoseTracked, DeviceIOSomaBodyJointRotationsTracked, @@ -85,6 +88,7 @@ "SpaceMouseRotationType", "SpaceMouseSource", "SpaceMouseTranslationType", + "GamepadSource", "KeyboardHeldType", "KeyboardPressedType", "KeyboardAttachment", @@ -112,6 +116,7 @@ "Generic3AxisPedalOutputTrackedType", "KeyboardOutputTrackedType", "SpaceMouseOutputTrackedType", + "GamepadSnapshotTrackedType", "JointStateOutputTrackedType", "FullBodyPoseTrackedType", "SomaBodyJointRotationsTrackedType", @@ -127,6 +132,7 @@ "DeviceIOGeneric3AxisPedalOutputTracked", "DeviceIOKeyboardOutputTracked", "DeviceIOSpaceMouseOutputTracked", + "DeviceIOGamepadSnapshotTracked", "DeviceIOJointStateOutputTracked", "DeviceIOFullBodyPoseTracked", "DeviceIOSomaBodyJointRotationsTracked", diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py index 474d11bedf..c062178318 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py @@ -21,6 +21,7 @@ Generic3AxisPedalOutput, KeyboardOutput, SpaceMouseOutput, + GamepadSnapshot, JointStateOutput, FullBodyPose, SomaBodyJointPoses, @@ -110,6 +111,12 @@ class SpaceMouseOutputTrackedType(_PayloadTensorType): _payload_cls = SpaceMouseOutput +class GamepadSnapshotTrackedType(_PayloadTensorType): + """GamepadSnapshot payload from DeviceIO GamepadTracker.""" + + _payload_cls = GamepadSnapshot + + class JointStateOutputTrackedType(_PayloadTensorType): """JointStateOutput payload from DeviceIO JointStateTracker.""" @@ -238,6 +245,18 @@ def DeviceIOSpaceMouseOutputTracked() -> TensorGroupType: ) +def DeviceIOGamepadSnapshotTracked() -> TensorGroupType: + """Tracked gamepad snapshot from DeviceIO GamepadTracker. + + Contains: + gamepad_tracked: GamepadSnapshot handle, or None while the plugin is not pushing + """ + return TensorGroupType( + "deviceio_gamepad_snapshot", + [GamepadSnapshotTrackedType("gamepad_tracked")], + ) + + def DeviceIOJointStateOutputTracked() -> TensorGroupType: """Tracked joint-state data from DeviceIO JointStateTracker. diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/gamepad_source.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/gamepad_source.py new file mode 100644 index 0000000000..30041c7292 --- /dev/null +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/gamepad_source.py @@ -0,0 +1,127 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Gamepad Source Node - DeviceIO to Retargeting Engine converter. + +Converts the GamepadSnapshot pushed by the gamepad plugin to the standard GamepadInput tensor group: +the buttons and axes every standard gamepad has, by name. The plugin resolves each pad's own +numbering, so west_click is X on an Xbox pad and Square on a PlayStation pad. + +Buttons are sampled: each frame reports the buttons held at the plugin's latest sample, so a press +and release between two samples is not seen. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from ..interface.retargeter_core_types import RetargeterIO, RetargeterIOType +from ..interface.tensor_group import TensorGroup +from ..interface.tensor_group_type import OptionalType +from ..tensor_types import GamepadInput, GamepadInputIndex +from .deviceio_tensor_types import DeviceIOGamepadSnapshotTracked +from .interface import IDeviceIOSource + +if TYPE_CHECKING: + from isaaccapture.deviceio_trackers import ITracker + from isaaccapture.schema import GamepadSnapshot + +# Default collection_id of the gamepad plugin and GamepadTracker. +DEFAULT_GAMEPAD_COLLECTION_ID = "gamepad" + + +class GamepadSource(IDeviceIOSource): + """ + Stateless converter: DeviceIO GamepadSnapshot → GamepadInput tensor group. + + Inputs: + - "deviceio_gamepad": GamepadSnapshot from GamepadTracker + + Outputs (Optional -- absent while the plugin is not running or has no gamepad open): + - "gamepad": GamepadInput, indexed by GamepadInputIndex + + The data comes from the gamepad plugin. Start it with the session, using the copy bundled with + isaaccapture:: + + PluginConfig( + plugin_name="gamepad", + plugin_root_id="gamepad", + search_paths=[BUNDLED_PLUGINS_DIR], + ) + + and pass ``--collection-id=`` in ``plugin_args`` for a non-default collection, + with ``--player=N`` to read the Nth connected pad. + """ + + def __init__( + self, name: str, collection_id: str = DEFAULT_GAMEPAD_COLLECTION_ID + ) -> None: + """Initialize the source node and its GamepadTracker. + + Args: + name: Unique name for this source node. + collection_id: Tensor collection the gamepad plugin pushes to. + """ + from isaaccapture.deviceio_trackers import GamepadTracker + + self._gamepad_tracker = GamepadTracker(collection_id) + super().__init__(name) + + def get_tracker(self) -> "ITracker": + """Get the GamepadTracker instance for TeleopSession to register.""" + return self._gamepad_tracker + + def poll_tracker(self, deviceio_session: Any) -> RetargeterIO: + """Poll the gamepad tracker. + + Returns: + Dict with "deviceio_gamepad" TensorGroup containing GamepadSnapshot | None. + """ + state = self._gamepad_tracker.get_gamepad_data(deviceio_session) + tg = TensorGroup(DeviceIOGamepadSnapshotTracked()) + tg[0] = state + return {"deviceio_gamepad": tg} + + def input_spec(self) -> RetargeterIOType: + """Declare DeviceIO gamepad input.""" + return {"deviceio_gamepad": DeviceIOGamepadSnapshotTracked()} + + def output_spec(self) -> RetargeterIOType: + """Declare the gamepad output (Optional).""" + return {"gamepad": OptionalType(GamepadInput())} + + def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> None: + """Convert GamepadSnapshot to GamepadInput, or set it to None without a pad.""" + snapshot: GamepadSnapshot | None = inputs["deviceio_gamepad"][0] + + group = outputs["gamepad"] + if snapshot is None or snapshot.inputs is None: + group.set_none() + return + + pad = snapshot.inputs + group[GamepadInputIndex.SOUTH_CLICK] = float(pad.south_click) + group[GamepadInputIndex.EAST_CLICK] = float(pad.east_click) + group[GamepadInputIndex.WEST_CLICK] = float(pad.west_click) + group[GamepadInputIndex.NORTH_CLICK] = float(pad.north_click) + group[GamepadInputIndex.VIEW_CLICK] = float(pad.view_click) + group[GamepadInputIndex.MENU_CLICK] = float(pad.menu_click) + group[GamepadInputIndex.SHOULDER_LEFT_CLICK] = float(pad.shoulder_left_click) + group[GamepadInputIndex.SHOULDER_RIGHT_CLICK] = float(pad.shoulder_right_click) + group[GamepadInputIndex.THUMBSTICK_LEFT_CLICK] = float( + pad.thumbstick_left_click + ) + group[GamepadInputIndex.THUMBSTICK_RIGHT_CLICK] = float( + pad.thumbstick_right_click + ) + group[GamepadInputIndex.DPAD_UP_CLICK] = float(pad.dpad_up_click) + group[GamepadInputIndex.DPAD_DOWN_CLICK] = float(pad.dpad_down_click) + group[GamepadInputIndex.DPAD_LEFT_CLICK] = float(pad.dpad_left_click) + group[GamepadInputIndex.DPAD_RIGHT_CLICK] = float(pad.dpad_right_click) + group[GamepadInputIndex.THUMBSTICK_LEFT_X] = float(pad.thumbstick_left_x) + group[GamepadInputIndex.THUMBSTICK_LEFT_Y] = float(pad.thumbstick_left_y) + group[GamepadInputIndex.THUMBSTICK_RIGHT_X] = float(pad.thumbstick_right_x) + group[GamepadInputIndex.THUMBSTICK_RIGHT_Y] = float(pad.thumbstick_right_y) + group[GamepadInputIndex.TRIGGER_LEFT_VALUE] = float(pad.trigger_left_value) + group[GamepadInputIndex.TRIGGER_RIGHT_VALUE] = float(pad.trigger_right_value) diff --git a/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py b/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py index 514b057465..2454c5c44d 100644 --- a/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py +++ b/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py @@ -12,6 +12,7 @@ SomaHandInput, HeadInput, ControllerInput, + GamepadInput, FullBodyInput, SomaBodyInput, TransformMatrix, @@ -37,6 +38,7 @@ SomaHandInputIndex, HeadInputIndex, ControllerInputIndex, + GamepadInputIndex, Generic3AxisPedalInputIndex, FullBodyInputIndex, SomaBodyInputIndex, @@ -59,6 +61,7 @@ "SomaHandInput", "HeadInput", "ControllerInput", + "GamepadInput", "FullBodyInput", "SomaBodyInput", "TransformMatrix", @@ -82,6 +85,7 @@ "SomaHandInputIndex", "HeadInputIndex", "ControllerInputIndex", + "GamepadInputIndex", "Generic3AxisPedalInputIndex", "FullBodyInputIndex", "SomaBodyInputIndex", diff --git a/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py b/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py index 40a954b7ac..32fa7abcea 100644 --- a/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py +++ b/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py @@ -20,6 +20,7 @@ SomaHandInput, HeadInput, ControllerInput, + GamepadInput, Generic3AxisPedalInput, FullBodyInput, SomaBodyInput, @@ -46,6 +47,9 @@ def _create_index_enum(name: str, group_type, prefix: str = "") -> IntEnum: ControllerInputIndex: Any = _create_index_enum( "ControllerInputIndex", ControllerInput(), "controller_" ) +GamepadInputIndex: Any = _create_index_enum( + "GamepadInputIndex", GamepadInput(), "gamepad_" +) Generic3AxisPedalInputIndex: Any = _create_index_enum( "Generic3AxisPedalInputIndex", Generic3AxisPedalInput(), "pedal_" ) diff --git a/src/python/isaaccapture/retargeting_engine/tensor_types/standard_types.py b/src/python/isaaccapture/retargeting_engine/tensor_types/standard_types.py index ab16e30ad4..6b165f952d 100644 --- a/src/python/isaaccapture/retargeting_engine/tensor_types/standard_types.py +++ b/src/python/isaaccapture/retargeting_engine/tensor_types/standard_types.py @@ -235,6 +235,42 @@ def input_spec(self) -> RetargeterIO: ) +def GamepadInput() -> TensorGroupType: + """ + Standard TensorGroupType for gamepad data: a generic controller split in two. + + Mirrors GamepadInputState in gamepad.fbs. Buttons are 1.0 while held and 0.0 otherwise; face + buttons are named by position (south is A on Xbox, Cross on PlayStation). Thumbsticks are in + [-1, 1] with right and up positive, as in OpenXR; triggers are in [0, 1]. Index fields with + GamepadInputIndex, e.g. ``gamepad[GamepadInputIndex.WEST_CLICK]``. + """ + return TensorGroupType( + "gamepad", + [ + FloatType("gamepad_south_click"), + FloatType("gamepad_east_click"), + FloatType("gamepad_west_click"), + FloatType("gamepad_north_click"), + FloatType("gamepad_view_click"), + FloatType("gamepad_menu_click"), + FloatType("gamepad_shoulder_left_click"), + FloatType("gamepad_shoulder_right_click"), + FloatType("gamepad_thumbstick_left_click"), + FloatType("gamepad_thumbstick_right_click"), + FloatType("gamepad_dpad_up_click"), + FloatType("gamepad_dpad_down_click"), + FloatType("gamepad_dpad_left_click"), + FloatType("gamepad_dpad_right_click"), + FloatType("gamepad_thumbstick_left_x"), + FloatType("gamepad_thumbstick_left_y"), + FloatType("gamepad_thumbstick_right_x"), + FloatType("gamepad_thumbstick_right_y"), + FloatType("gamepad_trigger_left_value"), + FloatType("gamepad_trigger_right_value"), + ], + ) + + # ============================================================================ # Full Body Tracking Types # ============================================================================ diff --git a/src/python/isaaccapture/schema/__init__.py b/src/python/isaaccapture/schema/__init__.py index 8fe0ec4fa4..c7f3ac934d 100644 --- a/src/python/isaaccapture/schema/__init__.py +++ b/src/python/isaaccapture/schema/__init__.py @@ -44,6 +44,10 @@ # SpaceMouse types. SpaceMouseOutput, SpaceMouseOutputRecord, + # Gamepad types. + GamepadInputState, + GamepadSnapshot, + GamepadSnapshotRecord, # Keyboard types (held keys plus ordered key events). KeyAction, KeyEvent, @@ -172,6 +176,10 @@ def __getattr__(name: str): # SpaceMouse types. "SpaceMouseOutput", "SpaceMouseOutputRecord", + # Gamepad types. + "GamepadInputState", + "GamepadSnapshot", + "GamepadSnapshotRecord", # Keyboard types (held keys plus ordered key events). "KeyAction", "KeyEvent", diff --git a/src/python/isaaccapture/teleop_session_manager/teleop_session.py b/src/python/isaaccapture/teleop_session_manager/teleop_session.py index 9645931141..d4bf267b68 100644 --- a/src/python/isaaccapture/teleop_session_manager/teleop_session.py +++ b/src/python/isaaccapture/teleop_session_manager/teleop_session.py @@ -62,7 +62,7 @@ logger = logging.getLogger(__name__) # Plugins that publish device status (see PluginDeviceStatusPublisher); others launch unmonitored. -_MONITORED_PLUGIN_NAMES = frozenset({"manus_hand_plugin", "spacemouse"}) +_MONITORED_PLUGIN_NAMES = frozenset({"manus_hand_plugin", "spacemouse", "gamepad"}) def _resolve_sink(entry: GraphExecutable) -> tuple[GraphExecutable, IDeviceIOSink]: diff --git a/tests/cpp/core/schema/CMakeLists.txt b/tests/cpp/core/schema/CMakeLists.txt index 9d66c6a949..3fa2548b78 100644 --- a/tests/cpp/core/schema/CMakeLists.txt +++ b/tests/cpp/core/schema/CMakeLists.txt @@ -13,6 +13,7 @@ add_executable(schema_tests test_pedals.cpp test_steering_wheel.cpp test_spacemouse.cpp + test_gamepad.cpp test_plugin_device_status.cpp test_joint_se3_pose.cpp test_se3_tracker.cpp diff --git a/tests/cpp/core/schema/test_gamepad.cpp b/tests/cpp/core/schema/test_gamepad.cpp new file mode 100644 index 0000000000..a9c11b28ee --- /dev/null +++ b/tests/cpp/core/schema/test_gamepad.cpp @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Unit tests for the generated Gamepad FlatBuffer types. + +#include +#include +#include +#include + +#include + +#define VT(field) (field + 2) * 2 + +static_assert(core::GamepadSnapshot::VT_INPUTS == VT(0)); + +static_assert(core::GamepadSnapshotRecord::VT_DATA == VT(0)); +static_assert(core::GamepadSnapshotRecord::VT_TIMESTAMP == VT(1)); + +// 14 bools padded to the floats' alignment, then 6 floats. +static_assert(sizeof(core::GamepadInputState) == 16 + 6 * 4); + +TEST_CASE("GamepadSnapshotT default construction", "[gamepad][native]") +{ + core::GamepadSnapshotT snapshot; + + CHECK(snapshot.inputs == nullptr); +} + +TEST_CASE("GamepadSnapshot serialization and deserialization", "[gamepad][serialize]") +{ + flatbuffers::FlatBufferBuilder builder; + + core::GamepadSnapshotT snapshot; + core::GamepadInputState inputs; + inputs.mutate_west_click(true); + inputs.mutate_dpad_up_click(true); + inputs.mutate_thumbstick_left_y(1.0f); + inputs.mutate_trigger_right_value(0.5f); + snapshot.inputs = std::make_shared(inputs); + + builder.Finish(core::GamepadSnapshot::Pack(builder, &snapshot)); + + const auto* deserialized = flatbuffers::GetRoot(builder.GetBufferPointer()); + REQUIRE(deserialized->inputs() != nullptr); + CHECK(deserialized->inputs()->west_click()); + CHECK(deserialized->inputs()->dpad_up_click()); + CHECK_FALSE(deserialized->inputs()->south_click()); + CHECK(deserialized->inputs()->thumbstick_left_y() == Catch::Approx(1.0f)); + CHECK(deserialized->inputs()->trigger_right_value() == Catch::Approx(0.5f)); +} diff --git a/tests/cpp/plugins/CMakeLists.txt b/tests/cpp/plugins/CMakeLists.txt index 454bc58ff0..1fd14aa203 100644 --- a/tests/cpp/plugins/CMakeLists.txt +++ b/tests/cpp/plugins/CMakeLists.txt @@ -3,9 +3,10 @@ add_subdirectory(plugin_utils) -# The SpaceMouse plugin (and its spacemouse_core library) is Linux only. +# The SpaceMouse and gamepad plugins (and their core libraries) are Linux only. if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_subdirectory(spacemouse) + add_subdirectory(gamepad) endif() if(BUILD_PLUGIN_OGLO) diff --git a/tests/cpp/plugins/gamepad/CMakeLists.txt b/tests/cpp/plugins/gamepad/CMakeLists.txt new file mode 100644 index 0000000000..b1a3931d5f --- /dev/null +++ b/tests/cpp/plugins/gamepad/CMakeLists.txt @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +add_executable(gamepad_plugin_tests + test_gamepad_connection.cpp +) + +target_link_libraries(gamepad_plugin_tests + PRIVATE + Catch2::Catch2WithMain + gamepad::gamepad_core + SDL3::SDL3-static +) + +catch_discover_tests(gamepad_plugin_tests) diff --git a/tests/cpp/plugins/gamepad/test_gamepad_connection.cpp b/tests/cpp/plugins/gamepad/test_gamepad_connection.cpp new file mode 100644 index 0000000000..628bcaf45f --- /dev/null +++ b/tests/cpp/plugins/gamepad/test_gamepad_connection.cpp @@ -0,0 +1,164 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// The connection reads SDL virtual gamepads: attaching one is a pad being plugged in, setting its +// buttons and axes is input, and detaching it is an unplug. The state is read back through +// GamepadInputState's named fields, so the plugin's SDL-to-schema mapping is checked end to end. Real +// pads are hidden from SDL so a pad on the test machine does not count. + +#include +#include +#include +#include + +#include +#include + +using plugins::gamepad::ConnectionStatus; +using plugins::gamepad::GamepadConnection; +using plugins::gamepad::GamepadConnectionConfig; + +namespace +{ + +// pid.codes test VID; each pad gets its own product ID, which SDL reports per device. +constexpr uint16_t kVirtualVendor = 0x1209; + +//! A connection that sees only the virtual pads this file attaches. +std::unique_ptr make_connection(int player = 1) +{ + SDL_SetHint(SDL_HINT_GAMECONTROLLER_IGNORE_DEVICES_EXCEPT, "0x1209/0x0001,0x1209/0x0002"); + return std::make_unique(GamepadConnectionConfig{ player }); +} + +//! A virtual gamepad with every standard button and axis, in SDL_GamepadButton/SDL_GamepadAxis order. +class VirtualPad +{ +public: + explicit VirtualPad(const char* name, uint16_t product_id = 0x0001) + { + SDL_VirtualJoystickDesc desc; + SDL_INIT_INTERFACE(&desc); + desc.type = SDL_JOYSTICK_TYPE_GAMEPAD; + desc.vendor_id = kVirtualVendor; + desc.product_id = product_id; + desc.naxes = SDL_GAMEPAD_AXIS_COUNT; + desc.nbuttons = SDL_GAMEPAD_BUTTON_COUNT; + desc.name = name; + id_ = SDL_AttachVirtualJoystick(&desc); + REQUIRE(id_ != 0); + joystick_ = SDL_OpenJoystick(id_); + REQUIRE(joystick_ != nullptr); + } + ~VirtualPad() + { + unplug(); + } + void press(SDL_GamepadButton button, bool down = true) + { + REQUIRE(SDL_SetJoystickVirtualButton(joystick_, button, down)); + } + void move(SDL_GamepadAxis axis, int16_t value) + { + REQUIRE(SDL_SetJoystickVirtualAxis(joystick_, axis, value)); + } + void unplug() + { + if (joystick_ != nullptr) + SDL_CloseJoystick(joystick_); + if (id_ != 0) + SDL_DetachVirtualJoystick(id_); + joystick_ = nullptr; + id_ = 0; + } + +private: + SDL_JoystickID id_ = 0; + SDL_Joystick* joystick_ = nullptr; +}; + +} // namespace + +TEST_CASE("No gamepad connected: disconnected at rest", "[gamepad][connection]") +{ + auto connection = make_connection(); + + connection->poll(); + + CHECK(connection->status() == ConnectionStatus::NoDevice); + CHECK(connection->description().empty()); + CHECK(connection->state().inputs.thumbstick_left_x() == 0.0f); +} + +TEST_CASE("A connected gamepad is read in the standard layout until it is unplugged", "[gamepad][connection]") +{ + auto connection = make_connection(); + VirtualPad pad("Test Pad"); + + connection->poll(); + REQUIRE(connection->status() == ConnectionStatus::Connected); + CHECK(connection->description().find("Test Pad") != std::string::npos); + CHECK(connection->description().find("1209:0001") != std::string::npos); + + pad.press(SDL_GAMEPAD_BUTTON_WEST); + pad.press(SDL_GAMEPAD_BUTTON_DPAD_UP); + pad.move(SDL_GAMEPAD_AXIS_LEFTY, -32768); // SDL: stick fully up + pad.move(SDL_GAMEPAD_AXIS_RIGHTX, 16384); + pad.move(SDL_GAMEPAD_AXIS_RIGHT_TRIGGER, 32767); + connection->poll(); + + const auto& state = connection->state(); + CHECK(state.inputs.west_click()); + CHECK(state.inputs.dpad_up_click()); + CHECK_FALSE(state.inputs.south_click()); + CHECK(state.inputs.thumbstick_left_y() == Catch::Approx(1.0f)); // up-positive + CHECK(state.inputs.thumbstick_right_x() == Catch::Approx(0.5f).margin(1e-4)); + CHECK(state.inputs.trigger_right_value() == Catch::Approx(1.0f)); + CHECK(state.inputs.trigger_left_value() == 0.0f); + + pad.press(SDL_GAMEPAD_BUTTON_WEST, false); + connection->poll(); + CHECK_FALSE(connection->state().inputs.west_click()); + + pad.unplug(); + connection->poll(); + CHECK(connection->status() == ConnectionStatus::NoDevice); + CHECK_FALSE(connection->state().inputs.dpad_up_click()); + CHECK(connection->state().inputs.thumbstick_left_y() == 0.0f); +} + +TEST_CASE("A gamepad plugged in later is picked up", "[gamepad][connection]") +{ + auto connection = make_connection(); + connection->poll(); + REQUIRE(connection->status() == ConnectionStatus::NoDevice); + + VirtualPad pad("Late Pad"); + connection->poll(); + + CHECK(connection->status() == ConnectionStatus::Connected); +} + +TEST_CASE("The player number selects the Nth connected gamepad and keeps it", "[gamepad][connection]") +{ + auto connection = make_connection(2); + VirtualPad first("First Pad", 0x0001); + connection->poll(); + CHECK(connection->status() == ConnectionStatus::NoDevice); + + VirtualPad second("Second Pad", 0x0002); + connection->poll(); + REQUIRE(connection->status() == ConnectionStatus::Connected); + CHECK(connection->description().find("1209:0002") != std::string::npos); + + // Player 1 leaving does not move player 2 onto another pad. + first.unplug(); + connection->poll(); + CHECK(connection->status() == ConnectionStatus::Connected); + CHECK(connection->description().find("1209:0002") != std::string::npos); +} + +TEST_CASE("A player number below 1 is rejected", "[gamepad][connection]") +{ + CHECK_THROWS_AS(GamepadConnection(GamepadConnectionConfig{ 0 }), std::invalid_argument); +} diff --git a/tests/python/core/retargeting_engine/test_gamepad_retargeter.py b/tests/python/core/retargeting_engine/test_gamepad_retargeter.py new file mode 100644 index 0000000000..184fe3e3de --- /dev/null +++ b/tests/python/core/retargeting_engine/test_gamepad_retargeter.py @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the gamepad retargeters, fed through GamepadSource from real GamepadSnapshot payloads.""" + +import numpy as np +import pytest + +from isaaccapture.retargeters import ( + GamepadGripperRetargeter, + GamepadToSe2Retargeter, + GamepadToSe2RetargeterConfig, + GamepadToSe3RelRetargeter, + GamepadToSe3RelRetargeterConfig, +) +from isaaccapture.retargeting_engine.deviceio_source_nodes import GamepadSource +from isaaccapture.retargeting_engine.interface.base_retargeter import _make_output_group +from isaaccapture.retargeting_engine.interface.execution_events import ExecutionEvents +from isaaccapture.retargeting_engine.interface.retargeter_core_types import ( + ComputeContext, +) +from isaaccapture.retargeting_engine.interface.tensor_group import TensorGroup +from isaaccapture.schema import GamepadInputState, GamepadSnapshot + + +def _gamepad(absent=False, **fields): + """GamepadSource's "gamepad" output for a pad in this state, or for no pad when absent.""" + src = GamepadSource(name="gamepad") + tg = TensorGroup(src.input_spec()["deviceio_gamepad"]) + tg[0] = None if absent else GamepadSnapshot(GamepadInputState(**fields)) + outputs = {name: _make_output_group(gt) for name, gt in src.output_spec().items()} + src.compute({"deviceio_gamepad": tg}, outputs) + return outputs["gamepad"] + + +def _run(retargeter, output, gamepad, reset=False): + out = {output: _make_output_group(retargeter.output_spec()[output])} + context = ComputeContext(execution_events=ExecutionEvents(reset=reset)) + retargeter.compute({"gamepad": gamepad}, out, context) + return np.asarray(out[output][0]) + + +def _se3(): + return GamepadToSe3RelRetargeter(GamepadToSe3RelRetargeterConfig(), name="se3") + + +def _se2(): + return GamepadToSe2Retargeter(GamepadToSe2RetargeterConfig(), name="se2") + + +class TestGamepadToSe3RelRetargeter: + def test_left_stick_up_produces_forward_delta(self): + delta = _run(_se3(), "ee_delta", _gamepad(thumbstick_left_y=1.0)) + + assert delta[0] == pytest.approx(0.4) # default pos_sensitivity + assert np.allclose(delta[1:], 0.0) + + def test_sticks_combine_on_independent_axes(self): + delta = _run( + _se3(), "ee_delta", _gamepad(thumbstick_left_y=1.0, thumbstick_right_y=1.0) + ) + + assert delta[0] == pytest.approx(0.4) # left stick up: +X + assert delta[2] == pytest.approx(0.4) # right stick up: +Z + assert delta[1] == pytest.approx(0.0) + assert np.allclose(delta[3:], 0.0) + + @pytest.mark.parametrize("direction", [1.0, -1.0]) + def test_stick_right_is_positive_y_and_yaw(self, direction): + delta = _run( + _se3(), + "ee_delta", + _gamepad(thumbstick_left_x=direction, thumbstick_right_x=direction), + ) + + assert np.sign(delta[1]) == direction + assert np.sign(delta[5]) == direction + assert np.allclose(delta[[0, 2, 3, 4]], 0.0) + + def test_dpad_rotates_roll_and_pitch(self): + delta = _run( + _se3(), "ee_delta", _gamepad(dpad_left_click=True, dpad_down_click=True) + ) + + assert np.allclose(delta[:3], 0.0) + assert delta[3] > 0 and delta[4] > 0 + + def test_absent_pad_yields_zero_delta(self): + assert np.allclose(_run(_se3(), "ee_delta", _gamepad(absent=True)), 0.0) + + def test_dead_zone_suppresses_stick_drift(self): + assert np.allclose( + _run(_se3(), "ee_delta", _gamepad(thumbstick_left_y=0.1)), 0.0 + ) + + +class TestGamepadGripperRetargeter: + @staticmethod + def _step(gripper, reset=False, **fields): + return float(_run(gripper, "gripper_command", _gamepad(**fields), reset=reset)) + + def test_gripper_toggles_on_west_rising_edge_only(self): + gripper = GamepadGripperRetargeter(name="gripper") + + assert self._step(gripper) == pytest.approx(1.0) # open (default) + assert self._step(gripper, west_click=True) == pytest.approx(-1.0) # close + assert self._step(gripper, west_click=True) == pytest.approx(-1.0) # held + assert self._step(gripper) == pytest.approx(-1.0) # released: stays closed + assert self._step(gripper, west_click=True) == pytest.approx(1.0) # open + + def test_absent_pad_yields_default_open(self): + gripper = GamepadGripperRetargeter(name="gripper") + + assert self._step(gripper, absent=True) == pytest.approx(1.0) + + def test_reset_does_not_toggle_while_west_is_held(self): + gripper = GamepadGripperRetargeter(name="gripper") + + assert self._step(gripper, west_click=True) == pytest.approx(-1.0) + # Reset reopens the gripper; west still held is not a new press. + assert self._step(gripper, reset=True, west_click=True) == pytest.approx(1.0) + assert self._step(gripper, west_click=True) == pytest.approx(1.0) + assert self._step(gripper) == pytest.approx(1.0) + assert self._step(gripper, west_click=True) == pytest.approx(-1.0) + + def test_reset_with_absent_pad_keeps_the_edge_state(self): + gripper = GamepadGripperRetargeter(name="gripper") + + assert self._step(gripper, west_click=True) == pytest.approx(-1.0) + assert self._step(gripper, reset=True, absent=True) == pytest.approx(1.0) + # West was held throughout, so it is not read as a new press when the pad returns. + assert self._step(gripper, west_click=True) == pytest.approx(1.0) + + def test_other_buttons_do_not_affect_the_gripper(self): + gripper = GamepadGripperRetargeter(name="gripper") + + assert self._step( + gripper, shoulder_right_click=True, south_click=True + ) == pytest.approx(1.0) + + +class TestGamepadToSe2Retargeter: + def test_left_and_right_stick_combine(self): + velocity = _run( + _se2(), + "base_command", + _gamepad(thumbstick_left_y=1.0, thumbstick_right_x=0.5), + ) + + assert velocity[0] == pytest.approx(1.0) # default v_x_sensitivity + assert velocity[1] == pytest.approx(0.0) + assert velocity[2] == pytest.approx(0.5) # default omega_z_sensitivity + + def test_dead_zone_suppresses_stick_drift(self): + assert np.allclose( + _run(_se2(), "base_command", _gamepad(thumbstick_left_y=0.1)), 0.0 + ) + + def test_absent_pad_yields_zero_velocity(self): + assert np.allclose(_run(_se2(), "base_command", _gamepad(absent=True)), 0.0) diff --git a/tests/python/core/retargeting_engine/test_gamepad_source.py b/tests/python/core/retargeting_engine/test_gamepad_source.py new file mode 100644 index 0000000000..55c61a0d57 --- /dev/null +++ b/tests/python/core/retargeting_engine/test_gamepad_source.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the GamepadSource DeviceIO converter. + +Feeds GamepadSnapshot payloads built through the real schema bindings into the converter, with no +plugin or OpenXR runtime involved. +""" + +import pytest + +from isaaccapture.retargeting_engine.deviceio_source_nodes import GamepadSource +from isaaccapture.retargeting_engine.interface.base_retargeter import _make_output_group +from isaaccapture.retargeting_engine.interface.tensor_group import TensorGroup +from isaaccapture.retargeting_engine.tensor_types import GamepadInput, GamepadInputIndex +from isaaccapture.schema import GamepadInputState, GamepadSnapshot + + +def _run(state): + src = GamepadSource(name="gamepad") + tg = TensorGroup(src.input_spec()["deviceio_gamepad"]) + tg[0] = state + outputs = {name: _make_output_group(gt) for name, gt in src.output_spec().items()} + src.compute({"deviceio_gamepad": tg}, outputs) + return outputs["gamepad"] + + +def test_source_reads_the_generated_tracker(): + tracker = GamepadSource(name="gamepad").get_tracker() + assert tracker.get_name() == "GamepadTracker" + + +def test_the_group_has_one_field_per_schema_field(): + assert len(GamepadInput().types) == 20 + assert len(GamepadInputIndex) == 20 + + +def test_fields_pass_through_by_name(): + inputs = GamepadInputState( + west_click=True, + dpad_up_click=True, + thumbstick_left_y=1.0, + trigger_right_value=0.5, + ) + gamepad = _run(GamepadSnapshot(inputs)) + + held = { + i.name + for i in GamepadInputIndex + if i.name.endswith("_CLICK") and gamepad[i] > 0.5 + } + assert held == {"WEST_CLICK", "DPAD_UP_CLICK"} + assert float(gamepad[GamepadInputIndex.THUMBSTICK_LEFT_Y]) == pytest.approx(1.0) + assert float(gamepad[GamepadInputIndex.TRIGGER_RIGHT_VALUE]) == pytest.approx(0.5) + assert float(gamepad[GamepadInputIndex.THUMBSTICK_RIGHT_X]) == 0.0 + + +@pytest.mark.parametrize( + "state", + [None, GamepadSnapshot()], + ids=["plugin-not-pushing", "no-device-open"], +) +def test_no_device_yields_absent_output(state): + assert _run(state).is_none diff --git a/tests/python/core/schema/test_gamepad.py b/tests/python/core/schema/test_gamepad.py new file mode 100644 index 0000000000..f22fc114e9 --- /dev/null +++ b/tests/python/core/schema/test_gamepad.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the gamepad types in isaaccapture.schema. + +- GamepadInputState: the buttons and axes every standard gamepad has +- GamepadSnapshot: the input state, absent while no pad is open +- GamepadSnapshotRecord: record wrapper carrying DeviceDataTimestamp +""" + +import pytest + +from isaaccapture.schema import ( + DeviceDataTimestamp, + GamepadInputState, + GamepadSnapshot, + GamepadSnapshotRecord, +) + + +class TestGamepadInputState: + def test_default_is_released_and_centered(self): + inputs = GamepadInputState() + + assert not inputs.south_click + assert not inputs.dpad_right_click + assert inputs.thumbstick_left_x == 0.0 + assert inputs.trigger_right_value == 0.0 + + def test_fields_by_name(self): + inputs = GamepadInputState( + west_click=True, thumbstick_left_y=1.0, trigger_right_value=0.5 + ) + + assert inputs.west_click + assert not inputs.east_click + assert inputs.thumbstick_left_y == pytest.approx(1.0) + assert inputs.trigger_right_value == pytest.approx(0.5) + + def test_repr_names_the_held_buttons(self): + repr_str = repr(GamepadInputState(south_click=True, dpad_up_click=True)) + + assert "held=[south, dpad_up]" in repr_str + + +class TestGamepadSnapshot: + def test_default_is_no_pad(self): + snapshot = GamepadSnapshot() + + assert snapshot.inputs is None + assert "inputs=None" in repr(snapshot) + + def test_fields_round_trip(self): + snapshot = GamepadSnapshot(GamepadInputState(north_click=True)) + + assert snapshot.inputs.north_click + + +class TestGamepadSnapshotRecord: + def test_record_carries_data_and_timestamp(self): + data = GamepadSnapshot(GamepadInputState(south_click=True)) + record = GamepadSnapshotRecord(data, DeviceDataTimestamp(1, 2, 3)) + + assert record.timestamp.available_time_local_common_clock == 1 + assert record.timestamp.sample_time_local_common_clock == 2 + assert record.timestamp.sample_time_raw_device_clock == 3 + assert record.data.inputs.south_click + + def test_payload_less_record(self): + record = GamepadSnapshotRecord(None, DeviceDataTimestamp(1, 2, 3)) + + assert record.data is None diff --git a/tests/python/core/teleop_session_manager/test_teleop_session.py b/tests/python/core/teleop_session_manager/test_teleop_session.py index 2f07777ba7..919e79de11 100644 --- a/tests/python/core/teleop_session_manager/test_teleop_session.py +++ b/tests/python/core/teleop_session_manager/test_teleop_session.py @@ -1563,15 +1563,14 @@ def test_plugin_args_default_empty(self, tmp_path): class TestStatusMonitoringIntegration: """Test monitored-plugin inventory, refresh, and provider failure integration.""" - def test_spacemouse_plugin_is_monitored(self, tmp_path): - manager = MockPluginManager(plugin_names=["spacemouse"]) + @pytest.mark.parametrize("plugin", ["spacemouse", "gamepad"]) + def test_device_plugin_is_monitored(self, tmp_path, plugin): + manager = MockPluginManager(plugin_names=[plugin]) manager.get_plugin_info = MagicMock( return_value=SimpleNamespace( - name="spacemouse", + name=plugin, devices=( - SimpleNamespace( - path="/spacemouse", type="spacemouse", description="SpaceMouse" - ), + SimpleNamespace(path=f"/{plugin}", type=plugin, description=plugin), ), ) ) @@ -1579,8 +1578,8 @@ def test_spacemouse_plugin_is_monitored(self, tmp_path): MockPipeline(leaf_nodes=[]), plugins=[ PluginConfig( - plugin_name="spacemouse", - plugin_root_id="spacemouse", + plugin_name=plugin, + plugin_root_id=plugin, search_paths=[tmp_path], ) ], @@ -1593,8 +1592,8 @@ def test_spacemouse_plugin_is_monitored(self, tmp_path): mock_session_dependencies(mock_pm=manager, collected_trackers=[]), ): with TeleopSession(config) as session: - tracker_cls.assert_called_once_with("spacemouse/device_status") - assert session.get_device_status("spacemouse/spacemouse") is not None + tracker_cls.assert_called_once_with(f"{plugin}/device_status") + assert session.get_device_status(f"{plugin}/{plugin}") is not None def test_only_manus_is_monitored_and_plugin_launch_order_is_preserved( self, tmp_path diff --git a/tests/python/packaging/test_wheel_contents.py b/tests/python/packaging/test_wheel_contents.py index 2d3e7eae4a..33c8c0674f 100644 --- a/tests/python/packaging/test_wheel_contents.py +++ b/tests/python/packaging/test_wheel_contents.py @@ -134,3 +134,18 @@ def test_capture_license_ships_in_distribution_metadata() -> None: license_path = metadata_path.removesuffix("METADATA") + "licenses/LICENSE.md" assert archive.read(license_path) == (repo_root() / "LICENSE.md").read_bytes() assert not any("isaaccapture_egopost/" in name for name in archive.namelist()) + + +@pytest.mark.skipif( + not sys.platform.startswith("linux"), reason="the plugin is Linux only" +) +def test_gamepad_plugin_ships_runnable_for_teleop_session() -> None: + """Apps start the bundled plugin from BUNDLED_PLUGINS_DIR, so it needs its manifest and exec bit.""" + bundle = f"{DIST}/plugins/gamepad" + with zipfile.ZipFile(_wheel()) as archive: + names = set(archive.namelist()) + binary = archive.getinfo(f"{bundle}/gamepad_plugin") + assert f"{bundle}/plugin.yaml" in names + assert (binary.external_attr >> 16) & 0o111, ( + "gamepad_plugin lost its executable bit" + )