From cfa02b9d48af71557f8429ac7b1385b95e7211fb Mon Sep 17 00:00:00 2001 From: Rafael Wiltz Date: Tue, 13 Oct 2026 10:38:09 -0400 Subject: [PATCH 1/4] Add a gamepad schema and manifest tracker GamepadInputState holds the inputs every standard gamepad has, the same set as OpenXR's xbox_controller profile, as named fields in the style of ControllerInputState: a generic controller split in two. Face buttons are named by position (south_click is A on an Xbox pad and Cross on a PlayStation pad), the thumbsticks are in [-1, 1] with right and up positive, and the triggers are in [0, 1]. GamepadSnapshot carries it, absent while the plugin has no pad open. GamepadTracker is generated from its trackers.toml entry, so it records and replays like the other schema-based trackers. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Rafael Wiltz --- docs/source/references/generated_trackers.rst | 3 + src/core/deviceio_trackers/trackers.toml | 5 + src/core/schema/fbs/gamepad.fbs | 55 +++++++++ src/core/schema/golden/gamepad.bfbs | Bin 0 -> 2112 bytes src/core/schema/python/CMakeLists.txt | 1 + src/core/schema/python/gamepad_bindings.h | 107 ++++++++++++++++++ src/core/schema/python/schema_module.cpp | 4 + src/python/isaaccapture/schema/__init__.py | 8 ++ tests/cpp/core/schema/CMakeLists.txt | 1 + tests/cpp/core/schema/test_gamepad.cpp | 51 +++++++++ tests/python/core/schema/test_gamepad.py | 72 ++++++++++++ 11 files changed, 307 insertions(+) create mode 100644 src/core/schema/fbs/gamepad.fbs create mode 100644 src/core/schema/golden/gamepad.bfbs create mode 100644 src/core/schema/python/gamepad_bindings.h create mode 100644 tests/cpp/core/schema/test_gamepad.cpp create mode 100644 tests/python/core/schema/test_gamepad.py diff --git a/docs/source/references/generated_trackers.rst b/docs/source/references/generated_trackers.rst index 0ccd8e0249..a2d13bec6a 100644 --- a/docs/source/references/generated_trackers.rst +++ b/docs/source/references/generated_trackers.rst @@ -116,6 +116,9 @@ Generated today * - ``spacemouse`` - ``SpaceMouseTracker`` - Overrides ``class`` and ``traits``; the defaults would spell ``Spacemouse`` + * - ``gamepad`` + - ``GamepadTracker`` + - Gamepad state pushed by the gamepad plugin (SDL3); only ``python_accessor`` is overridden Still hand-written ------------------ diff --git a/src/core/deviceio_trackers/trackers.toml b/src/core/deviceio_trackers/trackers.toml index 549561a285..d27518d43c 100644 --- a/src/core/deviceio_trackers/trackers.toml +++ b/src/core/deviceio_trackers/trackers.toml @@ -106,3 +106,8 @@ class = "SpaceMouseTracker" traits = "SpaceMouseRecordingTraits" max_flatbuffer_size = 256 python_accessor = "get_spacemouse_data" + +[[tracker]] +name = "gamepad" +table = "GamepadSnapshot" +python_accessor = "get_gamepad_data" diff --git a/src/core/schema/fbs/gamepad.fbs b/src/core/schema/fbs/gamepad.fbs new file mode 100644 index 0000000000..d83a4b6a79 --- /dev/null +++ b/src/core/schema/fbs/gamepad.fbs @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +include "timestamp.fbs"; + +namespace core; + +// Inputs every standard gamepad has, the same set as OpenXR's xbox_controller profile: a generic +// controller split in two (see ControllerInputState). Face buttons are named by position, so they +// mean the same on every pad: south is A on an Xbox pad, Cross on a PlayStation pad and B on a +// Nintendo pad. Pad-specific extras (guide, paddles, touchpad) are left out so every source can +// fill every field. +struct GamepadInputState { + // Button states. `*_click` is OpenXR's input component name for a button's boolean state, as in + // ControllerInputState: true for as long as the button is held, not a press-and-release event. + south_click: bool; + east_click: bool; + west_click: bool; + north_click: bool; + view_click: bool; + menu_click: bool; + shoulder_left_click: bool; + shoulder_right_click: bool; + thumbstick_left_click: bool; + thumbstick_right_click: bool; + dpad_up_click: bool; + dpad_down_click: bool; + dpad_left_click: bool; + dpad_right_click: bool; + + // Thumbsticks in [-1, 1] with right and up positive, as OpenXR's thumbstick x and y are. + thumbstick_left_x: float32; + thumbstick_left_y: float32; + thumbstick_right_x: float32; + thumbstick_right_y: float32; + + // Triggers in [0, 1]. + trigger_left_value: float32; + trigger_right_value: float32; +} + +// A gamepad, pushed by the gamepad plugin in the layout above, whatever the pad's own numbering. +table GamepadSnapshot { + // Absent while the plugin has no pad open, so a reader never keeps the last input of a pad that + // went away. + inputs: GamepadInputState (id: 0); +} + +// MCAP recording wrapper for GamepadSnapshot. +table GamepadSnapshotRecord { + data: GamepadSnapshot (id: 0); + timestamp: DeviceDataTimestamp (id: 1); +} + +root_type GamepadSnapshotRecord; diff --git a/src/core/schema/golden/gamepad.bfbs b/src/core/schema/golden/gamepad.bfbs new file mode 100644 index 0000000000000000000000000000000000000000..2b8f1874e5f2e496dd49767382366f7c278bfd87 GIT binary patch literal 2112 zcmai#zi$*r6vy8k_W5idu>pfnWJQid;R0a_3KtX>vOshwP8sP~-!8sW_I~JY&vqmi zDN>}MNRh$?1qFYE3zrlW6cj03@F!5Fh!n!|`R?qjmz7cG^lo0BlLR-+cP-i%f?mPoXg86$;)&SPbS$UhA?GRx#aae|M7fIzQ*8 zexY8$EK>>Ix8wiAP_%!6SpwlPZ%ddf^6lJC<{oCO`7uqu;S_K2?asHAoi9s9)1Pk^ z-$gnO?K^O)gGOEx4(a|E;A&jvS1;=NCN|luE6*a;P)L8}(RX4p!Ir&3u7z&j(@!cm zHByGlqH(Y@WEO3)!2I?#OgKCkv>m4PUz#q^!`{wG1*2Nc>`X^Ff?5PKPX~LD_ET&chocvi^47P`8TqS(tY%@ zf>q5I*pk)T^u?^r7B$qAEnw>!grf2bD9G%5#rkL1HTs`G_wS5K~Iigko literal 0 HcmV?d00001 diff --git a/src/core/schema/python/CMakeLists.txt b/src/core/schema/python/CMakeLists.txt index d8fc43bfd8..cee33a811b 100644 --- a/src/core/schema/python/CMakeLists.txt +++ b/src/core/schema/python/CMakeLists.txt @@ -5,6 +5,7 @@ pybind11_add_module(schema_py oak_bindings.h controller_bindings.h full_body_bindings.h + gamepad_bindings.h hand_bindings.h haptic_command_bindings.h head_bindings.h diff --git a/src/core/schema/python/gamepad_bindings.h b/src/core/schema/python/gamepad_bindings.h new file mode 100644 index 0000000000..418b5ceea0 --- /dev/null +++ b/src/core/schema/python/gamepad_bindings.h @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Python bindings for the Gamepad FlatBuffer schema. +// GamepadInputState is a struct; GamepadSnapshot is a table, exposed as an encoded view. + +#pragma once + +#include "schema_serialized.h" + +#include +#include +#include + +#include +#include +#include + +namespace py = pybind11; + +namespace core +{ + +inline void bind_gamepad(py::module& m) +{ + py::class_(m, "GamepadInputState") + .def(py::init(), + py::arg("south_click") = false, py::arg("east_click") = false, py::arg("west_click") = false, + py::arg("north_click") = false, py::arg("view_click") = false, py::arg("menu_click") = false, + py::arg("shoulder_left_click") = false, py::arg("shoulder_right_click") = false, + py::arg("thumbstick_left_click") = false, py::arg("thumbstick_right_click") = false, + py::arg("dpad_up_click") = false, py::arg("dpad_down_click") = false, py::arg("dpad_left_click") = false, + py::arg("dpad_right_click") = false, py::arg("thumbstick_left_x") = 0.0f, + py::arg("thumbstick_left_y") = 0.0f, py::arg("thumbstick_right_x") = 0.0f, + py::arg("thumbstick_right_y") = 0.0f, py::arg("trigger_left_value") = 0.0f, + py::arg("trigger_right_value") = 0.0f, + "Gamepad inputs; omitted buttons are released and omitted axes centered.") + .def_property_readonly("south_click", &GamepadInputState::south_click) + .def_property_readonly("east_click", &GamepadInputState::east_click) + .def_property_readonly("west_click", &GamepadInputState::west_click) + .def_property_readonly("north_click", &GamepadInputState::north_click) + .def_property_readonly("view_click", &GamepadInputState::view_click) + .def_property_readonly("menu_click", &GamepadInputState::menu_click) + .def_property_readonly("shoulder_left_click", &GamepadInputState::shoulder_left_click) + .def_property_readonly("shoulder_right_click", &GamepadInputState::shoulder_right_click) + .def_property_readonly("thumbstick_left_click", &GamepadInputState::thumbstick_left_click) + .def_property_readonly("thumbstick_right_click", &GamepadInputState::thumbstick_right_click) + .def_property_readonly("dpad_up_click", &GamepadInputState::dpad_up_click) + .def_property_readonly("dpad_down_click", &GamepadInputState::dpad_down_click) + .def_property_readonly("dpad_left_click", &GamepadInputState::dpad_left_click) + .def_property_readonly("dpad_right_click", &GamepadInputState::dpad_right_click) + .def_property_readonly("thumbstick_left_x", &GamepadInputState::thumbstick_left_x) + .def_property_readonly("thumbstick_left_y", &GamepadInputState::thumbstick_left_y) + .def_property_readonly("thumbstick_right_x", &GamepadInputState::thumbstick_right_x) + .def_property_readonly("thumbstick_right_y", &GamepadInputState::thumbstick_right_y) + .def_property_readonly("trigger_left_value", &GamepadInputState::trigger_left_value) + .def_property_readonly("trigger_right_value", &GamepadInputState::trigger_right_value) + .def("__repr__", + [](const GamepadInputState& self) + { + std::string held; + const auto add_held = [&held](const char* name, bool pressed) + { + if (pressed) + held += (held.empty() ? "" : ", ") + std::string(name); + }; + add_held("south", self.south_click()); + add_held("east", self.east_click()); + add_held("west", self.west_click()); + add_held("north", self.north_click()); + add_held("view", self.view_click()); + add_held("menu", self.menu_click()); + add_held("shoulder_left", self.shoulder_left_click()); + add_held("shoulder_right", self.shoulder_right_click()); + add_held("thumbstick_left", self.thumbstick_left_click()); + add_held("thumbstick_right", self.thumbstick_right_click()); + add_held("dpad_up", self.dpad_up_click()); + add_held("dpad_down", self.dpad_down_click()); + add_held("dpad_left", self.dpad_left_click()); + add_held("dpad_right", self.dpad_right_click()); + return "GamepadInputState(held=[" + held + "], left=(" + std::to_string(self.thumbstick_left_x()) + + ", " + std::to_string(self.thumbstick_left_y()) + "), right=(" + + std::to_string(self.thumbstick_right_x()) + ", " + std::to_string(self.thumbstick_right_y()) + + "), triggers=(" + std::to_string(self.trigger_left_value()) + ", " + + std::to_string(self.trigger_right_value()) + "))"; + }); + + serialized_class( + m, "GamepadSnapshot", "Encoded gamepad snapshot: the input state, absent while no pad is open.") + .def(py::init( + [](std::optional inputs) + { + GamepadSnapshotT native; + if (inputs) + native.inputs = std::make_shared(*inputs); + return pack(native); + }), + py::arg("inputs") = std::nullopt, "Encode a gamepad snapshot. Without inputs it reads as no pad open.") + .def_property_readonly("inputs", field(&GamepadSnapshot::inputs), py::return_value_policy::reference_internal) + .def("__repr__", [](const Serialized& self) + { return "GamepadSnapshot(inputs=" + std::string(self->inputs() != nullptr ? "..." : "None") + ")"; }); + + bind_record(m, "GamepadSnapshotRecord", "GamepadSnapshot"); +} + +} // namespace core diff --git a/src/core/schema/python/schema_module.cpp b/src/core/schema/python/schema_module.cpp index c76db6cc8a..80d32c9fca 100644 --- a/src/core/schema/python/schema_module.cpp +++ b/src/core/schema/python/schema_module.cpp @@ -8,6 +8,7 @@ // Include binding definitions. #include "controller_bindings.h" #include "full_body_bindings.h" +#include "gamepad_bindings.h" #include "hand_bindings.h" #include "haptic_command_bindings.h" #include "head_bindings.h" @@ -76,6 +77,9 @@ PYBIND11_MODULE(_schema, m) // Bind SpaceMouse types (SpaceMouseOutput table). core::bind_spacemouse(m); + // Bind gamepad types (GamepadInputState struct, GamepadSnapshot table). + core::bind_gamepad(m); + // Bind message channel types (MessageChannelMessages table). core::bind_message_channel(m); diff --git a/src/python/isaaccapture/schema/__init__.py b/src/python/isaaccapture/schema/__init__.py index 8fe0ec4fa4..c7f3ac934d 100644 --- a/src/python/isaaccapture/schema/__init__.py +++ b/src/python/isaaccapture/schema/__init__.py @@ -44,6 +44,10 @@ # SpaceMouse types. SpaceMouseOutput, SpaceMouseOutputRecord, + # Gamepad types. + GamepadInputState, + GamepadSnapshot, + GamepadSnapshotRecord, # Keyboard types (held keys plus ordered key events). KeyAction, KeyEvent, @@ -172,6 +176,10 @@ def __getattr__(name: str): # SpaceMouse types. "SpaceMouseOutput", "SpaceMouseOutputRecord", + # Gamepad types. + "GamepadInputState", + "GamepadSnapshot", + "GamepadSnapshotRecord", # Keyboard types (held keys plus ordered key events). "KeyAction", "KeyEvent", diff --git a/tests/cpp/core/schema/CMakeLists.txt b/tests/cpp/core/schema/CMakeLists.txt index 9d66c6a949..3fa2548b78 100644 --- a/tests/cpp/core/schema/CMakeLists.txt +++ b/tests/cpp/core/schema/CMakeLists.txt @@ -13,6 +13,7 @@ add_executable(schema_tests test_pedals.cpp test_steering_wheel.cpp test_spacemouse.cpp + test_gamepad.cpp test_plugin_device_status.cpp test_joint_se3_pose.cpp test_se3_tracker.cpp diff --git a/tests/cpp/core/schema/test_gamepad.cpp b/tests/cpp/core/schema/test_gamepad.cpp new file mode 100644 index 0000000000..a9c11b28ee --- /dev/null +++ b/tests/cpp/core/schema/test_gamepad.cpp @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Unit tests for the generated Gamepad FlatBuffer types. + +#include +#include +#include +#include + +#include + +#define VT(field) (field + 2) * 2 + +static_assert(core::GamepadSnapshot::VT_INPUTS == VT(0)); + +static_assert(core::GamepadSnapshotRecord::VT_DATA == VT(0)); +static_assert(core::GamepadSnapshotRecord::VT_TIMESTAMP == VT(1)); + +// 14 bools padded to the floats' alignment, then 6 floats. +static_assert(sizeof(core::GamepadInputState) == 16 + 6 * 4); + +TEST_CASE("GamepadSnapshotT default construction", "[gamepad][native]") +{ + core::GamepadSnapshotT snapshot; + + CHECK(snapshot.inputs == nullptr); +} + +TEST_CASE("GamepadSnapshot serialization and deserialization", "[gamepad][serialize]") +{ + flatbuffers::FlatBufferBuilder builder; + + core::GamepadSnapshotT snapshot; + core::GamepadInputState inputs; + inputs.mutate_west_click(true); + inputs.mutate_dpad_up_click(true); + inputs.mutate_thumbstick_left_y(1.0f); + inputs.mutate_trigger_right_value(0.5f); + snapshot.inputs = std::make_shared(inputs); + + builder.Finish(core::GamepadSnapshot::Pack(builder, &snapshot)); + + const auto* deserialized = flatbuffers::GetRoot(builder.GetBufferPointer()); + REQUIRE(deserialized->inputs() != nullptr); + CHECK(deserialized->inputs()->west_click()); + CHECK(deserialized->inputs()->dpad_up_click()); + CHECK_FALSE(deserialized->inputs()->south_click()); + CHECK(deserialized->inputs()->thumbstick_left_y() == Catch::Approx(1.0f)); + CHECK(deserialized->inputs()->trigger_right_value() == Catch::Approx(0.5f)); +} diff --git a/tests/python/core/schema/test_gamepad.py b/tests/python/core/schema/test_gamepad.py new file mode 100644 index 0000000000..f22fc114e9 --- /dev/null +++ b/tests/python/core/schema/test_gamepad.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the gamepad types in isaaccapture.schema. + +- GamepadInputState: the buttons and axes every standard gamepad has +- GamepadSnapshot: the input state, absent while no pad is open +- GamepadSnapshotRecord: record wrapper carrying DeviceDataTimestamp +""" + +import pytest + +from isaaccapture.schema import ( + DeviceDataTimestamp, + GamepadInputState, + GamepadSnapshot, + GamepadSnapshotRecord, +) + + +class TestGamepadInputState: + def test_default_is_released_and_centered(self): + inputs = GamepadInputState() + + assert not inputs.south_click + assert not inputs.dpad_right_click + assert inputs.thumbstick_left_x == 0.0 + assert inputs.trigger_right_value == 0.0 + + def test_fields_by_name(self): + inputs = GamepadInputState( + west_click=True, thumbstick_left_y=1.0, trigger_right_value=0.5 + ) + + assert inputs.west_click + assert not inputs.east_click + assert inputs.thumbstick_left_y == pytest.approx(1.0) + assert inputs.trigger_right_value == pytest.approx(0.5) + + def test_repr_names_the_held_buttons(self): + repr_str = repr(GamepadInputState(south_click=True, dpad_up_click=True)) + + assert "held=[south, dpad_up]" in repr_str + + +class TestGamepadSnapshot: + def test_default_is_no_pad(self): + snapshot = GamepadSnapshot() + + assert snapshot.inputs is None + assert "inputs=None" in repr(snapshot) + + def test_fields_round_trip(self): + snapshot = GamepadSnapshot(GamepadInputState(north_click=True)) + + assert snapshot.inputs.north_click + + +class TestGamepadSnapshotRecord: + def test_record_carries_data_and_timestamp(self): + data = GamepadSnapshot(GamepadInputState(south_click=True)) + record = GamepadSnapshotRecord(data, DeviceDataTimestamp(1, 2, 3)) + + assert record.timestamp.available_time_local_common_clock == 1 + assert record.timestamp.sample_time_local_common_clock == 2 + assert record.timestamp.sample_time_raw_device_clock == 3 + assert record.data.inputs.south_click + + def test_payload_less_record(self): + record = GamepadSnapshotRecord(None, DeviceDataTimestamp(1, 2, 3)) + + assert record.data is None From 4a3ca8c4ea63f99b6970602b02d36bc473f032c9 Mon Sep 17 00:00:00 2001 From: Rafael Wiltz Date: Tue, 13 Oct 2026 10:42:02 -0400 Subject: [PATCH 2/4] Add a gamepad plugin Reads a gamepad through SDL3 and pushes GamepadSnapshot at 90 Hz via SchemaPusher. SDL's drivers and SDL_GameControllerDB mappings resolve each pad's own numbering into the standard layout, so Xbox, PlayStation, Switch Pro, Steam and any mapped pad read the same, and the mapping is resolved once per device in the plugin rather than by every reader. --player=N reads the Nth connected pad and keeps it until it disconnects, so several pads run as one plugin each on their own collection. While no pad is open the plugin keeps pushing snapshots without inputs, so a reader never holds the last stick deflection of a pad that went away. The pad's name, type and USB IDs are logged when it is opened. SDL3 (zlib) is fetched at configure time and linked statically with only its joystick and HIDAPI subsystems, so the bundled binary still needs nothing beyond libc/libstdc++. core/ holds the reader, unit-tested against SDL virtual gamepads; app/ is the OpenXR push loop. With --plugin-root-id the plugin reports its device as CONNECTED, DISCONNECTED or FAILED through PluginDeviceStatusPublisher. The plugin is bundled into the isaaccapture package (isaaccapture/plugins/gamepad), so a wheel install can start it. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Rafael Wiltz --- src/plugins/CMakeLists.txt | 1 + src/plugins/gamepad/CMakeLists.txt | 42 +++++ src/plugins/gamepad/README.md | 60 +++++++ src/plugins/gamepad/app/CMakeLists.txt | 32 ++++ src/plugins/gamepad/app/gamepad_plugin.cpp | 116 +++++++++++++ src/plugins/gamepad/app/gamepad_plugin.hpp | 60 +++++++ src/plugins/gamepad/app/main.cpp | 88 ++++++++++ src/plugins/gamepad/core/CMakeLists.txt | 24 +++ .../gamepad/core/gamepad_connection.cpp | 159 +++++++++++++++++ .../core/inc/gamepad/gamepad_connection.hpp | 98 +++++++++++ src/plugins/gamepad/plugin.yaml | 11 ++ tests/cpp/plugins/CMakeLists.txt | 3 +- tests/cpp/plugins/gamepad/CMakeLists.txt | 15 ++ .../gamepad/test_gamepad_connection.cpp | 164 ++++++++++++++++++ tests/python/packaging/test_wheel_contents.py | 15 ++ 15 files changed, 887 insertions(+), 1 deletion(-) create mode 100644 src/plugins/gamepad/CMakeLists.txt create mode 100644 src/plugins/gamepad/README.md create mode 100644 src/plugins/gamepad/app/CMakeLists.txt create mode 100644 src/plugins/gamepad/app/gamepad_plugin.cpp create mode 100644 src/plugins/gamepad/app/gamepad_plugin.hpp create mode 100644 src/plugins/gamepad/app/main.cpp create mode 100644 src/plugins/gamepad/core/CMakeLists.txt create mode 100644 src/plugins/gamepad/core/gamepad_connection.cpp create mode 100644 src/plugins/gamepad/core/inc/gamepad/gamepad_connection.hpp create mode 100644 src/plugins/gamepad/plugin.yaml create mode 100644 tests/cpp/plugins/gamepad/CMakeLists.txt create mode 100644 tests/cpp/plugins/gamepad/test_gamepad_connection.cpp diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index 25a5cb98ea..4e5fa231f1 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -19,6 +19,7 @@ add_subdirectory(controller_synthetic_hands) add_subdirectory(generic_3axis_pedal) add_subdirectory(steering_wheel) add_subdirectory(spacemouse) +add_subdirectory(gamepad) add_subdirectory(xsens_full_body) add_subdirectory(so101_leader) add_subdirectory(rebot_devarm_leader) diff --git a/src/plugins/gamepad/CMakeLists.txt b/src/plugins/gamepad/CMakeLists.txt new file mode 100644 index 0000000000..566525bc9a --- /dev/null +++ b/src/plugins/gamepad/CMakeLists.txt @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Reads a gamepad through SDL3 and pushes GamepadSnapshot, in the standard gamepad layout, via OpenXR. +if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(STATUS "Skipping gamepad plugin (Linux only)") + add_custom_target(gamepad_plugin + COMMAND ${CMAKE_COMMAND} -E echo "Skipping gamepad: Linux only") + return() +endif() + +# ------------------------------------------------------------------------------ +# SDL3 (zlib): device support and the SDL_GameControllerDB mappings that turn each pad's own +# numbering into the standard layout. Built static with only the joystick and HIDAPI subsystems, so +# the bundled plugin still needs nothing beyond libc/libstdc++; libudev is loaded at runtime. +# ------------------------------------------------------------------------------ +include(FetchContent) +FetchContent_Declare( + SDL3 + GIT_REPOSITORY https://github.com/libsdl-org/SDL.git + GIT_TAG 829a65d769d935c4852f8159e964312c0957260a # release-3.4.18 +) +foreach(_sdl_option SDL_SHARED SDL_TEST_LIBRARY SDL_TESTS SDL_EXAMPLES SDL_INSTALL + SDL_AUDIO SDL_VIDEO SDL_HAPTIC SDL_POWER SDL_SENSOR SDL_DIALOG SDL_TRAY + SDL_DBUS SDL_IBUS SDL_HIDAPI_LIBUSB) + set(${_sdl_option} OFF CACHE BOOL "" FORCE) +endforeach() +# SDL_UNIX_CONSOLE_BUILD: no windows, so no X11 or Wayland development packages are needed. +foreach(_sdl_option SDL_STATIC SDL_JOYSTICK SDL_HIDAPI SDL_UNIX_CONSOLE_BUILD) + set(${_sdl_option} ON CACHE BOOL "" FORCE) +endforeach() +isaac_teleop_third_party_scope_begin() +FetchContent_MakeAvailable(SDL3) +isaac_teleop_third_party_scope_end() + +# app/ also stages the plugin with its manifest into the Python package. +set(GAMEPAD_PLUGIN_MANIFEST "${CMAKE_CURRENT_SOURCE_DIR}/plugin.yaml") + +add_subdirectory(core) +add_subdirectory(app) + +install(FILES plugin.yaml README.md DESTINATION plugins/gamepad) diff --git a/src/plugins/gamepad/README.md b/src/plugins/gamepad/README.md new file mode 100644 index 0000000000..634c4abb11 --- /dev/null +++ b/src/plugins/gamepad/README.md @@ -0,0 +1,60 @@ + + +# Gamepad Plugin + +Reads a gamepad through [SDL3](https://github.com/libsdl-org/SDL) and pushes `GamepadSnapshot` +via OpenXR. The inputs are `GamepadInputState` from `gamepad.fbs`: the buttons +and axes every standard pad has, the same set as OpenXR's `xbox_controller` profile, as named fields +like `ControllerInputState`'s, with face buttons named by position (`south_click`, `west_click`, +...). SDL resolves each pad's own numbering through its drivers and +[SDL_GameControllerDB](https://github.com/mdqinc/SDL_GameControllerDB) mappings, so `west_click` is X +on an Xbox pad and Square on a PlayStation pad. Read it with `GamepadTracker`, `GamepadSource`, or +any reader using the same `collection_id`. Linux only. + +The plugin is also bundled in the `isaaccapture` package: a `TeleopSession` starts it with +`PluginConfig(plugin_name="gamepad", plugin_root_id="gamepad", search_paths=[BUNDLED_PLUGINS_DIR])`. + +Supported pads: Xbox (360, One, Series, USB and Bluetooth), PlayStation (DualShock 4, DualSense), +Nintendo Switch Pro, Steam Controller, and any pad with an SDL_GameControllerDB mapping. Validated: +Xbox 360 Controller. + +SDL3 (zlib license) is fetched at configure time and linked statically with only its joystick and +HIDAPI subsystems, so the binary needs nothing beyond libc/libstdc++. + +## Device access + +On a desktop session udev grants the logged-in user access to gamepad nodes, so no setup is +needed. Without a local seat (SSH, a service, a container) add the user to the `input` group, and +pass the pad's `/dev/input/event*` node into a container. + +## Usage + +```bash +./gamepad_plugin [--player=N] [--collection-id=ID] +``` + +- **--player**: Read the Nth connected gamepad, in connection order (default 1). The pad is kept + until it disconnects; for several pads, run one plugin per player with its own collection. +- **--collection-id**: Default `gamepad`. Match this when creating `GamepadTracker`. +- **--plugin-root-id**: Set by the plugin launcher; enables device status reporting. + +The plugin pushes at 90 Hz. While no pad is open it keeps pushing snapshots without inputs, and +nothing held, so a reader never holds the last stick deflection of a pad that went away. The pad's +name, type and USB IDs are logged when it is opened. + +A pad without a mapping is not listed as a gamepad. Give SDL one through the +`SDL_GAMECONTROLLERCONFIG` environment variable (an SDL_GameControllerDB line) in the plugin's +environment. + +## Data + +- `inputs`: `GamepadInputState`. Sticks are in `[-1, 1]` with right and up positive, triggers in + `[0, 1]`. + +## Device status + +With `--plugin-root-id`, the plugin reports device `/gamepad` as `CONNECTED`, `DISCONNECTED` (no +pad for this player), or `FAILED` (SDL found the pad but could not open it). diff --git a/src/plugins/gamepad/app/CMakeLists.txt b/src/plugins/gamepad/app/CMakeLists.txt new file mode 100644 index 0000000000..fbd4799f31 --- /dev/null +++ b/src/plugins/gamepad/app/CMakeLists.txt @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +add_executable(gamepad_plugin + main.cpp + gamepad_plugin.cpp + gamepad_plugin.hpp +) + +target_link_libraries(gamepad_plugin PRIVATE + gamepad::gamepad_core + Teleop::plugin_utils + pusherio::pusherio + oxr::oxr_core + isaacteleop_schema + log_bridge::log_bridge_core +) + +install(TARGETS gamepad_plugin RUNTIME DESTINATION plugins/gamepad) + +# Ship the plugin inside the isaaccapture package, so a wheel install can start it (PluginConfig with +# search path BUNDLED_PLUGINS_DIR). The binary links nothing beyond libc/libstdc++. +if(TARGET python_package) + set(_bundle_dir "${CMAKE_BINARY_DIR}/python_package/$/isaaccapture/plugins/gamepad") + add_custom_command(TARGET gamepad_plugin POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory "${_bundle_dir}" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$" "${GAMEPAD_PLUGIN_MANIFEST}" "${_bundle_dir}/" + COMMENT "Bundling gamepad_plugin into the isaaccapture package") + # Files staged into the package tree must exist before the wheel is built from it. + add_dependencies(python_package gamepad_plugin) +endif() diff --git a/src/plugins/gamepad/app/gamepad_plugin.cpp b/src/plugins/gamepad/app/gamepad_plugin.cpp new file mode 100644 index 0000000000..6af089e09b --- /dev/null +++ b/src/plugins/gamepad/app/gamepad_plugin.cpp @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "gamepad_plugin.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace plugins +{ +namespace gamepad +{ + +namespace +{ + +// Matches max_flatbuffer_size in trackers.toml (the default), which the host's GamepadTracker reads +// with; a sample is under 100 bytes. +constexpr size_t kMaxFlatbufferSize = 512; + +std::vector required_extensions(bool monitoring) +{ + std::vector extensions = core::SchemaPusher::get_required_extensions(); + if (monitoring) + { + for (const auto& extension : plugin_utils::PluginDeviceStatusPublisher::get_required_extensions()) + { + if (std::find(extensions.begin(), extensions.end(), extension) == extensions.end()) + extensions.push_back(extension); + } + } + return extensions; +} + +std::shared_ptr make_session(const GamepadPluginConfig& config) +{ + if (config.plugin_root_id && config.plugin_root_id->empty()) + throw std::invalid_argument("GamepadPlugin: --plugin-root-id must not be empty"); + return std::make_shared("GamepadPlugin", required_extensions(config.plugin_root_id.has_value())); +} + +} // namespace + +GamepadPlugin::GamepadPlugin(GamepadPluginConfig config) + : connection_(std::move(config.connection)), + session_(make_session(config)), + pusher_(session_->get_handles(), + core::SchemaPusherConfig{ .collection_id = config.collection_id, + .max_flatbuffer_size = kMaxFlatbufferSize, + .tensor_identifier = "gamepad", + .localized_name = "Gamepad", + .app_name = "GamepadPlugin" }) +{ + if (config.plugin_root_id) + status_ = + std::make_unique(session_->get_handles(), *config.plugin_root_id); +} + +GamepadPlugin::~GamepadPlugin() = default; + +void GamepadPlugin::update() +{ + const int64_t now_ns = core::os_monotonic_now_ns(); + connection_.poll(); + push_state(now_ns); + publish_status(now_ns); +} + +void GamepadPlugin::push_state(int64_t now_ns) +{ + core::GamepadSnapshotT out; + // No inputs while no pad is open, so readers drop the last input of a pad that went away. + if (connection_.status() == ConnectionStatus::Connected) + out.inputs = std::make_shared(connection_.state().inputs); + + flatbuffers::FlatBufferBuilder builder(kMaxFlatbufferSize); + builder.Finish(core::GamepadSnapshot::Pack(builder, &out)); + pusher_.push_buffer(builder.GetBufferPointer(), builder.GetSize(), now_ns, now_ns); +} + +void GamepadPlugin::publish_status(int64_t now_ns) +{ + if (!status_) + return; + + plugin_utils::PluginDeviceStatusEntry entry{ .path = DEVICE_PATH }; + switch (connection_.status()) + { + case ConnectionStatus::Connected: + entry.state = core::PluginDeviceState_CONNECTED; + entry.reason = core::PluginDeviceReason_NONE; + break; + case ConnectionStatus::NoDevice: + entry.state = core::PluginDeviceState_DISCONNECTED; + entry.reason = core::PluginDeviceReason_NO_HARDWARE_SIGNAL; + break; + case ConnectionStatus::OpenFailed: + entry.state = core::PluginDeviceState_FAILED; + entry.reason = core::PluginDeviceReason_DEVICE_ERROR; + entry.error = connection_.error(); + break; + } + status_->publish_if_changed({ entry }, now_ns); +} + +} // namespace gamepad +} // namespace plugins diff --git a/src/plugins/gamepad/app/gamepad_plugin.hpp b/src/plugins/gamepad/app/gamepad_plugin.hpp new file mode 100644 index 0000000000..b7c33a6dec --- /dev/null +++ b/src/plugins/gamepad/app/gamepad_plugin.hpp @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include + +#include +#include +#include + +namespace core +{ +class OpenXRSession; +} + +namespace plugins +{ +namespace gamepad +{ + +struct GamepadPluginConfig +{ + GamepadConnectionConfig connection; + std::string collection_id = "gamepad"; + //! Set by the plugin launcher; enables device status reporting for monitoring. + std::optional plugin_root_id; +}; + +/*! + * @brief Reads a gamepad and pushes GamepadSnapshot via OpenXR SchemaPusher. + * + * A sample is pushed on every update, also while no device is open (no inputs, nothing + * held), so the host never keeps the last stick deflection of a pad that went away. + */ +class GamepadPlugin +{ +public: + //! Device path reported to plugin monitoring; matches plugin.yaml. + static constexpr const char* DEVICE_PATH = "/gamepad"; + + explicit GamepadPlugin(GamepadPluginConfig config); + ~GamepadPlugin(); + + void update(); + +private: + void push_state(int64_t now_ns); + void publish_status(int64_t now_ns); + + GamepadConnection connection_; + std::shared_ptr session_; + core::SchemaPusher pusher_; + std::unique_ptr status_; +}; + +} // namespace gamepad +} // namespace plugins diff --git a/src/plugins/gamepad/app/main.cpp b/src/plugins/gamepad/app/main.cpp new file mode 100644 index 0000000000..d6a0bb0f28 --- /dev/null +++ b/src/plugins/gamepad/app/main.cpp @@ -0,0 +1,88 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "gamepad_plugin.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace plugins::gamepad; + +namespace +{ + +constexpr std::string_view kUsage = "Usage: gamepad_plugin [--player=N] [--collection-id=ID] [--plugin-root-id=ID]"; + +// The value of `--name=value` in @p arg, or nullopt when @p arg is another flag. +std::optional flag_value(std::string_view arg, std::string_view name) +{ + if (arg.starts_with(name) && arg.size() > name.size() && arg[name.size()] == '=') + return std::string(arg.substr(name.size() + 1)); + return std::nullopt; +} + +} // namespace + +int main(int argc, char** argv) +try +{ + auto logger = isaaccapture::Logger::get("isaaccapture.plugins.gamepad.main"); + + GamepadPluginConfig config; + for (int i = 1; i < argc; ++i) + { + const std::string_view arg = argv[i]; + if (arg == "--help" || arg == "-h") + { + // Usage text is terminal UX, not a diagnostic. + std::cout << kUsage << std::endl; + return 0; + } + if (auto value = flag_value(arg, "--player")) + config.connection.player = std::stoi(*value); + else if (auto value = flag_value(arg, "--collection-id")) + config.collection_id = *value; + else if (auto value = flag_value(arg, "--plugin-root-id")) + config.plugin_root_id = *value; + else + logger->warn("ignoring unknown argument '{}'", arg); + } + + logger->info("Gamepad (player: {}, collection: {})", config.connection.player, config.collection_id); + + GamepadPlugin plugin(std::move(config)); + + // Push data at 90 Hz + const auto frame_duration = std::chrono::nanoseconds(1000000000 / 90); + const auto program_start = std::chrono::steady_clock::now(); + std::size_t frame_count = 0; + + while (true) + { + plugin.update(); + frame_count++; + std::this_thread::sleep_until(program_start + frame_duration * frame_count); + } + + return 0; +} +catch (const std::exception& e) +{ + auto logger = isaaccapture::Logger::get("isaaccapture.plugins.gamepad.main"); + logger->error("{}: {}", argv[0], e.what()); + return 1; +} +catch (...) +{ + auto logger = isaaccapture::Logger::get("isaaccapture.plugins.gamepad.main"); + logger->error("{}: Unknown error", argv[0]); + return 1; +} diff --git a/src/plugins/gamepad/core/CMakeLists.txt b/src/plugins/gamepad/core/CMakeLists.txt new file mode 100644 index 0000000000..5ac86928fe --- /dev/null +++ b/src/plugins/gamepad/core/CMakeLists.txt @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The SDL3 reader, kept apart from the OpenXR push loop so it can be unit-tested against SDL's +# virtual gamepads without a runtime or a device. +add_library(gamepad_core STATIC + gamepad_connection.cpp + inc/gamepad/gamepad_connection.hpp +) + +target_include_directories(gamepad_core + INTERFACE + $ +) + +target_link_libraries(gamepad_core + PUBLIC + log_bridge::log_bridge_core + isaacteleop_schema + PRIVATE + SDL3::SDL3-static +) + +add_library(gamepad::gamepad_core ALIAS gamepad_core) diff --git a/src/plugins/gamepad/core/gamepad_connection.cpp b/src/plugins/gamepad/core/gamepad_connection.cpp new file mode 100644 index 0000000000..1bf60c1156 --- /dev/null +++ b/src/plugins/gamepad/core/gamepad_connection.cpp @@ -0,0 +1,159 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/gamepad/gamepad_connection.hpp" + +#include + +#include +#include +#include +#include + +namespace plugins +{ +namespace gamepad +{ + +namespace +{ + +// SDL reports [-32768, 32767]; sticks map to [-1, 1] and triggers to [0, 1]. +float stick(SDL_Gamepad* gamepad, SDL_GamepadAxis axis, bool flip = false) +{ + const float value = static_cast(SDL_GetGamepadAxis(gamepad, axis)) / 32767.0f; + return std::clamp(flip ? -value : value, -1.0f, 1.0f); +} + +float trigger(SDL_Gamepad* gamepad, SDL_GamepadAxis axis) +{ + return std::clamp(static_cast(SDL_GetGamepadAxis(gamepad, axis)) / 32767.0f, 0.0f, 1.0f); +} + +} // namespace + +void GamepadState::clear() +{ + inputs = core::GamepadInputState(); +} + +GamepadConnection::GamepadConnection(GamepadConnectionConfig config) : config_(config) +{ + if (config_.player < 1) + throw std::invalid_argument("GamepadConnection: player must be 1 or more"); + + // The plugin launcher stops the plugin with a signal; SDL must not intercept it. + SDL_SetHint(SDL_HINT_NO_SIGNAL_HANDLERS, "1"); + // There is no window, so the process never has focus: read the pad regardless. + SDL_SetHint(SDL_HINT_JOYSTICK_ALLOW_BACKGROUND_EVENTS, "1"); + if (!SDL_InitSubSystem(SDL_INIT_GAMEPAD)) + throw std::runtime_error(std::string("GamepadConnection: SDL_Init failed: ") + SDL_GetError()); + // poll() reads state directly, so nothing drains an event queue. + SDL_SetJoystickEventsEnabled(false); + SDL_SetGamepadEventsEnabled(false); +} + +GamepadConnection::~GamepadConnection() +{ + if (gamepad_ != nullptr) + SDL_CloseGamepad(gamepad_); + SDL_QuitSubSystem(SDL_INIT_GAMEPAD); +} + +void GamepadConnection::poll() +{ + // Refreshes every pad's state and detects pads added or removed. + SDL_UpdateGamepads(); + if (gamepad_ != nullptr && !SDL_GamepadConnected(gamepad_)) + close_device(); + if (gamepad_ == nullptr) + open_device(); + if (gamepad_ != nullptr) + read_state(); +} + +void GamepadConnection::open_device() +{ + int count = 0; + SDL_JoystickID* ids = SDL_GetGamepads(&count); + if (ids == nullptr) + { + set_status(ConnectionStatus::NoDevice); + return; + } + // Instance IDs only grow, so ascending order is connection order. + std::sort(ids, ids + count); + const SDL_JoystickID id = count >= config_.player ? ids[config_.player - 1] : 0; + SDL_free(ids); + if (id == 0) + { + set_status(ConnectionStatus::NoDevice); + return; + } + + gamepad_ = SDL_OpenGamepad(id); + if (gamepad_ == nullptr) + { + set_status(ConnectionStatus::OpenFailed, SDL_GetError()); + return; + } + + char usb_ids[16]; + std::snprintf(usb_ids, sizeof(usb_ids), "%04x:%04x", SDL_GetGamepadVendor(gamepad_), SDL_GetGamepadProduct(gamepad_)); + const char* name = SDL_GetGamepadName(gamepad_); + const char* type = SDL_GetGamepadStringForType(SDL_GetGamepadType(gamepad_)); + description_ = std::string(name != nullptr ? name : "Unknown gamepad") + " (" + + (type != nullptr ? type : "unknown") + ", " + usb_ids + ")"; + state_.clear(); + set_status(ConnectionStatus::Connected); +} + +void GamepadConnection::close_device() +{ + SDL_CloseGamepad(gamepad_); + gamepad_ = nullptr; + logger_->info("Gamepad {} disconnected", description_); + description_.clear(); + state_.clear(); +} + +void GamepadConnection::read_state() +{ + const auto pressed = [this](SDL_GamepadButton button) { return SDL_GetGamepadButton(gamepad_, button); }; + // SDL reports stick Y down-positive; flip it to up-positive, as OpenXR's thumbstick y is. + state_.inputs = core::GamepadInputState( + pressed(SDL_GAMEPAD_BUTTON_SOUTH), pressed(SDL_GAMEPAD_BUTTON_EAST), pressed(SDL_GAMEPAD_BUTTON_WEST), + pressed(SDL_GAMEPAD_BUTTON_NORTH), pressed(SDL_GAMEPAD_BUTTON_BACK), pressed(SDL_GAMEPAD_BUTTON_START), + pressed(SDL_GAMEPAD_BUTTON_LEFT_SHOULDER), pressed(SDL_GAMEPAD_BUTTON_RIGHT_SHOULDER), + pressed(SDL_GAMEPAD_BUTTON_LEFT_STICK), pressed(SDL_GAMEPAD_BUTTON_RIGHT_STICK), + pressed(SDL_GAMEPAD_BUTTON_DPAD_UP), pressed(SDL_GAMEPAD_BUTTON_DPAD_DOWN), + pressed(SDL_GAMEPAD_BUTTON_DPAD_LEFT), pressed(SDL_GAMEPAD_BUTTON_DPAD_RIGHT), + stick(gamepad_, SDL_GAMEPAD_AXIS_LEFTX), stick(gamepad_, SDL_GAMEPAD_AXIS_LEFTY, true), + stick(gamepad_, SDL_GAMEPAD_AXIS_RIGHTX), stick(gamepad_, SDL_GAMEPAD_AXIS_RIGHTY, true), + trigger(gamepad_, SDL_GAMEPAD_AXIS_LEFT_TRIGGER), trigger(gamepad_, SDL_GAMEPAD_AXIS_RIGHT_TRIGGER)); +} + +void GamepadConnection::set_status(ConnectionStatus status, std::string error) +{ + const bool report = !status_logged_ || status != status_; + status_ = status; + error_ = std::move(error); + status_logged_ = true; + if (!report) + return; + switch (status) + { + case ConnectionStatus::Connected: + logger_->info("Opened gamepad {} as player {}", description_, config_.player); + break; + case ConnectionStatus::NoDevice: + logger_->info("No gamepad for player {}; waiting for one to connect", config_.player); + break; + case ConnectionStatus::OpenFailed: + logger_->warn("Cannot open gamepad for player {}: {}", config_.player, error_); + break; + } +} + +} // namespace gamepad +} // namespace plugins diff --git a/src/plugins/gamepad/core/inc/gamepad/gamepad_connection.hpp b/src/plugins/gamepad/core/inc/gamepad/gamepad_connection.hpp new file mode 100644 index 0000000000..aede7a8d84 --- /dev/null +++ b/src/plugins/gamepad/core/inc/gamepad/gamepad_connection.hpp @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +#include +#include + +struct SDL_Gamepad; + +namespace plugins +{ +namespace gamepad +{ + +//! Why the pad is or is not open. +enum class ConnectionStatus +{ + Connected, //!< The pad is open and being read. + NoDevice, //!< Fewer gamepads are connected than the configured player number. + OpenFailed, //!< The pad is connected but SDL could not open it. +}; + +struct GamepadConnectionConfig +{ + //! 1-based: read the Nth connected gamepad, in connection order. The pad is kept until it + //! disconnects, then the Nth one is taken again. + int player = 1; +}; + +struct GamepadState +{ + //! Sticks in [-1, 1] with right and up positive, triggers in [0, 1] (see gamepad.fbs). + core::GamepadInputState inputs; + + //! The state of a pad that is not open: no buttons held, axes at rest. + void clear(); +}; + +/*! + * @brief Reads one gamepad through SDL3 in the standard layout, reopening after a disconnect. + * + * SDL resolves each pad's own button and axis numbering through its built-in drivers and + * SDL_GameControllerDB mappings. poll() refreshes SDL, opens the configured player's pad when one + * is connected, and reads its state; a closed pad's state is cleared. + */ +class GamepadConnection +{ +public: + //! Initializes SDL's gamepad subsystem; throws std::runtime_error when it cannot. + explicit GamepadConnection(GamepadConnectionConfig config); + ~GamepadConnection(); + + GamepadConnection(const GamepadConnection&) = delete; + GamepadConnection& operator=(const GamepadConnection&) = delete; + + void poll(); + + ConnectionStatus status() const + { + return status_; + } + //! Why the last open failed; empty unless status() is OpenFailed. + const std::string& error() const + { + return error_; + } + //! The open pad's name, type and USB IDs; empty while none is open. + const std::string& description() const + { + return description_; + } + const GamepadState& state() const + { + return state_; + } + +private: + void open_device(); + void close_device(); + void read_state(); + void set_status(ConnectionStatus status, std::string error = {}); + + GamepadConnectionConfig config_; + SDL_Gamepad* gamepad_ = nullptr; + ConnectionStatus status_ = ConnectionStatus::NoDevice; + bool status_logged_ = false; //!< The current status() has been logged once. + std::string error_; + std::string description_; + GamepadState state_; + std::shared_ptr logger_ = isaaccapture::Logger::get("isaaccapture.plugins.gamepad.GamepadConnection"); +}; + +} // namespace gamepad +} // namespace plugins diff --git a/src/plugins/gamepad/plugin.yaml b/src/plugins/gamepad/plugin.yaml new file mode 100644 index 0000000000..df083a4f08 --- /dev/null +++ b/src/plugins/gamepad/plugin.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: gamepad +description: "Gamepad (Xbox, PlayStation, Switch Pro, ...) via SDL3" +command: "./gamepad_plugin" +version: "1.0.0" +devices: + - path: "/gamepad" + type: "gamepad" + description: "Gamepad in the standard layout" diff --git a/tests/cpp/plugins/CMakeLists.txt b/tests/cpp/plugins/CMakeLists.txt index 454bc58ff0..1fd14aa203 100644 --- a/tests/cpp/plugins/CMakeLists.txt +++ b/tests/cpp/plugins/CMakeLists.txt @@ -3,9 +3,10 @@ add_subdirectory(plugin_utils) -# The SpaceMouse plugin (and its spacemouse_core library) is Linux only. +# The SpaceMouse and gamepad plugins (and their core libraries) are Linux only. if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_subdirectory(spacemouse) + add_subdirectory(gamepad) endif() if(BUILD_PLUGIN_OGLO) diff --git a/tests/cpp/plugins/gamepad/CMakeLists.txt b/tests/cpp/plugins/gamepad/CMakeLists.txt new file mode 100644 index 0000000000..b1a3931d5f --- /dev/null +++ b/tests/cpp/plugins/gamepad/CMakeLists.txt @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +add_executable(gamepad_plugin_tests + test_gamepad_connection.cpp +) + +target_link_libraries(gamepad_plugin_tests + PRIVATE + Catch2::Catch2WithMain + gamepad::gamepad_core + SDL3::SDL3-static +) + +catch_discover_tests(gamepad_plugin_tests) diff --git a/tests/cpp/plugins/gamepad/test_gamepad_connection.cpp b/tests/cpp/plugins/gamepad/test_gamepad_connection.cpp new file mode 100644 index 0000000000..628bcaf45f --- /dev/null +++ b/tests/cpp/plugins/gamepad/test_gamepad_connection.cpp @@ -0,0 +1,164 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// The connection reads SDL virtual gamepads: attaching one is a pad being plugged in, setting its +// buttons and axes is input, and detaching it is an unplug. The state is read back through +// GamepadInputState's named fields, so the plugin's SDL-to-schema mapping is checked end to end. Real +// pads are hidden from SDL so a pad on the test machine does not count. + +#include +#include +#include +#include + +#include +#include + +using plugins::gamepad::ConnectionStatus; +using plugins::gamepad::GamepadConnection; +using plugins::gamepad::GamepadConnectionConfig; + +namespace +{ + +// pid.codes test VID; each pad gets its own product ID, which SDL reports per device. +constexpr uint16_t kVirtualVendor = 0x1209; + +//! A connection that sees only the virtual pads this file attaches. +std::unique_ptr make_connection(int player = 1) +{ + SDL_SetHint(SDL_HINT_GAMECONTROLLER_IGNORE_DEVICES_EXCEPT, "0x1209/0x0001,0x1209/0x0002"); + return std::make_unique(GamepadConnectionConfig{ player }); +} + +//! A virtual gamepad with every standard button and axis, in SDL_GamepadButton/SDL_GamepadAxis order. +class VirtualPad +{ +public: + explicit VirtualPad(const char* name, uint16_t product_id = 0x0001) + { + SDL_VirtualJoystickDesc desc; + SDL_INIT_INTERFACE(&desc); + desc.type = SDL_JOYSTICK_TYPE_GAMEPAD; + desc.vendor_id = kVirtualVendor; + desc.product_id = product_id; + desc.naxes = SDL_GAMEPAD_AXIS_COUNT; + desc.nbuttons = SDL_GAMEPAD_BUTTON_COUNT; + desc.name = name; + id_ = SDL_AttachVirtualJoystick(&desc); + REQUIRE(id_ != 0); + joystick_ = SDL_OpenJoystick(id_); + REQUIRE(joystick_ != nullptr); + } + ~VirtualPad() + { + unplug(); + } + void press(SDL_GamepadButton button, bool down = true) + { + REQUIRE(SDL_SetJoystickVirtualButton(joystick_, button, down)); + } + void move(SDL_GamepadAxis axis, int16_t value) + { + REQUIRE(SDL_SetJoystickVirtualAxis(joystick_, axis, value)); + } + void unplug() + { + if (joystick_ != nullptr) + SDL_CloseJoystick(joystick_); + if (id_ != 0) + SDL_DetachVirtualJoystick(id_); + joystick_ = nullptr; + id_ = 0; + } + +private: + SDL_JoystickID id_ = 0; + SDL_Joystick* joystick_ = nullptr; +}; + +} // namespace + +TEST_CASE("No gamepad connected: disconnected at rest", "[gamepad][connection]") +{ + auto connection = make_connection(); + + connection->poll(); + + CHECK(connection->status() == ConnectionStatus::NoDevice); + CHECK(connection->description().empty()); + CHECK(connection->state().inputs.thumbstick_left_x() == 0.0f); +} + +TEST_CASE("A connected gamepad is read in the standard layout until it is unplugged", "[gamepad][connection]") +{ + auto connection = make_connection(); + VirtualPad pad("Test Pad"); + + connection->poll(); + REQUIRE(connection->status() == ConnectionStatus::Connected); + CHECK(connection->description().find("Test Pad") != std::string::npos); + CHECK(connection->description().find("1209:0001") != std::string::npos); + + pad.press(SDL_GAMEPAD_BUTTON_WEST); + pad.press(SDL_GAMEPAD_BUTTON_DPAD_UP); + pad.move(SDL_GAMEPAD_AXIS_LEFTY, -32768); // SDL: stick fully up + pad.move(SDL_GAMEPAD_AXIS_RIGHTX, 16384); + pad.move(SDL_GAMEPAD_AXIS_RIGHT_TRIGGER, 32767); + connection->poll(); + + const auto& state = connection->state(); + CHECK(state.inputs.west_click()); + CHECK(state.inputs.dpad_up_click()); + CHECK_FALSE(state.inputs.south_click()); + CHECK(state.inputs.thumbstick_left_y() == Catch::Approx(1.0f)); // up-positive + CHECK(state.inputs.thumbstick_right_x() == Catch::Approx(0.5f).margin(1e-4)); + CHECK(state.inputs.trigger_right_value() == Catch::Approx(1.0f)); + CHECK(state.inputs.trigger_left_value() == 0.0f); + + pad.press(SDL_GAMEPAD_BUTTON_WEST, false); + connection->poll(); + CHECK_FALSE(connection->state().inputs.west_click()); + + pad.unplug(); + connection->poll(); + CHECK(connection->status() == ConnectionStatus::NoDevice); + CHECK_FALSE(connection->state().inputs.dpad_up_click()); + CHECK(connection->state().inputs.thumbstick_left_y() == 0.0f); +} + +TEST_CASE("A gamepad plugged in later is picked up", "[gamepad][connection]") +{ + auto connection = make_connection(); + connection->poll(); + REQUIRE(connection->status() == ConnectionStatus::NoDevice); + + VirtualPad pad("Late Pad"); + connection->poll(); + + CHECK(connection->status() == ConnectionStatus::Connected); +} + +TEST_CASE("The player number selects the Nth connected gamepad and keeps it", "[gamepad][connection]") +{ + auto connection = make_connection(2); + VirtualPad first("First Pad", 0x0001); + connection->poll(); + CHECK(connection->status() == ConnectionStatus::NoDevice); + + VirtualPad second("Second Pad", 0x0002); + connection->poll(); + REQUIRE(connection->status() == ConnectionStatus::Connected); + CHECK(connection->description().find("1209:0002") != std::string::npos); + + // Player 1 leaving does not move player 2 onto another pad. + first.unplug(); + connection->poll(); + CHECK(connection->status() == ConnectionStatus::Connected); + CHECK(connection->description().find("1209:0002") != std::string::npos); +} + +TEST_CASE("A player number below 1 is rejected", "[gamepad][connection]") +{ + CHECK_THROWS_AS(GamepadConnection(GamepadConnectionConfig{ 0 }), std::invalid_argument); +} diff --git a/tests/python/packaging/test_wheel_contents.py b/tests/python/packaging/test_wheel_contents.py index 2d3e7eae4a..33c8c0674f 100644 --- a/tests/python/packaging/test_wheel_contents.py +++ b/tests/python/packaging/test_wheel_contents.py @@ -134,3 +134,18 @@ def test_capture_license_ships_in_distribution_metadata() -> None: license_path = metadata_path.removesuffix("METADATA") + "licenses/LICENSE.md" assert archive.read(license_path) == (repo_root() / "LICENSE.md").read_bytes() assert not any("isaaccapture_egopost/" in name for name in archive.namelist()) + + +@pytest.mark.skipif( + not sys.platform.startswith("linux"), reason="the plugin is Linux only" +) +def test_gamepad_plugin_ships_runnable_for_teleop_session() -> None: + """Apps start the bundled plugin from BUNDLED_PLUGINS_DIR, so it needs its manifest and exec bit.""" + bundle = f"{DIST}/plugins/gamepad" + with zipfile.ZipFile(_wheel()) as archive: + names = set(archive.namelist()) + binary = archive.getinfo(f"{bundle}/gamepad_plugin") + assert f"{bundle}/plugin.yaml" in names + assert (binary.external_attr >> 16) & 0o111, ( + "gamepad_plugin lost its executable bit" + ) From 377d268eab766f038ad9152e095d9abf9b7eb374 Mon Sep 17 00:00:00 2001 From: Rafael Wiltz Date: Tue, 13 Oct 2026 10:42:32 -0400 Subject: [PATCH 3/4] Monitor the gamepad plugin A TeleopSession that starts the gamepad plugin now tracks its gamepad/device_status collection, so the pad's CONNECTED, DISCONNECTED and FAILED states reach get_device_status() like the SpaceMouse's. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Rafael Wiltz --- .../teleop_session_manager/teleop_session.py | 2 +- .../test_teleop_session.py | 19 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/python/isaaccapture/teleop_session_manager/teleop_session.py b/src/python/isaaccapture/teleop_session_manager/teleop_session.py index 9645931141..d4bf267b68 100644 --- a/src/python/isaaccapture/teleop_session_manager/teleop_session.py +++ b/src/python/isaaccapture/teleop_session_manager/teleop_session.py @@ -62,7 +62,7 @@ logger = logging.getLogger(__name__) # Plugins that publish device status (see PluginDeviceStatusPublisher); others launch unmonitored. -_MONITORED_PLUGIN_NAMES = frozenset({"manus_hand_plugin", "spacemouse"}) +_MONITORED_PLUGIN_NAMES = frozenset({"manus_hand_plugin", "spacemouse", "gamepad"}) def _resolve_sink(entry: GraphExecutable) -> tuple[GraphExecutable, IDeviceIOSink]: diff --git a/tests/python/core/teleop_session_manager/test_teleop_session.py b/tests/python/core/teleop_session_manager/test_teleop_session.py index 2f07777ba7..919e79de11 100644 --- a/tests/python/core/teleop_session_manager/test_teleop_session.py +++ b/tests/python/core/teleop_session_manager/test_teleop_session.py @@ -1563,15 +1563,14 @@ def test_plugin_args_default_empty(self, tmp_path): class TestStatusMonitoringIntegration: """Test monitored-plugin inventory, refresh, and provider failure integration.""" - def test_spacemouse_plugin_is_monitored(self, tmp_path): - manager = MockPluginManager(plugin_names=["spacemouse"]) + @pytest.mark.parametrize("plugin", ["spacemouse", "gamepad"]) + def test_device_plugin_is_monitored(self, tmp_path, plugin): + manager = MockPluginManager(plugin_names=[plugin]) manager.get_plugin_info = MagicMock( return_value=SimpleNamespace( - name="spacemouse", + name=plugin, devices=( - SimpleNamespace( - path="/spacemouse", type="spacemouse", description="SpaceMouse" - ), + SimpleNamespace(path=f"/{plugin}", type=plugin, description=plugin), ), ) ) @@ -1579,8 +1578,8 @@ def test_spacemouse_plugin_is_monitored(self, tmp_path): MockPipeline(leaf_nodes=[]), plugins=[ PluginConfig( - plugin_name="spacemouse", - plugin_root_id="spacemouse", + plugin_name=plugin, + plugin_root_id=plugin, search_paths=[tmp_path], ) ], @@ -1593,8 +1592,8 @@ def test_spacemouse_plugin_is_monitored(self, tmp_path): mock_session_dependencies(mock_pm=manager, collected_trackers=[]), ): with TeleopSession(config) as session: - tracker_cls.assert_called_once_with("spacemouse/device_status") - assert session.get_device_status("spacemouse/spacemouse") is not None + tracker_cls.assert_called_once_with(f"{plugin}/device_status") + assert session.get_device_status(f"{plugin}/{plugin}") is not None def test_only_manus_is_monitored_and_plugin_launch_order_is_preserved( self, tmp_path From af35e20b270664d3ecaac28732a0e41ab9e9dbd4 Mon Sep 17 00:00:00 2001 From: Rafael Wiltz Date: Tue, 13 Oct 2026 10:45:28 -0400 Subject: [PATCH 4/4] Add GamepadSource and gamepad retargeters GamepadSource reads the generated GamepadTracker and outputs a gamepad tensor group (GamepadInput, indexed by GamepadInputIndex) with the fields of GamepadInputState, absent while the plugin is not pushing or has no pad open. GamepadToSe3RelRetargeter and GamepadToSe2Retargeter bind the sticks and D-pad by position, so they drive the same way from any supported pad, to an end-effector delta and a base velocity. Their dead zone defaults to 0.15, since an idle pad's sticks rest up to 0.14 off center. GamepadGripperRetargeter toggles on the west face button (X on Xbox, Square on PlayStation) and lives with the other gripper retargeters, so it needs no scipy. Adds a printer example, which starts the bundled plugin and shows the raw input, the retargeters' outputs and the poses a downstream app would integrate from them, and the device page. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Rafael Wiltz --- docs/source/_data/devices.yaml | 16 ++ docs/source/device/gamepad.rst | 196 ++++++++++++++++ docs/source/index.rst | 1 + .../teleop/python/gamepad_printer_example.py | 221 ++++++++++++++++++ .../isaaccapture/retargeters/__init__.py | 35 +++ .../retargeters/gamepad_se2_retargeter.py | 99 ++++++++ .../retargeters/gamepad_se3_retargeter.py | 110 +++++++++ .../retargeters/gripper_retargeter.py | 52 ++++- .../deviceio_source_nodes/__init__.py | 6 + .../deviceio_tensor_types.py | 19 ++ .../deviceio_source_nodes/gamepad_source.py | 127 ++++++++++ .../tensor_types/__init__.py | 4 + .../tensor_types/indices.py | 4 + .../tensor_types/standard_types.py | 36 +++ .../test_gamepad_retargeter.py | 160 +++++++++++++ .../retargeting_engine/test_gamepad_source.py | 64 +++++ 16 files changed, 1149 insertions(+), 1 deletion(-) create mode 100644 docs/source/device/gamepad.rst create mode 100644 examples/teleop/python/gamepad_printer_example.py create mode 100644 src/python/isaaccapture/retargeters/gamepad_se2_retargeter.py create mode 100644 src/python/isaaccapture/retargeters/gamepad_se3_retargeter.py create mode 100644 src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/gamepad_source.py create mode 100644 tests/python/core/retargeting_engine/test_gamepad_retargeter.py create mode 100644 tests/python/core/retargeting_engine/test_gamepad_source.py diff --git a/docs/source/_data/devices.yaml b/docs/source/_data/devices.yaml index 3c9485de6c..b057eb5838 100644 --- a/docs/source/_data/devices.yaml +++ b/docs/source/_data/devices.yaml @@ -317,6 +317,22 @@ devices: - label: 3dconnexion.com url: https://3dconnexion.com/ + - id: gamepad + name: Gamepads + url: https://github.com/mdqinc/SDL_GameControllerDB + group: peripheral + modes: 6-DoF end effector, base velocity + since: main + details: + setup: + - label: Gamepad Plugin + url: https://github.com/NVIDIA/IsaacCapture/tree/main/src/plugins/gamepad + - label: Gamepad docs + doc: /device/gamepad + requirements: + - Linux workstation with the pad on USB or Bluetooth + - Xbox, PlayStation, Switch Pro, Steam or any pad SDL_GameControllerDB maps + - id: oak-d-camera name: OAK-D Camera url: https://shop.luxonis.com/products/oak-d diff --git a/docs/source/device/gamepad.rst b/docs/source/device/gamepad.rst new file mode 100644 index 0000000000..e3a1a1c945 --- /dev/null +++ b/docs/source/device/gamepad.rst @@ -0,0 +1,196 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Gamepad Plugin +============== + +C++ plugin that streams a **gamepad** (USB or Bluetooth) into the standard DeviceIO path in one +layout for every pad: buttons named by position and the two sticks and triggers. It reads the pad +through `SDL3 `_, whose drivers and +`SDL_GameControllerDB `_ mappings resolve each pad's +own numbering, and pushes the state at 90 Hz. Source and plugin README: +:code-file:`src/plugins/gamepad/README.md`. + +Supported pads: Xbox (360, One, Series, USB and Bluetooth), PlayStation (DualShock 4, DualSense), +Nintendo Switch Pro, Steam Controller, and any pad with an SDL_GameControllerDB mapping. Validated: +Xbox 360 Controller. + +This plugin follows the :doc:`add_device` pattern (FlatBuffer schema → plugin ``SchemaPusher`` → +tracker → MCAP). + +.. contents:: On this page + :local: + :depth: 2 + +Components +---------- + +- **Schema** — :code-file:`src/core/schema/fbs/gamepad.fbs`: ``GamepadInputState``, the standard + layout as named fields, inside ``GamepadSnapshot`` / ``GamepadSnapshotRecord``. +- **Plugin** — :code-dir:`src/plugins/gamepad`: ``core/`` reads the pad through SDL3 and reopens it + after an unplug; ``app/`` is the OpenXR push loop. +- **Tracker** — ``GamepadTracker``, generated at configure time from the ``gamepad`` entry in + :code-file:`src/core/deviceio_trackers/trackers.toml`. +- **Source node** — ``GamepadSource`` turns the tracker's data into a ``gamepad`` tensor group + (``GamepadInput``, indexed by ``GamepadInputIndex``) for retargeters. + +Standard layout +--------------- + +``GamepadInputState`` is a generic controller split in two, with the same naming as +``ControllerInputState``. Face buttons are named by position, so a binding means the same on every +pad: + +.. list-table:: + :header-rows: 1 + :widths: 25 25 25 25 + + * - Field + - Xbox + - PlayStation + - Nintendo + * - ``south_click`` + - A + - Cross + - B + * - ``east_click`` + - B + - Circle + - A + * - ``west_click`` + - X + - Square + - Y + * - ``north_click`` + - Y + - Triangle + - X + * - ``view_click`` / ``menu_click`` + - View / Menu + - Create / Options + - − / + + +The other buttons are ``shoulder_[left|right]_click``, ``thumbstick_[left|right]_click`` and +``dpad_[up|down|left|right]_click``; the axes are ``thumbstick_[left|right]_[x|y]`` in ``[-1, 1]`` +(right and up positive, as in OpenXR) and ``trigger_[left|right]_value`` in ``[0, 1]``. These are the +inputs every standard pad has, the same set as OpenXR's ``xbox_controller`` profile; pad-specific +extras such as the guide button, paddles and touchpad are not reported. + +Device access +------------- + +On a desktop session udev grants the logged-in user access to gamepad nodes, so no setup is +needed. Without a local seat (SSH, a service, a container), add the user to the ``input`` group, +and pass the pad's ``/dev/input/event*`` node into a container. + +Build +----- + +Linux only. The plugin builds with the default plugin set and fetches SDL3 (zlib license) at +configure time, linking it statically with only its joystick and HIDAPI subsystems: + +.. code-block:: bash + + cmake -B build + cmake --build build --target gamepad_plugin --parallel + +Usage +----- + +Start the plugin with the session through ``TeleopSessionConfig.plugins``, from the copy bundled +with ``isaaccapture`` (wheel installs included), and read it with ``GamepadSource``, as +:code-file:`examples/teleop/python/gamepad_printer_example.py` does: + +.. code-block:: python + + from isaaccapture.retargeting_engine.deviceio_source_nodes import GamepadSource + from isaaccapture.teleop_session_manager import ( + BUNDLED_PLUGINS_DIR, + PluginConfig, + TeleopSessionConfig, + ) + + config = TeleopSessionConfig( + app_name="MyApp", + pipeline=pipeline_reading(GamepadSource(name="gamepad")), + plugins=[ + PluginConfig( + plugin_name="gamepad", + plugin_root_id="gamepad", + search_paths=[BUNDLED_PLUGINS_DIR], + ) + ], + ) + +The plugin reads the first connected pad, keeps it until it disconnects, and then takes the first +one again. Its name, type and USB IDs are logged when it is opened. + +The plugin can also run on its own, for example on another machine pushing to the same runtime, +next to any reader of the same collection: + +.. code-block:: bash + + ./install/plugins/gamepad/gamepad_plugin + +Multiple pads +~~~~~~~~~~~~~ + +Run one plugin per pad. ``--player=N`` reads the Nth connected pad, and each plugin pushes to its +own collection, read by a ``GamepadSource`` with the same ``collection_id``: + +.. code-block:: python + + plugins = [ + PluginConfig( + plugin_name="gamepad", + plugin_root_id=f"gamepad_{n}", + search_paths=[BUNDLED_PLUGINS_DIR], + plugin_args=[f"--player={n}", f"--collection-id=gamepad_{n}"], + ) + for n in (1, 2) + ] + sources = [GamepadSource(name=f"gamepad_{n}", collection_id=f"gamepad_{n}") for n in (1, 2)] + +Published data +-------------- + +The plugin pushes ``GamepadSnapshot`` to collection ``gamepad`` (``--collection-id`` changes it; +match it in ``GamepadSource`` / ``GamepadTracker``): + +- ``inputs`` — the ``GamepadInputState`` above. + +While no pad is open the plugin keeps pushing snapshots without ``inputs``, so a +reader never holds the last stick deflection of an unplugged pad; ``GamepadSource`` then reports +its outputs as absent. Recordings use MCAP channels ``gamepad`` and ``gamepad_tracked``. + +Retargeters +----------- + +``GamepadToSe3RelRetargeter`` (end-effector delta from the sticks and D-pad), +``GamepadToSe2Retargeter`` (base velocity from the sticks) and ``GamepadGripperRetargeter`` +(toggle on ``WEST``) consume ``GamepadSource``. Their dead zone defaults to 0.15: an idle pad's +sticks rest up to 0.14 off center. + +Device status +------------- + +When ``TeleopSession`` starts the plugin, it reports device ``/gamepad`` +(``gamepad/gamepad`` by default) through :doc:`../references/device_provider_monitoring`: +``CONNECTED``, ``DISCONNECTED`` when no pad is connected for its player, or ``FAILED`` when SDL +finds the pad but cannot open it. + +Troubleshooting +--------------- + +- **Status DISCONNECTED with the pad connected** — SDL lists a device as a gamepad only when it + has a mapping. Find the pad's line in SDL_GameControllerDB (or write one) and set it in the + ``SDL_GAMECONTROLLERCONFIG`` environment variable of the process that starts the plugin. +- **Status DISCONNECTED over SSH or in a container** — the user cannot read the pad's + ``/dev/input/event*`` node; see `Device access`_. +- **The robot drifts with the pad at rest** — raise the retargeter's ``dead_zone``. + +Tests +----- + +``gamepad_plugin_tests`` reads SDL virtual gamepads: the standard layout, stick and trigger +ranges, a pad plugged in late, an unplug, and player selection. diff --git a/docs/source/index.rst b/docs/source/index.rst index e817825e86..94ab5e5a25 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -73,6 +73,7 @@ Table of Contents device/sharpa_avatar device/haptikos device/spacemouse + device/gamepad .. toctree:: :maxdepth: 2 diff --git a/examples/teleop/python/gamepad_printer_example.py b/examples/teleop/python/gamepad_printer_example.py new file mode 100644 index 0000000000..5842cbf3a2 --- /dev/null +++ b/examples/teleop/python/gamepad_printer_example.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Gamepad Printer Example. + +Prints the gamepad's sticks, triggers and held buttons each frame, via GamepadSource's "gamepad" +output, and what the gamepad retargeters make of them: the end-effector delta +(GamepadToSe3RelRetargeter), the base velocity (GamepadToSe2Retargeter) and the gripper command +(GamepadGripperRetargeter, toggled by the west face button: X on Xbox, Square on PlayStation). +Buttons print by their field name in the standard layout, so the names are the same on every pad. + +Like a downstream app driving a robot, it also integrates the end-effector delta into an absolute +pose and the base velocity into a base pose, treating both as rates per second. Menu resets them. +Stick drift above --dead-zone shows up as a pose that creeps with the pad at rest. + +The session starts the gamepad plugin bundled with isaaccapture; nothing else needs to run. Like +every TeleopSession it opens an OpenXR session; the CloudXR launcher arguments start a runtime, and +no headset needs to connect. GamepadToSe3RelRetargeter needs scipy (the retargeters-lite extra). +""" + +import argparse +import math +import sys +import time + +import numpy as np +from scipy.spatial.transform import Rotation + +from isaaccapture.cloudxr import CloudXRLauncher +from isaaccapture.retargeters import ( + GamepadGripperRetargeter, + GamepadToSe2Retargeter, + GamepadToSe2RetargeterConfig, + GamepadToSe3RelRetargeter, + GamepadToSe3RelRetargeterConfig, +) +from isaaccapture.retargeting_engine.deviceio_source_nodes import GamepadSource +from isaaccapture.retargeting_engine.interface import OutputCombiner +from isaaccapture.retargeting_engine.tensor_types import GamepadInputIndex +from isaaccapture.teleop_session_manager import ( + BUNDLED_PLUGINS_DIR, + PluginConfig, + TeleopSession, + TeleopSessionConfig, +) + +STATUS_LINES = 3 + + +def build_pipeline(dead_zone: float) -> OutputCombiner: + """The source's raw outputs next to the SE(3), SE(2) and gripper retargeters' outputs.""" + source = GamepadSource(name="gamepad") + gamepad = {"gamepad": source.output("gamepad")} + se3 = GamepadToSe3RelRetargeter( + GamepadToSe3RelRetargeterConfig(dead_zone=dead_zone), name="se3" + ) + se2 = GamepadToSe2Retargeter( + GamepadToSe2RetargeterConfig(dead_zone=dead_zone), name="se2" + ) + gripper = GamepadGripperRetargeter(name="gripper") + return OutputCombiner( + { + "gamepad": source.output("gamepad"), + "ee_delta": se3.connect(gamepad).output("ee_delta"), + "base_command": se2.connect(gamepad).output("base_command"), + "gripper_command": gripper.connect(gamepad).output("gripper_command"), + } + ) + + +class IntegratedPoses: + """End-effector and base poses accumulated from the retargeters' rate outputs.""" + + def __init__(self) -> None: + self.reset() + + def reset(self) -> None: + self.ee_position = np.zeros(3) + self.ee_rotation = Rotation.identity() + self.base = np.zeros(3) # x, y, yaw + + def step(self, ee_delta, base_command, dt: float) -> None: + delta = np.asarray(ee_delta, dtype=np.float64) + self.ee_position += delta[:3] * dt + # Left-multiplied: the delta rotates about the base frame's axes, as a relative IK + # controller applies it. + self.ee_rotation = Rotation.from_rotvec(delta[3:] * dt) * self.ee_rotation + + v_x, v_y, omega_z = np.asarray(base_command, dtype=np.float64) + yaw = self.base[2] + self.base[0] += (v_x * math.cos(yaw) - v_y * math.sin(yaw)) * dt + self.base[1] += (v_x * math.sin(yaw) + v_y * math.cos(yaw)) * dt + self.base[2] = math.remainder(yaw + omega_z * dt, math.tau) + + def describe(self) -> str: + rpy = self.ee_rotation.as_euler("XYZ", degrees=True) + return ( + f"EE pose: pos [{_fmt(self.ee_position)}] rpy [{_fmt(rpy, 1)}] deg " + f"Base pose: [{_fmt(self.base[:2])}] yaw {math.degrees(self.base[2]):+.1f} deg" + ) + + +# GamepadInput's buttons, printed without the "_click" suffix, and its sticks and triggers. +BUTTONS = [i for i in GamepadInputIndex if i.name.endswith("_CLICK")] +AXES = [i for i in GamepadInputIndex if not i.name.endswith("_CLICK")] + + +def _name(button) -> str: + return button.name.removesuffix("_CLICK").lower() + + +def _fmt(values, decimals: int = 2) -> str: + return " ".join(f"{v:+.{decimals}f}" for v in values) + + +def _show(lines: list[str]) -> None: + """Rewrite the status block in place: move up to its first line, then print each line.""" + print( + f"\033[{STATUS_LINES - 1}F" + "\n".join(f"{line}\033[K" for line in lines), + end="", + flush=True, + ) + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--player", + type=int, + default=1, + help="Read the Nth connected gamepad (default: the first)", + ) + parser.add_argument( + "--dead-zone", + type=float, + default=GamepadToSe3RelRetargeterConfig().dead_zone, + help="Stick deflection below which the retargeters read zero", + ) + parser.add_argument("--duration", type=float, default=30.0, help="Seconds to run") + CloudXRLauncher.add_launcher_arguments(parser) + args = parser.parse_args() + + session_config = TeleopSessionConfig( + app_name="GamepadPrinterExample", + pipeline=build_pipeline(args.dead_zone), + plugins=[ + PluginConfig( + plugin_name="gamepad", + plugin_root_id="gamepad", + search_paths=[BUNDLED_PLUGINS_DIR], + plugin_args=[f"--player={args.player}"], + required=True, + ) + ], + ) + + print( + "Move the sticks or D-pad, or press a button. west toggles the gripper, menu resets the poses." + ) + print("\n" * (STATUS_LINES - 1), end="") + poses = IntegratedPoses() + with CloudXRLauncher.launch_context(args), TeleopSession(session_config) as session: + start_time = time.time() + last_step = time.monotonic() + prev_pressed: set = set() + + while time.time() - start_time < args.duration: + result = session.step() + now = time.monotonic() + poses.step( + result["ee_delta"][0], result["base_command"][0], now - last_step + ) + last_step = now + + elapsed = session.get_elapsed_time() + gripper = "closed" if result["gripper_command"][0] < 0 else "open" + retargeted = ( + f"EE delta: [{_fmt(result['ee_delta'][0])}] " + f"Base cmd: [{_fmt(result['base_command'][0])}] Gripper: {gripper}" + ) + if result["gamepad"].is_none: + _show([f"[{elapsed:5.1f}s] (no gamepad)", retargeted, poses.describe()]) + time.sleep(0.01) + continue + + gamepad = result["gamepad"] + pressed = {b for b in BUTTONS if float(gamepad[b]) > 0.5} + if GamepadInputIndex.MENU_CLICK in pressed - prev_pressed: + poses.reset() + held = " ".join(_name(i) for i in sorted(pressed)) or "-" + _show( + [ + f"[{elapsed:5.1f}s] Axes: [{_fmt(float(gamepad[a]) for a in AXES)}] Held: {held}", + retargeted, + poses.describe(), + ] + ) + # Log every transition above the status block: a quick tap can flash by in it. + events = [ + f"[{elapsed:5.1f}s] {_name(i)} down" + for i in sorted(pressed - prev_pressed) + ] + events += [ + f"[{elapsed:5.1f}s] {_name(i)} up" + for i in sorted(prev_pressed - pressed) + ] + if events: + print("\n" + "\n".join(events) + "\n" * STATUS_LINES, end="") + prev_pressed = pressed + + time.sleep(0.01) + + print("\nDone.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/python/isaaccapture/retargeters/__init__.py b/src/python/isaaccapture/retargeters/__init__.py index f5e5ee71eb..518fb9b2bf 100644 --- a/src/python/isaaccapture/retargeters/__init__.py +++ b/src/python/isaaccapture/retargeters/__init__.py @@ -23,6 +23,9 @@ - SpaceMouseToSe3RelRetargeter: SpaceMouse axes -> relative EE delta control - SpaceMouseGripperRetargeter: SpaceMouse left-button toggle -> gripper open/closed - SpaceMouseToSe2Retargeter: SpaceMouse axes -> base velocity command (v_x, v_y, omega_z) + - GamepadToSe3RelRetargeter: Gamepad stick/dpad axes -> relative EE delta control + - GamepadGripperRetargeter: Gamepad X-button toggle -> gripper open/closed + - GamepadToSe2Retargeter: Gamepad stick axes -> base velocity command (v_x, v_y, omega_z) - SO101ClutchRetargeter: Clutch-rebased absolute EE pose for the SO-101 5-DOF arm -- re-latches BOTH home position and orientation on every engage, base-frame left-composed, no fixed offset @@ -136,6 +139,11 @@ "SpaceMouseGripperRetargeter", None, ), + "GamepadGripperRetargeter": ( + ".gripper_retargeter", + "GamepadGripperRetargeter", + None, + ), "KeyboardGripperRetargeter": ( ".gripper_retargeter", "KeyboardGripperRetargeter", @@ -174,6 +182,28 @@ "SpaceMouseToSe2RetargeterConfig", None, ), + # .gamepad_se3_retargeter (requires retargeters-lite extra: scipy) + "GamepadToSe3RelRetargeter": ( + ".gamepad_se3_retargeter", + "GamepadToSe3RelRetargeter", + "retargeters-lite", + ), + "GamepadToSe3RelRetargeterConfig": ( + ".gamepad_se3_retargeter", + "GamepadToSe3RelRetargeterConfig", + "retargeters-lite", + ), + # .gamepad_se2_retargeter + "GamepadToSe2Retargeter": ( + ".gamepad_se2_retargeter", + "GamepadToSe2Retargeter", + None, + ), + "GamepadToSe2RetargeterConfig": ( + ".gamepad_se2_retargeter", + "GamepadToSe2RetargeterConfig", + None, + ), # .keyboard_se2_retargeter "KeyboardToSe2Retargeter": ( ".keyboard_se2_retargeter", @@ -369,6 +399,11 @@ def __getattr__(name: str): "SpaceMouseGripperRetargeter", "SpaceMouseToSe2Retargeter", "SpaceMouseToSe2RetargeterConfig", + "GamepadToSe3RelRetargeter", + "GamepadToSe3RelRetargeterConfig", + "GamepadGripperRetargeter", + "GamepadToSe2Retargeter", + "GamepadToSe2RetargeterConfig", # SO-101 5-DOF arm retargeters "SO101ClutchRetargeter", "SO101GripperRetargeter", diff --git a/src/python/isaaccapture/retargeters/gamepad_se2_retargeter.py b/src/python/isaaccapture/retargeters/gamepad_se2_retargeter.py new file mode 100644 index 0000000000..02b061cafa --- /dev/null +++ b/src/python/isaaccapture/retargeters/gamepad_se2_retargeter.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Gamepad SE2 Retargeter Module. + +Maps raw gamepad axis state to a base velocity command (v_x, v_y, omega_z). +""" + +from dataclasses import dataclass + +import numpy as np + +from isaaccapture.retargeting_engine.interface import ( + BaseRetargeter, + RetargeterIOType, +) +from isaaccapture.retargeting_engine.interface.retargeter_core_types import RetargeterIO +from isaaccapture.retargeting_engine.interface.tensor_group_type import ( + OptionalType, + TensorGroupType, +) +from isaaccapture.retargeting_engine.tensor_types import ( + DLDataType, + GamepadInput, + GamepadInputIndex, + NDArrayType, +) + + +@dataclass +class GamepadToSe2RetargeterConfig: + """Configuration for the gamepad-to-SE2 base-velocity retargeter.""" + + v_x_sensitivity: float = 1.0 + v_y_sensitivity: float = 1.0 + omega_z_sensitivity: float = 1.0 + # An idle Xbox 360 pad's stick drift reaches 0.14. + dead_zone: float = 0.15 + + +class GamepadToSe2Retargeter(BaseRetargeter): + """ + Maps gamepad stick state to a 3D base velocity command (v_x, v_y, omega_z). + + Stick bindings: + Left stick up/down: +/-v_x Left stick right/left: +/-v_y + Right stick right/left: +/-omega_z + + Output is the instantaneous command implied by the current stick deflection + (scaled by sensitivity), not an integrated velocity -- matching a continuous-axis + input device. + """ + + def __init__(self, config: GamepadToSe2RetargeterConfig, name: str) -> None: + self._config = config + super().__init__(name=name) + + def input_spec(self) -> RetargeterIOType: + return {"gamepad": OptionalType(GamepadInput())} + + def output_spec(self) -> RetargeterIOType: + return { + "base_command": TensorGroupType( + "base_command", + [ + NDArrayType( + "velocity", shape=(3,), dtype=DLDataType.FLOAT, dtype_bits=32 + ) + ], + ) + } + + def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> None: + base_command = outputs["base_command"] + gamepad = inputs["gamepad"] + if gamepad.is_none: + base_command[0] = np.zeros(3, dtype=np.float32) + return + + dead_zone = self._config.dead_zone + + def deadzoned(value: float) -> float: + return 0.0 if abs(value) < dead_zone else value + + v_x = ( + deadzoned(float(gamepad[GamepadInputIndex.THUMBSTICK_LEFT_Y])) + * self._config.v_x_sensitivity + ) + v_y = ( + deadzoned(float(gamepad[GamepadInputIndex.THUMBSTICK_LEFT_X])) + * self._config.v_y_sensitivity + ) + omega_z = ( + deadzoned(float(gamepad[GamepadInputIndex.THUMBSTICK_RIGHT_X])) + * self._config.omega_z_sensitivity + ) + + base_command[0] = np.array([v_x, v_y, omega_z], dtype=np.float32) diff --git a/src/python/isaaccapture/retargeters/gamepad_se3_retargeter.py b/src/python/isaaccapture/retargeters/gamepad_se3_retargeter.py new file mode 100644 index 0000000000..127b362db0 --- /dev/null +++ b/src/python/isaaccapture/retargeters/gamepad_se3_retargeter.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Gamepad SE3 Retargeter Module. + +Maps raw gamepad axis state to end-effector delta commands. +""" + +from dataclasses import dataclass + +import numpy as np +from scipy.spatial.transform import Rotation + +from isaaccapture.retargeting_engine.interface import ( + BaseRetargeter, + RetargeterIOType, +) +from isaaccapture.retargeting_engine.interface.retargeter_core_types import RetargeterIO +from isaaccapture.retargeting_engine.interface.tensor_group_type import ( + OptionalType, + TensorGroupType, +) +from isaaccapture.retargeting_engine.tensor_types import ( + DLDataType, + GamepadInput, + GamepadInputIndex, + NDArrayType, +) + + +@dataclass +class GamepadToSe3RelRetargeterConfig: + """Configuration for the gamepad-to-SE3-relative retargeter.""" + + pos_sensitivity: float = 0.4 + rot_sensitivity: float = 0.8 + # An idle Xbox 360 pad's stick drift reaches 0.14. + dead_zone: float = 0.15 + + +class GamepadToSe3RelRetargeter(BaseRetargeter): + """ + Maps gamepad stick/dpad state to a 6D end-effector delta command. + + Stick/D-pad bindings: + Left stick up/down: +/-X, Left stick right/left: +/-Y, + Right stick up/down: +/-Z (position) + D-pad left/right: +/-roll, D-pad down/up: +/-pitch, + Right stick right/left: +/-yaw (rotation) + + Output is the instantaneous command implied by the current stick/dpad deflection + (scaled by sensitivity), not an integrated delta -- matching a continuous-axis + input device. + """ + + def __init__(self, config: GamepadToSe3RelRetargeterConfig, name: str) -> None: + self._config = config + super().__init__(name=name) + + def input_spec(self) -> RetargeterIOType: + return {"gamepad": OptionalType(GamepadInput())} + + def output_spec(self) -> RetargeterIOType: + return { + "ee_delta": TensorGroupType( + "ee_delta", + [ + NDArrayType( + "delta", shape=(6,), dtype=DLDataType.FLOAT, dtype_bits=32 + ) + ], + ) + } + + def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> None: + ee_delta = outputs["ee_delta"] + gamepad = inputs["gamepad"] + if gamepad.is_none: + ee_delta[0] = np.zeros(6, dtype=np.float32) + return + + def value(field) -> float: + return float(gamepad[field]) + + i = GamepadInputIndex + pos_sens = self._config.pos_sensitivity + rot_sens = self._config.rot_sensitivity + dead_zone = self._config.dead_zone + + def deadzoned(value: float) -> float: + return 0.0 if abs(value) < dead_zone else value + + delta_pos = np.zeros(3) + delta_pos[0] = deadzoned(value(i.THUMBSTICK_LEFT_Y)) * pos_sens + delta_pos[1] = deadzoned(value(i.THUMBSTICK_LEFT_X)) * pos_sens + delta_pos[2] = deadzoned(value(i.THUMBSTICK_RIGHT_Y)) * pos_sens + + delta_euler = np.zeros(3) + delta_euler[0] = ( + (value(i.DPAD_LEFT_CLICK) - value(i.DPAD_RIGHT_CLICK)) * rot_sens * 0.8 + ) + delta_euler[1] = ( + (value(i.DPAD_DOWN_CLICK) - value(i.DPAD_UP_CLICK)) * rot_sens * 0.8 + ) + delta_euler[2] = deadzoned(value(i.THUMBSTICK_RIGHT_X)) * rot_sens + + delta_rot = Rotation.from_euler("XYZ", delta_euler).as_rotvec() + + ee_delta[0] = np.concatenate([delta_pos, delta_rot]).astype(np.float32) diff --git a/src/python/isaaccapture/retargeters/gripper_retargeter.py b/src/python/isaaccapture/retargeters/gripper_retargeter.py index 4844ab8997..9dbf5bf20d 100644 --- a/src/python/isaaccapture/retargeters/gripper_retargeter.py +++ b/src/python/isaaccapture/retargeters/gripper_retargeter.py @@ -5,7 +5,8 @@ Gripper Retargeter Module. Gripper control from hand tracking or controller input (GripperRetargeter), and toggles from a -keyboard key (KeyboardGripperRetargeter) or a SpaceMouse button (SpaceMouseGripperRetargeter). +keyboard key (KeyboardGripperRetargeter), a SpaceMouse button (SpaceMouseGripperRetargeter) or a +gamepad button (GamepadGripperRetargeter). """ import numpy as np @@ -32,6 +33,8 @@ HandInputIndex, HandJointIndex, ControllerInputIndex, + GamepadInput, + GamepadInputIndex, ) @@ -225,3 +228,50 @@ def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> N self._closed = False gripper_out[0] = -1.0 if self._closed else 1.0 + + +class GamepadGripperRetargeter(BaseRetargeter): + """ + Toggles a gripper open/closed state on each press of the gamepad's west face button (X on an + Xbox pad, Square on a PlayStation pad). + + Buttons are sampled, so a press is its rising edge between frames. A reset reopens the + gripper, and a button held through the reset does not toggle again. + + Output matches GripperRetargeter's convention: -1.0 when closed, 1.0 when open. + """ + + TOGGLE_BUTTON = GamepadInputIndex.WEST_CLICK + + def __init__(self, name: str) -> None: + super().__init__(name=name) + self._closed = False + self._was_pressed = False + + def input_spec(self) -> RetargeterIOType: + return {"gamepad": OptionalType(GamepadInput())} + + def output_spec(self) -> RetargeterIOType: + return { + "gripper_command": TensorGroupType( + "gripper_command", [FloatType("command")] + ) + } + + def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> None: + gripper_out = outputs["gripper_command"] + gamepad = inputs["gamepad"] + + # Without a device the last edge state is kept, so a button still held when the + # device returns is not read as a new press. + if not gamepad.is_none: + pressed = float(gamepad[self.TOGGLE_BUTTON]) > 0.5 + if context.execution_events.reset: + self._closed = False + elif pressed and not self._was_pressed: + self._closed = not self._closed + self._was_pressed = pressed + elif context.execution_events.reset: + self._closed = False + + gripper_out[0] = -1.0 if self._closed else 1.0 diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py index e0c0deabd3..338327a763 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/__init__.py @@ -17,6 +17,7 @@ SpaceMouseSource, SpaceMouseTranslationType, ) +from .gamepad_source import GamepadSource from .keyboard_source import ( KeyboardHeldType, @@ -48,6 +49,7 @@ Generic3AxisPedalOutputTrackedType, KeyboardOutputTrackedType, SpaceMouseOutputTrackedType, + GamepadSnapshotTrackedType, JointStateOutputTrackedType, FullBodyPoseTrackedType, SomaBodyJointRotationsTrackedType, @@ -60,6 +62,7 @@ DeviceIOGeneric3AxisPedalOutputTracked, DeviceIOKeyboardOutputTracked, DeviceIOSpaceMouseOutputTracked, + DeviceIOGamepadSnapshotTracked, DeviceIOJointStateOutputTracked, DeviceIOFullBodyPoseTracked, DeviceIOSomaBodyJointRotationsTracked, @@ -85,6 +88,7 @@ "SpaceMouseRotationType", "SpaceMouseSource", "SpaceMouseTranslationType", + "GamepadSource", "KeyboardHeldType", "KeyboardPressedType", "KeyboardAttachment", @@ -112,6 +116,7 @@ "Generic3AxisPedalOutputTrackedType", "KeyboardOutputTrackedType", "SpaceMouseOutputTrackedType", + "GamepadSnapshotTrackedType", "JointStateOutputTrackedType", "FullBodyPoseTrackedType", "SomaBodyJointRotationsTrackedType", @@ -127,6 +132,7 @@ "DeviceIOGeneric3AxisPedalOutputTracked", "DeviceIOKeyboardOutputTracked", "DeviceIOSpaceMouseOutputTracked", + "DeviceIOGamepadSnapshotTracked", "DeviceIOJointStateOutputTracked", "DeviceIOFullBodyPoseTracked", "DeviceIOSomaBodyJointRotationsTracked", diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py index 474d11bedf..c062178318 100644 --- a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/deviceio_tensor_types.py @@ -21,6 +21,7 @@ Generic3AxisPedalOutput, KeyboardOutput, SpaceMouseOutput, + GamepadSnapshot, JointStateOutput, FullBodyPose, SomaBodyJointPoses, @@ -110,6 +111,12 @@ class SpaceMouseOutputTrackedType(_PayloadTensorType): _payload_cls = SpaceMouseOutput +class GamepadSnapshotTrackedType(_PayloadTensorType): + """GamepadSnapshot payload from DeviceIO GamepadTracker.""" + + _payload_cls = GamepadSnapshot + + class JointStateOutputTrackedType(_PayloadTensorType): """JointStateOutput payload from DeviceIO JointStateTracker.""" @@ -238,6 +245,18 @@ def DeviceIOSpaceMouseOutputTracked() -> TensorGroupType: ) +def DeviceIOGamepadSnapshotTracked() -> TensorGroupType: + """Tracked gamepad snapshot from DeviceIO GamepadTracker. + + Contains: + gamepad_tracked: GamepadSnapshot handle, or None while the plugin is not pushing + """ + return TensorGroupType( + "deviceio_gamepad_snapshot", + [GamepadSnapshotTrackedType("gamepad_tracked")], + ) + + def DeviceIOJointStateOutputTracked() -> TensorGroupType: """Tracked joint-state data from DeviceIO JointStateTracker. diff --git a/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/gamepad_source.py b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/gamepad_source.py new file mode 100644 index 0000000000..30041c7292 --- /dev/null +++ b/src/python/isaaccapture/retargeting_engine/deviceio_source_nodes/gamepad_source.py @@ -0,0 +1,127 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Gamepad Source Node - DeviceIO to Retargeting Engine converter. + +Converts the GamepadSnapshot pushed by the gamepad plugin to the standard GamepadInput tensor group: +the buttons and axes every standard gamepad has, by name. The plugin resolves each pad's own +numbering, so west_click is X on an Xbox pad and Square on a PlayStation pad. + +Buttons are sampled: each frame reports the buttons held at the plugin's latest sample, so a press +and release between two samples is not seen. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from ..interface.retargeter_core_types import RetargeterIO, RetargeterIOType +from ..interface.tensor_group import TensorGroup +from ..interface.tensor_group_type import OptionalType +from ..tensor_types import GamepadInput, GamepadInputIndex +from .deviceio_tensor_types import DeviceIOGamepadSnapshotTracked +from .interface import IDeviceIOSource + +if TYPE_CHECKING: + from isaaccapture.deviceio_trackers import ITracker + from isaaccapture.schema import GamepadSnapshot + +# Default collection_id of the gamepad plugin and GamepadTracker. +DEFAULT_GAMEPAD_COLLECTION_ID = "gamepad" + + +class GamepadSource(IDeviceIOSource): + """ + Stateless converter: DeviceIO GamepadSnapshot → GamepadInput tensor group. + + Inputs: + - "deviceio_gamepad": GamepadSnapshot from GamepadTracker + + Outputs (Optional -- absent while the plugin is not running or has no gamepad open): + - "gamepad": GamepadInput, indexed by GamepadInputIndex + + The data comes from the gamepad plugin. Start it with the session, using the copy bundled with + isaaccapture:: + + PluginConfig( + plugin_name="gamepad", + plugin_root_id="gamepad", + search_paths=[BUNDLED_PLUGINS_DIR], + ) + + and pass ``--collection-id=`` in ``plugin_args`` for a non-default collection, + with ``--player=N`` to read the Nth connected pad. + """ + + def __init__( + self, name: str, collection_id: str = DEFAULT_GAMEPAD_COLLECTION_ID + ) -> None: + """Initialize the source node and its GamepadTracker. + + Args: + name: Unique name for this source node. + collection_id: Tensor collection the gamepad plugin pushes to. + """ + from isaaccapture.deviceio_trackers import GamepadTracker + + self._gamepad_tracker = GamepadTracker(collection_id) + super().__init__(name) + + def get_tracker(self) -> "ITracker": + """Get the GamepadTracker instance for TeleopSession to register.""" + return self._gamepad_tracker + + def poll_tracker(self, deviceio_session: Any) -> RetargeterIO: + """Poll the gamepad tracker. + + Returns: + Dict with "deviceio_gamepad" TensorGroup containing GamepadSnapshot | None. + """ + state = self._gamepad_tracker.get_gamepad_data(deviceio_session) + tg = TensorGroup(DeviceIOGamepadSnapshotTracked()) + tg[0] = state + return {"deviceio_gamepad": tg} + + def input_spec(self) -> RetargeterIOType: + """Declare DeviceIO gamepad input.""" + return {"deviceio_gamepad": DeviceIOGamepadSnapshotTracked()} + + def output_spec(self) -> RetargeterIOType: + """Declare the gamepad output (Optional).""" + return {"gamepad": OptionalType(GamepadInput())} + + def _compute_fn(self, inputs: RetargeterIO, outputs: RetargeterIO, context) -> None: + """Convert GamepadSnapshot to GamepadInput, or set it to None without a pad.""" + snapshot: GamepadSnapshot | None = inputs["deviceio_gamepad"][0] + + group = outputs["gamepad"] + if snapshot is None or snapshot.inputs is None: + group.set_none() + return + + pad = snapshot.inputs + group[GamepadInputIndex.SOUTH_CLICK] = float(pad.south_click) + group[GamepadInputIndex.EAST_CLICK] = float(pad.east_click) + group[GamepadInputIndex.WEST_CLICK] = float(pad.west_click) + group[GamepadInputIndex.NORTH_CLICK] = float(pad.north_click) + group[GamepadInputIndex.VIEW_CLICK] = float(pad.view_click) + group[GamepadInputIndex.MENU_CLICK] = float(pad.menu_click) + group[GamepadInputIndex.SHOULDER_LEFT_CLICK] = float(pad.shoulder_left_click) + group[GamepadInputIndex.SHOULDER_RIGHT_CLICK] = float(pad.shoulder_right_click) + group[GamepadInputIndex.THUMBSTICK_LEFT_CLICK] = float( + pad.thumbstick_left_click + ) + group[GamepadInputIndex.THUMBSTICK_RIGHT_CLICK] = float( + pad.thumbstick_right_click + ) + group[GamepadInputIndex.DPAD_UP_CLICK] = float(pad.dpad_up_click) + group[GamepadInputIndex.DPAD_DOWN_CLICK] = float(pad.dpad_down_click) + group[GamepadInputIndex.DPAD_LEFT_CLICK] = float(pad.dpad_left_click) + group[GamepadInputIndex.DPAD_RIGHT_CLICK] = float(pad.dpad_right_click) + group[GamepadInputIndex.THUMBSTICK_LEFT_X] = float(pad.thumbstick_left_x) + group[GamepadInputIndex.THUMBSTICK_LEFT_Y] = float(pad.thumbstick_left_y) + group[GamepadInputIndex.THUMBSTICK_RIGHT_X] = float(pad.thumbstick_right_x) + group[GamepadInputIndex.THUMBSTICK_RIGHT_Y] = float(pad.thumbstick_right_y) + group[GamepadInputIndex.TRIGGER_LEFT_VALUE] = float(pad.trigger_left_value) + group[GamepadInputIndex.TRIGGER_RIGHT_VALUE] = float(pad.trigger_right_value) diff --git a/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py b/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py index 514b057465..2454c5c44d 100644 --- a/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py +++ b/src/python/isaaccapture/retargeting_engine/tensor_types/__init__.py @@ -12,6 +12,7 @@ SomaHandInput, HeadInput, ControllerInput, + GamepadInput, FullBodyInput, SomaBodyInput, TransformMatrix, @@ -37,6 +38,7 @@ SomaHandInputIndex, HeadInputIndex, ControllerInputIndex, + GamepadInputIndex, Generic3AxisPedalInputIndex, FullBodyInputIndex, SomaBodyInputIndex, @@ -59,6 +61,7 @@ "SomaHandInput", "HeadInput", "ControllerInput", + "GamepadInput", "FullBodyInput", "SomaBodyInput", "TransformMatrix", @@ -82,6 +85,7 @@ "SomaHandInputIndex", "HeadInputIndex", "ControllerInputIndex", + "GamepadInputIndex", "Generic3AxisPedalInputIndex", "FullBodyInputIndex", "SomaBodyInputIndex", diff --git a/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py b/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py index 40a954b7ac..32fa7abcea 100644 --- a/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py +++ b/src/python/isaaccapture/retargeting_engine/tensor_types/indices.py @@ -20,6 +20,7 @@ SomaHandInput, HeadInput, ControllerInput, + GamepadInput, Generic3AxisPedalInput, FullBodyInput, SomaBodyInput, @@ -46,6 +47,9 @@ def _create_index_enum(name: str, group_type, prefix: str = "") -> IntEnum: ControllerInputIndex: Any = _create_index_enum( "ControllerInputIndex", ControllerInput(), "controller_" ) +GamepadInputIndex: Any = _create_index_enum( + "GamepadInputIndex", GamepadInput(), "gamepad_" +) Generic3AxisPedalInputIndex: Any = _create_index_enum( "Generic3AxisPedalInputIndex", Generic3AxisPedalInput(), "pedal_" ) diff --git a/src/python/isaaccapture/retargeting_engine/tensor_types/standard_types.py b/src/python/isaaccapture/retargeting_engine/tensor_types/standard_types.py index ab16e30ad4..6b165f952d 100644 --- a/src/python/isaaccapture/retargeting_engine/tensor_types/standard_types.py +++ b/src/python/isaaccapture/retargeting_engine/tensor_types/standard_types.py @@ -235,6 +235,42 @@ def input_spec(self) -> RetargeterIO: ) +def GamepadInput() -> TensorGroupType: + """ + Standard TensorGroupType for gamepad data: a generic controller split in two. + + Mirrors GamepadInputState in gamepad.fbs. Buttons are 1.0 while held and 0.0 otherwise; face + buttons are named by position (south is A on Xbox, Cross on PlayStation). Thumbsticks are in + [-1, 1] with right and up positive, as in OpenXR; triggers are in [0, 1]. Index fields with + GamepadInputIndex, e.g. ``gamepad[GamepadInputIndex.WEST_CLICK]``. + """ + return TensorGroupType( + "gamepad", + [ + FloatType("gamepad_south_click"), + FloatType("gamepad_east_click"), + FloatType("gamepad_west_click"), + FloatType("gamepad_north_click"), + FloatType("gamepad_view_click"), + FloatType("gamepad_menu_click"), + FloatType("gamepad_shoulder_left_click"), + FloatType("gamepad_shoulder_right_click"), + FloatType("gamepad_thumbstick_left_click"), + FloatType("gamepad_thumbstick_right_click"), + FloatType("gamepad_dpad_up_click"), + FloatType("gamepad_dpad_down_click"), + FloatType("gamepad_dpad_left_click"), + FloatType("gamepad_dpad_right_click"), + FloatType("gamepad_thumbstick_left_x"), + FloatType("gamepad_thumbstick_left_y"), + FloatType("gamepad_thumbstick_right_x"), + FloatType("gamepad_thumbstick_right_y"), + FloatType("gamepad_trigger_left_value"), + FloatType("gamepad_trigger_right_value"), + ], + ) + + # ============================================================================ # Full Body Tracking Types # ============================================================================ diff --git a/tests/python/core/retargeting_engine/test_gamepad_retargeter.py b/tests/python/core/retargeting_engine/test_gamepad_retargeter.py new file mode 100644 index 0000000000..184fe3e3de --- /dev/null +++ b/tests/python/core/retargeting_engine/test_gamepad_retargeter.py @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the gamepad retargeters, fed through GamepadSource from real GamepadSnapshot payloads.""" + +import numpy as np +import pytest + +from isaaccapture.retargeters import ( + GamepadGripperRetargeter, + GamepadToSe2Retargeter, + GamepadToSe2RetargeterConfig, + GamepadToSe3RelRetargeter, + GamepadToSe3RelRetargeterConfig, +) +from isaaccapture.retargeting_engine.deviceio_source_nodes import GamepadSource +from isaaccapture.retargeting_engine.interface.base_retargeter import _make_output_group +from isaaccapture.retargeting_engine.interface.execution_events import ExecutionEvents +from isaaccapture.retargeting_engine.interface.retargeter_core_types import ( + ComputeContext, +) +from isaaccapture.retargeting_engine.interface.tensor_group import TensorGroup +from isaaccapture.schema import GamepadInputState, GamepadSnapshot + + +def _gamepad(absent=False, **fields): + """GamepadSource's "gamepad" output for a pad in this state, or for no pad when absent.""" + src = GamepadSource(name="gamepad") + tg = TensorGroup(src.input_spec()["deviceio_gamepad"]) + tg[0] = None if absent else GamepadSnapshot(GamepadInputState(**fields)) + outputs = {name: _make_output_group(gt) for name, gt in src.output_spec().items()} + src.compute({"deviceio_gamepad": tg}, outputs) + return outputs["gamepad"] + + +def _run(retargeter, output, gamepad, reset=False): + out = {output: _make_output_group(retargeter.output_spec()[output])} + context = ComputeContext(execution_events=ExecutionEvents(reset=reset)) + retargeter.compute({"gamepad": gamepad}, out, context) + return np.asarray(out[output][0]) + + +def _se3(): + return GamepadToSe3RelRetargeter(GamepadToSe3RelRetargeterConfig(), name="se3") + + +def _se2(): + return GamepadToSe2Retargeter(GamepadToSe2RetargeterConfig(), name="se2") + + +class TestGamepadToSe3RelRetargeter: + def test_left_stick_up_produces_forward_delta(self): + delta = _run(_se3(), "ee_delta", _gamepad(thumbstick_left_y=1.0)) + + assert delta[0] == pytest.approx(0.4) # default pos_sensitivity + assert np.allclose(delta[1:], 0.0) + + def test_sticks_combine_on_independent_axes(self): + delta = _run( + _se3(), "ee_delta", _gamepad(thumbstick_left_y=1.0, thumbstick_right_y=1.0) + ) + + assert delta[0] == pytest.approx(0.4) # left stick up: +X + assert delta[2] == pytest.approx(0.4) # right stick up: +Z + assert delta[1] == pytest.approx(0.0) + assert np.allclose(delta[3:], 0.0) + + @pytest.mark.parametrize("direction", [1.0, -1.0]) + def test_stick_right_is_positive_y_and_yaw(self, direction): + delta = _run( + _se3(), + "ee_delta", + _gamepad(thumbstick_left_x=direction, thumbstick_right_x=direction), + ) + + assert np.sign(delta[1]) == direction + assert np.sign(delta[5]) == direction + assert np.allclose(delta[[0, 2, 3, 4]], 0.0) + + def test_dpad_rotates_roll_and_pitch(self): + delta = _run( + _se3(), "ee_delta", _gamepad(dpad_left_click=True, dpad_down_click=True) + ) + + assert np.allclose(delta[:3], 0.0) + assert delta[3] > 0 and delta[4] > 0 + + def test_absent_pad_yields_zero_delta(self): + assert np.allclose(_run(_se3(), "ee_delta", _gamepad(absent=True)), 0.0) + + def test_dead_zone_suppresses_stick_drift(self): + assert np.allclose( + _run(_se3(), "ee_delta", _gamepad(thumbstick_left_y=0.1)), 0.0 + ) + + +class TestGamepadGripperRetargeter: + @staticmethod + def _step(gripper, reset=False, **fields): + return float(_run(gripper, "gripper_command", _gamepad(**fields), reset=reset)) + + def test_gripper_toggles_on_west_rising_edge_only(self): + gripper = GamepadGripperRetargeter(name="gripper") + + assert self._step(gripper) == pytest.approx(1.0) # open (default) + assert self._step(gripper, west_click=True) == pytest.approx(-1.0) # close + assert self._step(gripper, west_click=True) == pytest.approx(-1.0) # held + assert self._step(gripper) == pytest.approx(-1.0) # released: stays closed + assert self._step(gripper, west_click=True) == pytest.approx(1.0) # open + + def test_absent_pad_yields_default_open(self): + gripper = GamepadGripperRetargeter(name="gripper") + + assert self._step(gripper, absent=True) == pytest.approx(1.0) + + def test_reset_does_not_toggle_while_west_is_held(self): + gripper = GamepadGripperRetargeter(name="gripper") + + assert self._step(gripper, west_click=True) == pytest.approx(-1.0) + # Reset reopens the gripper; west still held is not a new press. + assert self._step(gripper, reset=True, west_click=True) == pytest.approx(1.0) + assert self._step(gripper, west_click=True) == pytest.approx(1.0) + assert self._step(gripper) == pytest.approx(1.0) + assert self._step(gripper, west_click=True) == pytest.approx(-1.0) + + def test_reset_with_absent_pad_keeps_the_edge_state(self): + gripper = GamepadGripperRetargeter(name="gripper") + + assert self._step(gripper, west_click=True) == pytest.approx(-1.0) + assert self._step(gripper, reset=True, absent=True) == pytest.approx(1.0) + # West was held throughout, so it is not read as a new press when the pad returns. + assert self._step(gripper, west_click=True) == pytest.approx(1.0) + + def test_other_buttons_do_not_affect_the_gripper(self): + gripper = GamepadGripperRetargeter(name="gripper") + + assert self._step( + gripper, shoulder_right_click=True, south_click=True + ) == pytest.approx(1.0) + + +class TestGamepadToSe2Retargeter: + def test_left_and_right_stick_combine(self): + velocity = _run( + _se2(), + "base_command", + _gamepad(thumbstick_left_y=1.0, thumbstick_right_x=0.5), + ) + + assert velocity[0] == pytest.approx(1.0) # default v_x_sensitivity + assert velocity[1] == pytest.approx(0.0) + assert velocity[2] == pytest.approx(0.5) # default omega_z_sensitivity + + def test_dead_zone_suppresses_stick_drift(self): + assert np.allclose( + _run(_se2(), "base_command", _gamepad(thumbstick_left_y=0.1)), 0.0 + ) + + def test_absent_pad_yields_zero_velocity(self): + assert np.allclose(_run(_se2(), "base_command", _gamepad(absent=True)), 0.0) diff --git a/tests/python/core/retargeting_engine/test_gamepad_source.py b/tests/python/core/retargeting_engine/test_gamepad_source.py new file mode 100644 index 0000000000..55c61a0d57 --- /dev/null +++ b/tests/python/core/retargeting_engine/test_gamepad_source.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the GamepadSource DeviceIO converter. + +Feeds GamepadSnapshot payloads built through the real schema bindings into the converter, with no +plugin or OpenXR runtime involved. +""" + +import pytest + +from isaaccapture.retargeting_engine.deviceio_source_nodes import GamepadSource +from isaaccapture.retargeting_engine.interface.base_retargeter import _make_output_group +from isaaccapture.retargeting_engine.interface.tensor_group import TensorGroup +from isaaccapture.retargeting_engine.tensor_types import GamepadInput, GamepadInputIndex +from isaaccapture.schema import GamepadInputState, GamepadSnapshot + + +def _run(state): + src = GamepadSource(name="gamepad") + tg = TensorGroup(src.input_spec()["deviceio_gamepad"]) + tg[0] = state + outputs = {name: _make_output_group(gt) for name, gt in src.output_spec().items()} + src.compute({"deviceio_gamepad": tg}, outputs) + return outputs["gamepad"] + + +def test_source_reads_the_generated_tracker(): + tracker = GamepadSource(name="gamepad").get_tracker() + assert tracker.get_name() == "GamepadTracker" + + +def test_the_group_has_one_field_per_schema_field(): + assert len(GamepadInput().types) == 20 + assert len(GamepadInputIndex) == 20 + + +def test_fields_pass_through_by_name(): + inputs = GamepadInputState( + west_click=True, + dpad_up_click=True, + thumbstick_left_y=1.0, + trigger_right_value=0.5, + ) + gamepad = _run(GamepadSnapshot(inputs)) + + held = { + i.name + for i in GamepadInputIndex + if i.name.endswith("_CLICK") and gamepad[i] > 0.5 + } + assert held == {"WEST_CLICK", "DPAD_UP_CLICK"} + assert float(gamepad[GamepadInputIndex.THUMBSTICK_LEFT_Y]) == pytest.approx(1.0) + assert float(gamepad[GamepadInputIndex.TRIGGER_RIGHT_VALUE]) == pytest.approx(0.5) + assert float(gamepad[GamepadInputIndex.THUMBSTICK_RIGHT_X]) == 0.0 + + +@pytest.mark.parametrize( + "state", + [None, GamepadSnapshot()], + ids=["plugin-not-pushing", "no-device-open"], +) +def test_no_device_yields_absent_output(state): + assert _run(state).is_none