diff --git a/artemis/agents/operator/operator.py b/artemis/agents/operator/operator.py index d2453627..f8985209 100644 --- a/artemis/agents/operator/operator.py +++ b/artemis/agents/operator/operator.py @@ -1252,6 +1252,20 @@ def target_fields(el: dict) -> dict: ) return actions, None + elif tool_name == "perform_gesture": + from artemis.mcp.gestures import validate_phases + + try: + phases = validate_phases(args.get("phases")) + description = args.get("target_description") + if not isinstance(description, str) or not description.strip(): + raise ValueError("target_description is required for a gesture") + except (ValueError, TypeError) as exc: + return [], f"Invalid gesture: {exc}" + return [ + {"action": "perform_gesture", "phases": phases, "target_description": description} + ], None + elif tool_name == "swipe": kind, target, parsed_duration = parse_swipe_parameters(args, default_duration=None) duration = args.get("duration") diff --git a/artemis/agents/operator/prompts.py b/artemis/agents/operator/prompts.py index e800de23..19c08d50 100644 --- a/artemis/agents/operator/prompts.py +++ b/artemis/agents/operator/prompts.py @@ -83,6 +83,7 @@ def load_operator_prompts() -> dict[str, str]: _TURN_ENDING_ORDER = ( "click", "swipe", + "perform_gesture", "input_text", "long_press", "press_key", diff --git a/artemis/agents/validator/execution_loop.py b/artemis/agents/validator/execution_loop.py index cd3b7a4a..f868dfa5 100644 --- a/artemis/agents/validator/execution_loop.py +++ b/artemis/agents/validator/execution_loop.py @@ -205,7 +205,9 @@ async def _process_action( if burst: max_local_retries = 1 else: - max_local_retries = 1 if action_name == "launch_app" else 2 + # A timed-out gesture may already be dispatched; never replay it. + # Keep the existing retry policy for every other action. + max_local_retries = 1 if action_name in ("launch_app", "perform_gesture") else 2 outcome.success, outcome.error_msg = await _attempt_local_execution( node, session, diff --git a/artemis/agents/validator/tool_declarations.py b/artemis/agents/validator/tool_declarations.py index fba8dd75..a8aed2c9 100644 --- a/artemis/agents/validator/tool_declarations.py +++ b/artemis/agents/validator/tool_declarations.py @@ -238,6 +238,8 @@ def normalize_coordinate_target(target: Any) -> int | list[int] | Any: SWIPE_TOOL = tool_declaration("swipe") +PERFORM_GESTURE_TOOL = tool_declaration("perform_gesture") + PRESS_KEY_TOOL = tool_declaration("press_key") READ_NOTE_TOOL = ToolDeclaration( @@ -324,6 +326,7 @@ def normalize_coordinate_target(target: Any) -> int | list[int] | Any: LONG_PRESS_TOOL, INPUT_TEXT_TOOL, SWIPE_TOOL, + PERFORM_GESTURE_TOOL, PRESS_KEY_TOOL, READ_NOTE_TOOL, LIST_NOTES_TOOL, diff --git a/artemis/clients/accessibility_client.py b/artemis/clients/accessibility_client.py index f14b8ab3..7df5aa35 100644 --- a/artemis/clients/accessibility_client.py +++ b/artemis/clients/accessibility_client.py @@ -413,6 +413,51 @@ def press_key(self, key: str) -> bool: def tap(self, x: float, y: float) -> bool: return self._rpc("tap", {"x": x, "y": y}) + def perform_gesture(self, phases: list[dict], request_id: str) -> dict[str, Any]: + from artemis.mcp.gestures import gesture_duration_bound_ms, validate_phases + + phases = validate_phases(phases) + info = json.loads(self._http("/ping").decode("utf-8")) + if "perform_gesture" not in info.get("capabilities", []): + return { + "success": False, + "status": "unsupported", + "error": "Upgrade Accessibility Helper: perform_gesture is not supported", + } + long_press_drag = phases[0].get("kind") == "long_press_drag" + if long_press_drag and "gesture_long_press_drag" not in info.get("capabilities", []): + return { + "success": False, + "status": "unsupported", + "error": "Upgrade Accessibility Helper: long-press dragging is not supported", + } + if any( + "control_points" in phase + or any("control_points" in p for p in phase.get("pointers", [])) + for phase in phases + ) and ("gesture_cubic_bezier" not in info.get("capabilities", [])): + return { + "success": False, + "status": "unsupported", + "error": "Upgrade Accessibility Helper: native cubic Bezier is not supported", + } + if (len(phases) > 1 or long_press_drag) and not info.get("gesture_continuation", False): + return { + "success": False, + "status": "unsupported", + "error": "Continuous gestures require Android 8.0+", + } + timeout = gesture_duration_bound_ms(phases) / 1000 + 10 + payload = {"cmd": "perform_gesture", "request_id": request_id, "phases": phases} + return json.loads(self._http("/action", payload, timeout=timeout).decode("utf-8")) + + def cancel_gesture(self, request_id: str) -> dict[str, Any]: + return json.loads( + self._http("/action", {"cmd": "cancel_gesture", "request_id": request_id}).decode( + "utf-8" + ) + ) + def swipe(self, x1: float, y1: float, x2: float, y2: float, duration_ms: int = 300) -> bool: return self._rpc("swipe", {"x1": x1, "y1": y1, "x2": x2, "y2": y2, "duration": duration_ms}) diff --git a/artemis/controllers/unified_controller.py b/artemis/controllers/unified_controller.py index d0dc1d03..2c14d49f 100644 --- a/artemis/controllers/unified_controller.py +++ b/artemis/controllers/unified_controller.py @@ -56,6 +56,16 @@ def __init__(self, ctx: ArtemisContext): super().__init__(ctx, get_driver(ctx)) self._segment_cache: dict[tuple[str, int, float, float], VideoRecordingResult] = {} + async def perform_gesture(self, phases: list[dict]) -> dict: + method = getattr(self._driver, "perform_gesture", None) + if method is None: + return { + "success": False, + "status": "unsupported", + "error": "Driver has no continuous multi-touch support", + } + return await method(phases) + @staticmethod async def _spawn_scrcpy(command: list[str]) -> asyncio.subprocess.Process: kwargs: dict[str, Any] = { diff --git a/artemis/drivers/android/adb_driver.py b/artemis/drivers/android/adb_driver.py index 360aa486..a8ec1ee9 100644 --- a/artemis/drivers/android/adb_driver.py +++ b/artemis/drivers/android/adb_driver.py @@ -254,6 +254,30 @@ async def tap( async def long_press(self, x: int, y: int, duration_ms: int = 1000) -> bool: return await self.tap(x=x, y=y, duration_ms=duration_ms) + async def perform_gesture(self, phases: list[dict]) -> dict: + from artemis.clients.accessibility_client import AccessibilityClient + from artemis.mcp.gestures import validate_phases + from uuid import uuid4 + + phases = validate_phases(phases) + client = AccessibilityClient(self.device_id, provision_on_connect=False) + request_id = uuid4().hex + task = asyncio.create_task(asyncio.to_thread(client.perform_gesture, phases, request_id)) + try: + return await asyncio.shield(task) + except asyncio.CancelledError: + # The action request is never replayed. Ask the device to finish the current + # phase and lift its held contacts, then wait for the terminal reply. + try: + await asyncio.shield(asyncio.to_thread(client.cancel_gesture, request_id)) + await asyncio.wait_for(asyncio.shield(task), timeout=10.0) + except Exception: + logger.warning( + "Gesture cancellation could not be confirmed; device needs reconciliation" + ) + task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None) + raise + async def swipe( self, start_x: int, diff --git a/artemis/drivers/mock/mock_driver.py b/artemis/drivers/mock/mock_driver.py index c4b12c44..b46c3336 100644 --- a/artemis/drivers/mock/mock_driver.py +++ b/artemis/drivers/mock/mock_driver.py @@ -106,6 +106,22 @@ async def long_press(self, x: int, y: int, duration_ms: int = 1000) -> bool: ) return True + async def perform_gesture(self, phases: list[dict]) -> dict: + from artemis.mcp.gestures import validate_phases + + phases = validate_phases(phases) + self.action_history.append({"action": "perform_gesture", "phases": phases}) + return { + "success": True, + "status": "completed", + "release_confirmed": True, + "phases_completed": ( + 2 + int(phases[0].get("release_delay_ms", 0) > 0) + if phases[0].get("kind") == "long_press_drag" + else len(phases) + ), + } + async def swipe( self, start_x: int, diff --git a/artemis/mcp/action_executor.py b/artemis/mcp/action_executor.py index 79b399cd..502d56d0 100644 --- a/artemis/mcp/action_executor.py +++ b/artemis/mcp/action_executor.py @@ -351,6 +351,20 @@ def _translate( recorded, ) + if raw_name == "perform_gesture": + from artemis.mcp.gestures import validate_phases + + try: + phases = validate_phases(args.get("phases")) + except (ValueError, TypeError) as exc: + raise _ArgError(f"Invalid gesture: {exc}") from exc + return ( + "perform_gesture", + {"phases": phases}, + None, + self._require_description(args, raw_name), + ) + if raw_name == "swipe": return self._translate_swipe(args, state, index_elements=index_elements) diff --git a/artemis/mcp/action_manifest.py b/artemis/mcp/action_manifest.py index 735024a8..0e808fa6 100644 --- a/artemis/mcp/action_manifest.py +++ b/artemis/mcp/action_manifest.py @@ -79,6 +79,7 @@ "long_press", "input_text", "swipe", + "perform_gesture", "press_key", "manage_app", "wait_for_delay", diff --git a/artemis/mcp/action_names.py b/artemis/mcp/action_names.py index 78cef0a5..dcc76d6f 100644 --- a/artemis/mcp/action_names.py +++ b/artemis/mcp/action_names.py @@ -35,6 +35,7 @@ "tap": "click", "long_press_on": "long_press", "swipe": "swipe", + "perform_gesture": "perform_gesture", "focus_and_input_text": "input_text", "focus_and_clear_text": "focus_and_clear_text", "erase_one_char": "erase_one_char", @@ -106,6 +107,11 @@ def to_canonical_call( "duration_ms": action_item.get("duration", 1000), } + if verb == "perform_gesture": + from artemis.mcp.gestures import validate_phases + + return "perform_gesture", {"phases": validate_phases(action_item.get("phases"))} + if verb == "swipe": coords = action_item.get("normalized_coordinates") if not (isinstance(coords, (list, tuple)) and len(coords) == 4): diff --git a/artemis/mcp/action_specs.py b/artemis/mcp/action_specs.py index 1ea69a11..7011abe5 100644 --- a/artemis/mcp/action_specs.py +++ b/artemis/mcp/action_specs.py @@ -64,7 +64,9 @@ from langchain_core.tools import StructuredTool from mcp.types import CallToolResult -from pydantic import Field, create_model +from pydantic import BaseModel, Field, create_model + +from artemis.mcp.gestures import GestureInput, validate_phases from artemis.core.tool_declaration import ToolDeclaration from artemis.mcp.action_types import ActionResult @@ -86,6 +88,21 @@ ] +GESTURE_DESCRIPTION = ( + "[ACTION] Perform continuous touch gestures that a normal swipe cannot express: " + "simultaneous multi-finger input, continuous holds and drags, or delayed release. " + "Coordinates use 0-1000 screen space. In explicit phases, pointers move together; " + "a one-point path holds still. Keep the same pointer IDs and join endpoints across " + "phases to maintain contact until the final release. Optional control_points " + "define a cubic Bezier path with two endpoints and two controls. " + "Alternatively, provide one long_press_drag entry with start and end coordinates " + "and optional control_points. The endpoint is freely chosen, including inside the screen. " + "duration_ms is movement time; " + "release_delay_ms is the endpoint hold before lifting (zero releases immediately). " + "Observe the screen afterward to verify the intended effect." +) + + # --- Spec structure ------------------------------------------------------------------ @@ -206,6 +223,10 @@ async def _wire_input_text(actuator: Any, a: dict[str, Any]) -> ActionResult: return await actuator.input_text(a["text"], norm, clear_exist=a["clear_exist"]) +async def _wire_perform_gesture(actuator: Any, a: dict[str, Any]) -> ActionResult: + return await actuator.perform_gesture(validate_phases(a["phases"])) + + async def _wire_swipe(actuator: Any, a: dict[str, Any]) -> ActionResult: start, end = a["start"], a["end"] return await actuator.swipe( @@ -463,6 +484,27 @@ async def _wire_focus_and_clear_text(actuator: Any, a: dict[str, Any]) -> Action " the wire; required only for coordinate targets." ), ), + ActionSpec( + name="perform_gesture", + operator=OperatorDialect( + description=GESTURE_DESCRIPTION + + " target_description states the intended surface/objects and gesture purpose, not a claimed observation.", + params=( + ParamSpec( + "phases", + list[GestureInput], + "Explicit continuous phases or a single native long_press_drag object.", + ), + ParamSpec("target_description", str, "Target surface/objects and gesture purpose."), + ), + ), + wire=WireDialect( + description=GESTURE_DESCRIPTION, + params=(ParamSpec("phases", list[GestureInput]),), + bind=_wire_perform_gesture, + ), + differences="target_description is recorded by the agent and is not sent to the helper.", + ), ActionSpec( name="swipe", operator=OperatorDialect( @@ -686,6 +728,7 @@ async def _wire_focus_and_clear_text(actuator: Any, a: dict[str, Any]) -> Action "click", "input_text", "swipe", + "perform_gesture", "press_key", "manage_app", "wait_for_delay", @@ -765,6 +808,20 @@ def _json_schema(annotation: Any) -> dict[str, Any]: ``required`` flag says so); they never appear in the projected schema, so the declaration carries no ``null`` alternatives or ``default`` noise. """ + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + schema = annotation.model_json_schema() + definitions = schema.get("$defs", {}) + + def inline(value): + if isinstance(value, list): + return [inline(v) for v in value] + if not isinstance(value, dict): + return value + if "$ref" in value: + return inline(definitions[value["$ref"].split("/")[-1]]) + return {k: inline(v) for k, v in value.items() if k not in ("$defs", "title")} + + return inline(schema) origin = get_origin(annotation) if origin in (Union, types.UnionType): members = [a for a in get_args(annotation) if a is not type(None)] diff --git a/artemis/mcp/actuators/adb.py b/artemis/mcp/actuators/adb.py index 4580e436..85fe3f94 100644 --- a/artemis/mcp/actuators/adb.py +++ b/artemis/mcp/actuators/adb.py @@ -186,6 +186,32 @@ async def long_press(self, nx: int, ny: int, duration_ms: int = 1000) -> ActionR duration_ms=duration_ms, ) + async def perform_gesture(self, phases: list[dict]) -> ActionResult: + from artemis.mcp.gestures import validate_phases + + phases = validate_phases(phases) + result = await self.controller.perform_gesture(phases) + if not ( + result.get("success") is True + and result.get("status") == "completed" + and result.get("release_confirmed") is True + ): + return ActionResult.failure( + "perform_gesture", + f"Gesture {result.get('status', 'failed')}: {result.get('error', 'input not completed')}", + code={ + "unsupported": ActionCode.UNSUPPORTED, + "invalid_arguments": ActionCode.INVALID_ARGS, + "timeout": ActionCode.TIMEOUT, + }.get(result.get("status"), ActionCode.DEVICE_ERROR), + detail=str(result), + ) + return ActionResult.success( + "perform_gesture", + "Gesture completed. Observe the screen to verify the intended effect.", + detail=str(result), + ) + async def input_text( self, text: str, diff --git a/artemis/mcp/actuators/base.py b/artemis/mcp/actuators/base.py index 1dd49553..f2b03576 100644 --- a/artemis/mcp/actuators/base.py +++ b/artemis/mcp/actuators/base.py @@ -75,6 +75,8 @@ async def input_text( clear_exist: bool = True, ) -> ActionResult: ... + async def perform_gesture(self, phases: list[dict]) -> ActionResult: ... + async def swipe( self, start: tuple[int, int], diff --git a/artemis/mcp/gestures.py b/artemis/mcp/gestures.py new file mode 100644 index 00000000..eee09e93 --- /dev/null +++ b/artemis/mcp/gestures.py @@ -0,0 +1,113 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Validated, executor-neutral continuous touch plans (0..1000 coordinates).""" + +from typing import Annotated, Any, Literal +from pydantic import BaseModel, ConfigDict, Field, StrictInt, model_validator + +Coordinate = Annotated[StrictInt, Field(ge=0, le=1000)] +Point = Annotated[list[Coordinate], Field(min_length=2, max_length=2)] + + +class GesturePointer(BaseModel): + model_config = ConfigDict(extra="forbid") + id: Annotated[StrictInt, Field(ge=0, le=9)] + path: Annotated[list[Point], Field(min_length=1, max_length=128)] + control_points: Annotated[list[Point], Field(min_length=2, max_length=2)] | None = Field( + default=None, + description="Two cubic Bezier control points. When supplied, path must contain only start and end; Android builds the curve natively.", + ) + + @model_validator(mode="after") + def curve_endpoints(self): + if self.control_points is not None and len(self.path) != 2: + raise ValueError("A cubic Bezier path requires exactly start and end") + return self + + +class GesturePhase(BaseModel): + model_config = ConfigDict(extra="forbid") + duration_ms: Annotated[StrictInt, Field(ge=1, le=5000)] + pointers: Annotated[list[GesturePointer], Field(min_length=1, max_length=10)] + + +class LongPressDrag(BaseModel): + """Hold, move to a caller-selected endpoint, optionally dwell, then release.""" + + model_config = ConfigDict(extra="forbid") + kind: Literal["long_press_drag"] + start: Point + end: Point + control_points: Annotated[list[Point], Field(min_length=2, max_length=2)] | None = Field( + default=None, + description="Two cubic Bezier controls between start and end. Omit for a straight path.", + ) + duration_ms: Annotated[StrictInt, Field(ge=1, le=5000)] = 800 + hold_ms: Annotated[StrictInt, Field(ge=1, le=5000)] | None = Field( + default=None, description="Omit to use the device long-press timeout plus 150ms." + ) + release_delay_ms: Annotated[StrictInt, Field(ge=0, le=5000)] = Field( + default=0, + description="Keep the same finger down at the endpoint for this long, then lift. Zero lifts immediately after moving; independent of movement duration.", + ) + + +GestureInput = GesturePhase | LongPressDrag + + +class GesturePlan(BaseModel): + model_config = ConfigDict(extra="forbid") + phases: Annotated[list[GestureInput], Field(min_length=1, max_length=32)] + + @model_validator(mode="after") + def continuous(self): + if any(isinstance(p, LongPressDrag) for p in self.phases): + if len(self.phases) != 1: + raise ValueError( + "long_press_drag must be the only entry; it includes hold, move and release" + ) + return self + if sum(p.duration_ms for p in self.phases) > 30000: + raise ValueError("Total gesture duration must not exceed 30000ms") + ends = None + for phase in self.phases: + ids = [p.id for p in phase.pointers] + if len(ids) != len(set(ids)): + raise ValueError("Pointer IDs must be unique within a phase") + if ends is not None: + if set(ids) != set(ends): + raise ValueError("Keep the same pointer IDs across all phases") + if any(p.path[0] != ends[p.id] for p in phase.pointers): + raise ValueError("Each continuation must start at the previous endpoint") + ends = {p.id: p.path[-1] for p in phase.pointers} + return self + + +def validate_phases(phases: Any) -> list[dict]: + """Validate before dispatch; never coerce strings/floats/bools into coordinates.""" + return GesturePlan(phases=phases).model_dump(exclude_none=True)["phases"] + + +def gesture_duration_bound_ms(phases: list[dict]) -> int: + """Bound transport timeout including APK-resolved hold and endpoint dwell.""" + return sum( + p["duration_ms"] + + ( + p.get("hold_ms", 5000) + p.get("release_delay_ms", 0) + if p.get("kind") == "long_press_drag" + else 0 + ) + for p in phases + ) diff --git a/docs/multi_touch.md b/docs/multi_touch.md new file mode 100644 index 00000000..6580a4cd --- /dev/null +++ b/docs/multi_touch.md @@ -0,0 +1,124 @@ +# Advanced gesture input + +`perform_gesture` adds continuous and multi-finger input alongside the existing +`click` and `swipe` primitives. Use it when a normal swipe cannot express the +interaction. It uses the existing Operator, Validator, MCP and Android input path. + +Coordinates are integers from **0 to 1000**, relative to the current screen. Each +example below is a direct MCP call. Operator and Flash calls also require +`target_description`, describing the intended target and purpose. + +## Multi-touch + +All pointers in a phase start together. This example pinches inward; reverse the +paths to zoom outward. Add another pointer for a three-finger gesture. + +```json +{ + "action": "perform_gesture", + "phases": [{ + "duration_ms": 800, + "pointers": [ + {"id": 0, "path": [[300, 500], [450, 500]]}, + {"id": 1, "path": [[700, 500], [550, 500]]} + ] + }] +} +``` + +## Long press and drag + +Use `long_press_drag` to hold an object, move it to a chosen endpoint, optionally +hold there, then release. **The endpoint can be anywhere on screen.** + +```json +{ + "action": "perform_gesture", + "phases": [{ + "kind": "long_press_drag", + "start": [300, 400], + "end": [650, 550], + "hold_ms": 700, + "duration_ms": 800, + "release_delay_ms": 500 + }] +} +``` + +- `hold_ms`: initial long press, 1–5000 ms. If omitted, uses the device's configured + long-press timeout plus 150 ms. +- `duration_ms`: movement time, 1–5000 ms; default 800. +- `release_delay_ms`: time to stay pressed at `end`, 0–5000 ms; default 0. Zero + releases as soon as movement completes. A positive delay maintains contact + before release. + +For an edge dwell, choose an endpoint near the desired edge, such as +`"end": [980, 400]`, with a positive `release_delay_ms`. This is the same generic +drag operation. The target UI determines its edge activation region; inspect the +result afterward. + +### Curved drag + +Add two `control_points` for a cubic Bézier curve between the chosen `start` and +`end`. They shape the path; they do not replace or constrain its endpoint. + +```json +{ + "action": "perform_gesture", + "phases": [{ + "kind": "long_press_drag", + "start": [300, 400], + "end": [650, 550], + "control_points": [[400, 250], [600, 250]], + "duration_ms": 800, + "release_delay_ms": 500 + }] +} +``` + +Omit `control_points` for a straight path. The helper constructs the curve from +the supplied points. It never redirects the endpoint to an edge. + +## Explicit continuous phases + +For multiple pointers or intermediate waypoints, use explicit phases. Keep the +same pointer IDs across phases and start each path at its previous endpoint. +Contacts stay down between phases and lift after the final phase. A one-point +path holds still, allowing one pointer to hold while another moves. + +```json +{ + "action": "perform_gesture", + "phases": [ + {"duration_ms": 700, "pointers": [{"id": 0, "path": [[300, 400]]}]}, + {"duration_ms": 800, "pointers": [{"id": 0, "path": [[300, 400], [650, 550]]}]}, + {"duration_ms": 500, "pointers": [{"id": 0, "path": [[650, 550]]}]} + ] +} +``` + +An explicit pointer path may also contain `control_points: [[c1x,c1y],[c2x,c2y]]`; +then `path` must contain exactly the start and end. Do not mix a `long_press_drag` +entry with explicit phases in one call. + +Limits: 1–10 pointers, 1–32 phases, 1–128 path points per pointer, 1–5000 ms per +phase and at most 30000 ms total. Coordinates and controls must stay within +0–1000. Pointer IDs are unique integers from 0 to 9. Invalid plans are rejected +before input starts. + +## Compatibility and results + +The bundled helper is **v1.3.3 / version code 10**. Single-phase gestures support +Android 7+; continuous phases and `long_press_drag` require Android 8+. +Unsupported helpers return `UNSUPPORTED`. Multi-touch and continuous gestures +are never emulated with separate swipes. Ship the APK and its matching +`helper_manifest.json` together. + +Results use the existing `ActionResult` (`ok`, `code`, `message`). Success means +input completed; verify the intended UI effect afterward. A timeout does not +prove the input was never dispatched, so `perform_gesture` is not automatically +retried. Existing actions keep their retry and history behavior. + +Cancellation requests release of held contacts after the current bounded phase. +If release cannot be confirmed, the helper blocks further gesture injection until +recovery. This helper protection does not cover raw ADB input. diff --git a/packages/artemis-accessibility-helper/.gitignore b/packages/artemis-accessibility-helper/.gitignore index afc00f55..7e1f8575 100644 --- a/packages/artemis-accessibility-helper/.gitignore +++ b/packages/artemis-accessibility-helper/.gitignore @@ -3,3 +3,4 @@ build/ .gradle/ *.class .DS_Store +local.properties diff --git a/packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk b/packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk index 16f44800..be68bfbf 100644 Binary files a/packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk and b/packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk differ diff --git a/packages/artemis-accessibility-helper/app/build.gradle.kts b/packages/artemis-accessibility-helper/app/build.gradle.kts index ad802d61..3e0ff81a 100644 --- a/packages/artemis-accessibility-helper/app/build.gradle.kts +++ b/packages/artemis-accessibility-helper/app/build.gradle.kts @@ -10,8 +10,8 @@ android { applicationId = "com.artemis.helper" minSdk = 24 targetSdk = 35 - versionCode = 6 - versionName = "1.2.0" + versionCode = 10 + versionName = "1.3.3" } buildTypes { @@ -23,6 +23,9 @@ android { ) } } + testOptions { + unitTests.isIncludeAndroidResources = true + } compileOptions { sourceCompatibility = JavaVersion.VERSION_1_8 targetCompatibility = JavaVersion.VERSION_1_8 @@ -31,5 +34,7 @@ android { dependencies { // Pure standard Android SDK APIs (android.accessibilityservice, android.view.accessibility, org.json) - // Zero external dependencies to maximize stability, guarantee compatibility, and keep APK under 30KB. + // No runtime dependencies; the following dependencies are JVM test-only. + testImplementation("junit:junit:4.13.2") + testImplementation("org.robolectric:robolectric:4.14.1") } diff --git a/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/CommandServer.java b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/CommandServer.java index 328f7174..ed0fcfa0 100644 --- a/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/CommandServer.java +++ b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/CommandServer.java @@ -338,6 +338,10 @@ private JSONObject executeCommand(String cmd, JSONObject params) { resp.put("success", GestureController.longPress(service, lpx, lpy, lpDuration, 2500L)); } break; + case "perform_gesture": + return ContinuousGesture.execute(service, params); + case "cancel_gesture": + return ContinuousGesture.cancel(params.optString("request_id", "")); case "swipe": float x1 = (float) params.optDouble("x1", -1.0); float y1 = (float) params.optDouble("y1", -1.0); @@ -389,6 +393,10 @@ private JSONObject buildPing(boolean authed) throws Exception { r.put("version_name", service.getVersionName()); r.put("protocol_version", ArtemisAccessibilityService.PROTOCOL_VERSION); r.put("port", port); + r.put("capabilities", new org.json.JSONArray().put("perform_gesture") + .put("gesture_cubic_bezier").put("gesture_long_press_drag")); + r.put("gesture_continuation", android.os.Build.VERSION.SDK_INT >= 26); + r.put("gesture_max_pointers", Math.min(10, android.accessibilityservice.GestureDescription.getMaxStrokeCount())); r.put("auth_required", true); r.put("token_set", TokenStore.isSet()); r.put("authenticated", authed); diff --git a/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/ContinuousGesture.java b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/ContinuousGesture.java new file mode 100644 index 00000000..fcab9ffc --- /dev/null +++ b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/ContinuousGesture.java @@ -0,0 +1,287 @@ +package com.artemis.helper; + +import android.accessibilityservice.AccessibilityService; +import android.accessibilityservice.GestureDescription; +import android.graphics.Path; +import android.os.Build; +import android.os.Handler; +import android.os.Looper; +import android.os.SystemClock; +import android.util.DisplayMetrics; +import android.view.WindowManager; +import android.view.ViewConfiguration; +import org.json.JSONArray; +import org.json.JSONObject; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.TreeMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** One bounded transaction owns every contact until the final phase lifts it. */ +public final class ContinuousGesture { + private static final Handler MAIN = new Handler(Looper.getMainLooper()); + private static final LinkedHashSet CANCELLED = new LinkedHashSet<>(); + private static ContinuousGesture active; + private final AccessibilityService service; + private final String requestId; + private final List phases; + private final CountDownLatch done = new CountDownLatch(1); + private volatile boolean cancelRequested; + private volatile JSONObject result; + private volatile int completed; + private boolean dispatched; + private boolean releasing; + + static final class Phase { + final long duration; + final TreeMap paths; + final TreeMap ends; + GestureDescription.StrokeDescription[] strokes; + GestureDescription gesture; + Phase(long duration, TreeMap paths, TreeMap ends) { + this.duration = duration; this.paths = paths; this.ends = ends; + } + } + + private ContinuousGesture(AccessibilityService service, String requestId, List phases) { + this.service = service; this.requestId = requestId; this.phases = phases; + } + + static JSONObject reply(String id, String status, int count, boolean released, String error) { + JSONObject result = new JSONObject(); + try { + result.put("request_id", id); + result.put("success", "completed".equals(status)); + result.put("status", status); + result.put("phases_completed", count); + result.put("release_confirmed", released); + if (error != null) result.put("error", error); + } catch (Exception impossible) { throw new IllegalStateException(impossible); } + return result; + } + + private static int integer(Object value, int min, int max) { + if (!(value instanceof Integer || value instanceof Long)) + throw new IllegalArgumentException("Expected an integer"); + long n = ((Number) value).longValue(); + if (n < min || n > max) throw new IllegalArgumentException("Integer outside allowed range"); + return (int) n; + } + + // Validate the ENTIRE plan before building or dispatching any input. + static List parse(JSONArray input, int width, int height) throws Exception { + if (width < 1 || height < 1 || input == null || input.length() < 1 || input.length() > 32) + throw new IllegalArgumentException("Invalid display or phase count"); + if ("long_press_drag".equals(input.getJSONObject(0).optString("kind"))) { + if (input.length() != 1) + throw new IllegalArgumentException("long_press_drag must be the only entry"); + input = LongPressDrag.expand(input.getJSONObject(0), + ViewConfiguration.getLongPressTimeout()); + } + if (input.length() > 1 && Build.VERSION.SDK_INT < 26) + throw new UnsupportedOperationException("Continuous gestures require Android 8.0+"); + List phases = new ArrayList<>(); + TreeMap previousEnds = null; + long total = 0; + for (int i = 0; i < input.length(); i++) { + JSONObject raw = input.getJSONObject(i); + if (raw.length() != 2) throw new IllegalArgumentException("Unknown phase fields"); + int duration = integer(raw.get("duration_ms"), 1, 5000); + total += duration; + if (total > 30000 || total > GestureDescription.getMaxGestureDuration()) + throw new IllegalArgumentException("Gesture exceeds duration limit"); + JSONArray pointers = raw.getJSONArray("pointers"); + if (pointers.length() < 1 || pointers.length() > Math.min(10, GestureDescription.getMaxStrokeCount())) + throw new IllegalArgumentException("Invalid pointer count"); + TreeMap paths = new TreeMap<>(); + TreeMap ends = new TreeMap<>(); + TreeMap normalizedEnds = new TreeMap<>(); + for (int j = 0; j < pointers.length(); j++) { + JSONObject pointer = pointers.getJSONObject(j); + boolean curved = pointer.has("control_points"); + if (pointer.length() != (curved ? 3 : 2)) + throw new IllegalArgumentException("Unknown pointer fields"); + int id = integer(pointer.get("id"), 0, 9); + if (paths.containsKey(id)) throw new IllegalArgumentException("Duplicate pointer ID"); + JSONArray points = pointer.getJSONArray("path"); + if (points.length() < 1 || points.length() > 128) + throw new IllegalArgumentException("Invalid path length"); + float[] controls = null; + if (curved) { + JSONArray rawControls = pointer.getJSONArray("control_points"); + if (points.length() != 2 || rawControls.length() != 2) + throw new IllegalArgumentException("Cubic Bezier requires two endpoints and two control points"); + controls = new float[4]; + for (int c = 0; c < 2; c++) { + JSONArray point = rawControls.getJSONArray(c); + if (point.length() != 2) throw new IllegalArgumentException("Expected control [x,y]"); + controls[c * 2] = integer(point.get(0), 0, 1000) * (width - 1) / 1000f; + controls[c * 2 + 1] = integer(point.get(1), 0, 1000) * (height - 1) / 1000f; + } + } + Path path = new Path(); + for (int k = 0; k < points.length(); k++) { + JSONArray point = points.getJSONArray(k); + if (point.length() != 2) throw new IllegalArgumentException("Expected [x,y]"); + int nx = integer(point.get(0), 0, 1000); + int ny = integer(point.get(1), 0, 1000); + String normalized = nx + "," + ny; + if (k == 0 && previousEnds != null && !normalized.equals(previousEnds.get(id))) + throw new IllegalArgumentException("Discontinuous pointer path"); + float x = nx * (width - 1) / 1000f, y = ny * (height - 1) / 1000f; + if (k == 0) path.moveTo(x, y); + else if (curved) path.cubicTo(controls[0], controls[1], controls[2], controls[3], x, y); + else path.lineTo(x, y); + ends.put(id, new float[]{x, y}); + normalizedEnds.put(id, normalized); + } + paths.put(id, path); + } + if (previousEnds != null && !previousEnds.keySet().equals(paths.keySet())) + throw new IllegalArgumentException("Keep the same pointer IDs across phases"); + previousEnds = normalizedEnds; + phases.add(new Phase(duration, paths, ends)); + } + GestureDescription.StrokeDescription[] prior = null; + for (int i = 0; i < phases.size(); i++) { + Phase phase = phases.get(i); + boolean more = i + 1 < phases.size(); + phase.strokes = new GestureDescription.StrokeDescription[phase.paths.size()]; + GestureDescription.Builder builder = new GestureDescription.Builder(); + int j = 0; + for (Path path : phase.paths.values()) { + GestureDescription.StrokeDescription stroke; + if (prior != null) stroke = prior[j].continueStroke(path, 0, phase.duration, more); + else if (Build.VERSION.SDK_INT >= 26) stroke = new GestureDescription.StrokeDescription(path, 0, phase.duration, more); + else stroke = new GestureDescription.StrokeDescription(path, 0, phase.duration); + phase.strokes[j++] = stroke; + builder.addStroke(stroke); + } + phase.gesture = builder.build(); + prior = phase.strokes; + } + return phases; + } + + public static JSONObject execute(AccessibilityService service, JSONObject params) { + String id = params.optString("request_id", ""); + if (!id.matches("[A-Za-z0-9_-]{1,64}")) + return reply(id, "invalid_arguments", 0, true, "A bounded request_id is required"); + if (Looper.myLooper() == Looper.getMainLooper()) + return reply(id, "rejected", 0, true, "Call from a worker thread"); + final ContinuousGesture run; + try { + DisplayMetrics metrics = new DisplayMetrics(); + ((WindowManager) service.getSystemService(AccessibilityService.WINDOW_SERVICE)) + .getDefaultDisplay().getRealMetrics(metrics); + run = new ContinuousGesture(service, id, + parse(params.getJSONArray("phases"), metrics.widthPixels, metrics.heightPixels)); + } catch (UnsupportedOperationException e) { + return reply(id, "unsupported", 0, true, e.getMessage()); + } catch (Exception e) { + return reply(id, "invalid_arguments", 0, true, e.getMessage()); + } + synchronized (ContinuousGesture.class) { + if (CANCELLED.remove(id)) return reply(id, "cancelled", 0, true, "Cancelled before dispatch"); + if (!GestureController.INPUT_BUSY.compareAndSet(false, true)) + return reply(id, "busy", 0, true, "Another gesture is active or needs recovery"); + active = run; + } + long total = 0; + for (Phase p : run.phases) total += p.duration; + MAIN.post(() -> run.next()); + try { + if (run.done.await(total + 5000, TimeUnit.MILLISECONDS)) return run.result; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + run.cancelRequested = true; + // Do not unlock or replay: late callbacks still own and release held contacts. + return reply(id, "timeout", run.completed, false, + "Gesture outcome unknown; cancellation requested. Reconcile device before reuse."); + } + + public static synchronized JSONObject cancel(String id) { + if (id == null || !id.matches("[A-Za-z0-9_-]{1,64}")) + return reply(id, "invalid_arguments", 0, true, "Invalid request_id"); + if (active != null && active.requestId.equals(id)) { + active.cancelRequested = true; + return reply(id, "cancelling", active.completed, false, "Will release after the current bounded phase"); + } + // Covers cancellation racing the host's in-flight capability check / request. + CANCELLED.add(id); + while (CANCELLED.size() > 128) CANCELLED.remove(CANCELLED.iterator().next()); + return reply(id, "cancelled", 0, true, "No matching active gesture"); + } + + private void finish(String status, boolean released, String error) { + if (result != null) return; + result = reply(requestId, status, completed, released, error); + synchronized (ContinuousGesture.class) { + if (active == this) active = null; + } + // Unknown held contacts fence all further injected gestures until recovery. + if (released || !dispatched) GestureController.INPUT_BUSY.set(false); + done.countDown(); + } + + private void next() { + if (result != null) return; + if (cancelRequested) { release("cancelled"); return; } + if (completed == phases.size()) { finish("completed", true, null); return; } + Phase phase = phases.get(completed); + final long phaseEndsAt = SystemClock.uptimeMillis() + phase.duration; + try { + boolean accepted = service.dispatchGesture(phase.gesture, + new AccessibilityService.GestureResultCallback() { + @Override public void onCompleted(GestureDescription ignored) { + completed++; + if (completed == phases.size()) + finish(cancelRequested ? "cancelled" : "completed", true, + cancelRequested ? "Cancellation requested" : null); + else { + // Android completes a stationary continued stroke after DOWN: + // duplicate MOVE events are omitted and there is no UP yet. + // Keep contact held until its requested time before continuing. + MAIN.postAtTime(() -> next(), phaseEndsAt); + } + } + @Override public void onCancelled(GestureDescription ignored) { + // Android cancels this stream. Conservatively do not certify release. + finish("cancelled", false, "System cancelled gesture; reconcile device"); + } + }, MAIN); + if (!accepted) release("rejected"); else dispatched = true; + } catch (Exception e) { release("rejected"); } + } + + private void release(final String status) { + if (completed == 0 || completed == phases.size()) { + finish(status, !dispatched || completed == phases.size(), "Gesture " + status); + return; + } + if (releasing) return; + releasing = true; + try { + Phase previous = phases.get(completed - 1); + GestureDescription.Builder builder = new GestureDescription.Builder(); + int j = 0; + for (float[] endpoint : previous.ends.values()) { + Path still = new Path(); still.moveTo(endpoint[0], endpoint[1]); + builder.addStroke(previous.strokes[j++].continueStroke(still, 0, 1, false)); + } + boolean accepted = service.dispatchGesture(builder.build(), + new AccessibilityService.GestureResultCallback() { + @Override public void onCompleted(GestureDescription ignored) { + finish(status, true, "Gesture " + status + "; held contacts released"); + } + @Override public void onCancelled(GestureDescription ignored) { + finish(status, false, "Release cancelled; reconcile device"); + } + }, MAIN); + if (!accepted) finish(status, false, "Release rejected; reconcile device"); + } catch (Exception e) { finish(status, false, "Release failed; reconcile device"); } + } +} diff --git a/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/GestureController.java b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/GestureController.java index 1714a634..c77ba05f 100644 --- a/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/GestureController.java +++ b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/GestureController.java @@ -23,6 +23,9 @@ */ public final class GestureController { + // Owned only by ContinuousGesture; ordinary actions retain their dispatch semantics. + static final AtomicBoolean INPUT_BUSY = new AtomicBoolean(false); + private static final String TAG = "ArtemisGestureCtrl"; private static final Handler MAIN_HANDLER = new Handler(Looper.getMainLooper()); @@ -162,12 +165,18 @@ private static boolean dispatchSynchronous( final GestureDescription gesture, long timeoutMs ) { + if (INPUT_BUSY.get()) return false; final CountDownLatch latch = new CountDownLatch(1); final AtomicBoolean result = new AtomicBoolean(false); MAIN_HANDLER.post(new Runnable() { @Override public void run() { + // A continuous gesture may have started while this input was queued. + if (INPUT_BUSY.get()) { + latch.countDown(); + return; + } try { service.dispatchGesture(gesture, new AccessibilityService.GestureResultCallback() { @Override diff --git a/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/LongPressDrag.java b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/LongPressDrag.java new file mode 100644 index 00000000..d877bea6 --- /dev/null +++ b/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/LongPressDrag.java @@ -0,0 +1,62 @@ +package com.artemis.helper; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import org.json.JSONArray; +import org.json.JSONObject; + +/** Expands a long-press drag; ContinuousGesture owns dispatch and release. */ +final class LongPressDrag { + private static final Set FIELDS = new HashSet<>(Arrays.asList( + "kind", "start", "end", "control_points", "duration_ms", "hold_ms", "release_delay_ms")); + + private static int integer(Object value, int min, int max) { + if (!(value instanceof Integer || value instanceof Long)) + throw new IllegalArgumentException("Expected an integer"); + long n = ((Number) value).longValue(); + if (n < min || n > max) throw new IllegalArgumentException("Integer outside allowed range"); + return (int) n; + } + + private static JSONArray point(JSONArray value) throws Exception { + if (value.length() != 2) throw new IllegalArgumentException("Expected [x,y]"); + return new JSONArray().put(integer(value.get(0), 0, 1000)) + .put(integer(value.get(1), 0, 1000)); + } + + private static JSONObject phase(int duration, JSONArray path, JSONArray controls) throws Exception { + JSONObject pointer = new JSONObject().put("id", 0).put("path", path); + if (controls != null) pointer.put("control_points", controls); + return new JSONObject().put("duration_ms", duration) + .put("pointers", new JSONArray().put(pointer)); + } + + static JSONArray expand(JSONObject args, int longPressMs) throws Exception { + for (Iterator keys = args.keys(); keys.hasNext();) + if (!FIELDS.contains(keys.next())) throw new IllegalArgumentException("Unknown long-press drag field"); + if (!"long_press_drag".equals(args.get("kind"))) + throw new IllegalArgumentException("Invalid long-press drag kind"); + JSONArray start = point(args.getJSONArray("start")); + JSONArray end = point(args.getJSONArray("end")); + JSONArray controls = null; + if (args.has("control_points")) { + JSONArray raw = args.getJSONArray("control_points"); + if (raw.length() != 2) + throw new IllegalArgumentException("Cubic Bezier requires two control points"); + controls = new JSONArray().put(point(raw.getJSONArray(0))).put(point(raw.getJSONArray(1))); + } + int duration = integer(args.has("duration_ms") ? args.get("duration_ms") : 800, 1, 5000); + // Do not silently shorten a device's configured long press to fit the budget. + int hold = integer(args.has("hold_ms") ? args.get("hold_ms") : (long) longPressMs + 150, 1, 5000); + int delay = integer(args.has("release_delay_ms") ? args.get("release_delay_ms") : 0, 0, 5000); + + JSONArray phases = new JSONArray() + .put(phase(hold, new JSONArray().put(start), null)) + .put(phase(duration, new JSONArray().put(start).put(end), controls)); + // Zero lifts on the movement stroke; a positive delay holds at the chosen endpoint. + if (delay > 0) phases.put(phase(delay, new JSONArray().put(end), null)); + return phases; + } +} diff --git a/packages/artemis-accessibility-helper/app/src/test/java/com/artemis/helper/ContinuousGestureTest.java b/packages/artemis-accessibility-helper/app/src/test/java/com/artemis/helper/ContinuousGestureTest.java new file mode 100644 index 00000000..0b7d5e34 --- /dev/null +++ b/packages/artemis-accessibility-helper/app/src/test/java/com/artemis/helper/ContinuousGestureTest.java @@ -0,0 +1,401 @@ +package com.artemis.helper; + +import static org.junit.Assert.*; +import static org.robolectric.Shadows.shadowOf; + +import android.accessibilityservice.AccessibilityService; +import android.accessibilityservice.GestureDescription; +import android.graphics.PathMeasure; +import android.os.Handler; +import android.os.Looper; +import android.view.accessibility.AccessibilityEvent; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.Robolectric; +import org.robolectric.RobolectricTestRunner; +import org.robolectric.annotation.Config; +import org.robolectric.annotation.Implementation; +import org.robolectric.annotation.Implements; +import org.robolectric.annotation.LooperMode; +import org.robolectric.annotation.GraphicsMode; +import java.util.ArrayList; +import java.util.List; +import java.time.Duration; +import java.util.concurrent.*; + +@RunWith(RobolectricTestRunner.class) +@Config(sdk = 28, shadows = ContinuousGestureTest.InputShadow.class) +@LooperMode(LooperMode.Mode.PAUSED) +public class ContinuousGestureTest { + private ExecutorService worker; + private TestService service; + + public static class TestService extends AccessibilityService { + @Override public void onAccessibilityEvent(AccessibilityEvent event) {} + @Override public void onInterrupt() {} + } + + @Implements(AccessibilityService.class) + public static class InputShadow { + static final List inputs = new ArrayList<>(); + static final List callbacks = new ArrayList<>(); + static int rejectIndex = -1; + @Implementation + protected boolean dispatchGesture(GestureDescription gesture, + AccessibilityService.GestureResultCallback callback, Handler handler) { + inputs.add(gesture); callbacks.add(callback); + return inputs.size() - 1 != rejectIndex; + } + } + + @Before public void setup() { + InputShadow.inputs.clear(); InputShadow.callbacks.clear(); InputShadow.rejectIndex = -1; + GestureController.INPUT_BUSY.set(false); + service = Robolectric.buildService(TestService.class).create().get(); + worker = Executors.newSingleThreadExecutor(); + } + @After public void teardown() { worker.shutdownNow(); } + + private static JSONObject phase(int duration, int... points) throws Exception { + JSONArray path = new JSONArray(); + for (int i = 0; i < points.length; i += 2) + path.put(new JSONArray().put(points[i]).put(points[i + 1])); + return new JSONObject().put("duration_ms", duration).put("pointers", new JSONArray().put( + new JSONObject().put("id", 0).put("path", path))); + } + private static JSONArray drag() throws Exception { + return new JSONArray().put(phase(600, 300, 400)) + .put(phase(800, 300, 400, 980, 400)).put(phase(900, 980, 400)); + } + private Future execute(String id, JSONArray phases) throws Exception { + JSONObject args = new JSONObject().put("request_id", id).put("phases", phases); + return worker.submit(() -> ContinuousGesture.execute(service, args)); + } + private void awaitInputs(int count) throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (InputShadow.inputs.size() < count && System.nanoTime() < deadline) { + shadowOf(Looper.getMainLooper()).idle(); Thread.sleep(5); + } + assertEquals(count, InputShadow.inputs.size()); + } + private void complete(int index) { + InputShadow.callbacks.get(index).onCompleted(InputShadow.inputs.get(index)); + shadowOf(Looper.getMainLooper()).idleFor(Duration.ofMillis( + InputShadow.inputs.get(index).getStroke(0).getDuration())); + } + + @Test public void earlyDownCallbackCannotSkipLongPressTime() throws Exception { + Future result = execute("early-down", new JSONArray().put( + longPressDrag(0).put("hold_ms", 1500))); + awaitInputs(1); + InputShadow.callbacks.get(0).onCompleted(InputShadow.inputs.get(0)); + shadowOf(Looper.getMainLooper()).idleFor(Duration.ofMillis(1499)); + assertEquals("Do not move while the icon is still being long-pressed", 1, InputShadow.inputs.size()); + assertTrue(GestureController.INPUT_BUSY.get()); + assertFalse(result.isDone()); + shadowOf(Looper.getMainLooper()).idleFor(Duration.ofMillis(1)); + awaitInputs(2); + complete(1); + assertTrue(result.get(5, TimeUnit.SECONDS).getBoolean("release_confirmed")); + } + + @Test public void cancellationWhileWaitingForLongPressReleasesWithoutMoving() throws Exception { + Future result = execute("cancel-hold-wait", new JSONArray().put( + longPressDrag(0).put("hold_ms", 1500))); + awaitInputs(1); + InputShadow.callbacks.get(0).onCompleted(InputShadow.inputs.get(0)); + shadowOf(Looper.getMainLooper()).idleFor(Duration.ofMillis(500)); + ContinuousGesture.cancel("cancel-hold-wait"); + shadowOf(Looper.getMainLooper()).idleFor(Duration.ofMillis(1000)); + awaitInputs(2); + assertEquals(1, InputShadow.inputs.get(1).getStroke(0).getDuration()); + assertFalse(InputShadow.inputs.get(1).getStroke(0).willContinue()); + complete(1); + JSONObject receipt = result.get(5, TimeUnit.SECONDS); + assertEquals("cancelled", receipt.getString("status")); + assertTrue(receipt.getBoolean("release_confirmed")); + assertFalse(GestureController.INPUT_BUSY.get()); + } + + @Test public void allFingersAreSimultaneousAndLiftTogether() throws Exception { + JSONObject phase = phase(500, 100, 200, 100, 800); + JSONArray pointers = phase.getJSONArray("pointers"); + for (int id = 1; id < 3; id++) { + JSONObject p = new JSONObject(pointers.getJSONObject(0).toString()).put("id", id); + p.put("path", new JSONArray().put(new JSONArray().put(100 + id * 250).put(200)) + .put(new JSONArray().put(100 + id * 250).put(800))); + pointers.put(p); + } + GestureDescription gesture = ContinuousGesture.parse(new JSONArray().put(phase), 1080, 2400).get(0).gesture; + assertEquals(3, gesture.getStrokeCount()); + for (int i = 0; i < 3; i++) { + assertEquals(0, gesture.getStroke(i).getStartTime()); + assertEquals(500, gesture.getStroke(i).getDuration()); + assertFalse(gesture.getStroke(i).willContinue()); + } + } + + @Test public void dragKeepsContactUntilFinalPhase() throws Exception { + Future result = execute("drag", drag()); + for (int i = 0; i < 3; i++) { + awaitInputs(i + 1); + assertEquals(i < 2, InputShadow.inputs.get(i).getStroke(0).willContinue()); + assertTrue(GestureController.INPUT_BUSY.get()); + complete(i); + } + assertTrue(result.get(5, TimeUnit.SECONDS).getBoolean("success")); + assertFalse(GestureController.INPUT_BUSY.get()); + } + + private static JSONArray curvedDrag() throws Exception { + JSONArray phases = drag(); + phases.getJSONObject(1).getJSONArray("pointers").getJSONObject(0).put("control_points", + new JSONArray("[[450,200],[800,200]]")); + return phases; + } + + private static JSONObject longPressDrag(int delay) throws Exception { + return new JSONObject().put("kind", "long_press_drag").put("start", new JSONArray("[300,400]")) + .put("end", new JSONArray("[650,550]")).put("duration_ms", 800).put("release_delay_ms", delay); + } + + @Test public void zeroDelayLiftsOnMovementWithoutAnotherPhase() throws Exception { + Future result = execute("drag-immediate", new JSONArray().put(longPressDrag(0))); + awaitInputs(1); + assertTrue(InputShadow.inputs.get(0).getStroke(0).willContinue()); + complete(0); + awaitInputs(2); + GestureDescription.StrokeDescription movement = InputShadow.inputs.get(1).getStroke(0); + assertEquals(800, movement.getDuration()); + assertFalse(movement.willContinue()); + complete(1); + assertTrue(result.get(5, TimeUnit.SECONDS).getBoolean("release_confirmed")); + assertEquals(2, InputShadow.inputs.size()); + } + + @Test public void positiveDelayHoldsAtEndpointAndOnlyThenReleases() throws Exception { + Future result = execute("drag-delayed", new JSONArray().put(longPressDrag(1200))); + awaitInputs(1); complete(0); awaitInputs(2); + assertTrue(InputShadow.inputs.get(1).getStroke(0).willContinue()); + assertEquals(800, InputShadow.inputs.get(1).getStroke(0).getDuration()); + complete(1); awaitInputs(3); + GestureDescription.StrokeDescription dwell = InputShadow.inputs.get(2).getStroke(0); + assertEquals(1200, dwell.getDuration()); + assertFalse(dwell.willContinue()); + assertFalse(result.isDone()); + assertTrue(GestureController.INPUT_BUSY.get()); + complete(2); + assertTrue(result.get(5, TimeUnit.SECONDS).getBoolean("release_confirmed")); + assertFalse(GestureController.INPUT_BUSY.get()); + } + + @Test @GraphicsMode(GraphicsMode.Mode.NATIVE) + public void longPressDragPreservesChosenEndpointAndDeviceHold() throws Exception { + for (int[] endpoint : new int[][]{{650, 550}, {400, 200}, {20, 400}, {980, 400}}) { + for (int[] size : new int[][]{{1080, 2400}, {2400, 1080}}) { + JSONObject intent = longPressDrag(1200).put("end", new JSONArray(endpoint)) + .put("control_points", new JSONArray("[[400,250],[600,250]]")); + JSONArray raw = LongPressDrag.expand(intent, 1000); + assertEquals(1150, raw.getJSONObject(0).getInt("duration_ms")); + List parsed = ContinuousGesture.parse(raw, size[0], size[1]); + float[] end = parsed.get(1).ends.get(0); + int nx = endpoint[0], ny = endpoint[1]; + assertArrayEquals(new float[]{nx * (size[0] - 1) / 1000f, ny * (size[1] - 1) / 1000f}, end, 0.1f); + PathMeasure curve = new PathMeasure(parsed.get(1).paths.get(0), false); + float[] actualEnd = new float[2]; + assertTrue(curve.getPosTan(curve.getLength(), actualEnd, null)); + assertArrayEquals(end, actualEnd, 0.1f); + PathMeasure still = new PathMeasure(parsed.get(2).paths.get(0), false); + assertEquals(0f, still.getLength(), 0.01f); + assertArrayEquals(end, parsed.get(2).ends.get(0), 0.01f); + intent.put("hold_ms", 650); + assertEquals(650, LongPressDrag.expand(intent, 1000).getJSONObject(0).getInt("duration_ms")); + } + } + } + + @Test public void invalidReleaseDelayAndMixedPlansNeverTouchDevice() throws Exception { + for (Object value : new Object[]{-1, 5001, 1.5, true, "1000"}) { + JSONObject invalid = longPressDrag(0).put("release_delay_ms", value); + assertEquals("invalid_arguments", execute("bad-delay", new JSONArray().put(invalid)) + .get(5, TimeUnit.SECONDS).getString("status")); + } + assertEquals("invalid_arguments", execute("mixed", new JSONArray().put(longPressDrag(0)).put(phase(100, 300, 400))) + .get(5, TimeUnit.SECONDS).getString("status")); + assertTrue(InputShadow.inputs.isEmpty()); + } + + @Test public void invalidLongPressDragEndpointsAndControlsNeverTouchDevice() throws Exception { + JSONObject missingEnd = longPressDrag(0); + missingEnd.remove("end"); + JSONObject[] invalid = { + missingEnd, + longPressDrag(0).put("end", new JSONArray("[1001,500]")), + longPressDrag(0).put("end", new JSONArray("[500]")), + longPressDrag(0).put("end", new JSONArray("[true,500]")), + longPressDrag(0).put("control_points", new JSONArray("[[100,200]]")), + longPressDrag(0).put("control_points", new JSONArray("[[100,200],[300,1001]]")), + longPressDrag(0).put("direction", "left") + }; + for (int i = 0; i < invalid.length; i++) + assertEquals("invalid_arguments", execute("bad-drag-" + i, new JSONArray().put(invalid[i])) + .get(5, TimeUnit.SECONDS).getString("status")); + assertTrue(InputShadow.inputs.isEmpty()); + } + + @Test public void longPressDragWithoutControlsUsesAStraightPath() throws Exception { + JSONArray phases = LongPressDrag.expand(longPressDrag(0), 500); + JSONObject movement = phases.getJSONObject(1).getJSONArray("pointers").getJSONObject(0); + assertEquals("[[300,400],[650,550]]", movement.getJSONArray("path").toString()); + assertFalse(movement.has("control_points")); + } + + @Test public void cancelDuringEndpointDwellStillReleasesAndUnlocks() throws Exception { + Future result = execute("drag-cancel", new JSONArray().put(longPressDrag(1000))); + awaitInputs(1); complete(0); awaitInputs(2); complete(1); awaitInputs(3); + ContinuousGesture.cancel("drag-cancel"); + complete(2); + JSONObject receipt = result.get(5, TimeUnit.SECONDS); + assertEquals("cancelled", receipt.getString("status")); + assertTrue(receipt.getBoolean("release_confirmed")); + assertFalse(GestureController.INPUT_BUSY.get()); + } + + @Test @GraphicsMode(GraphicsMode.Mode.NATIVE) + public void cubicPathHasCurvatureAndScaledEndpointsWithoutLifting() throws Exception { + List phases = ContinuousGesture.parse(curvedDrag(), 1001, 2001); + PathMeasure measure = new PathMeasure(phases.get(1).paths.get(0), false); + assertTrue(measure.getLength() > 680); + float[] position = new float[2]; + assertTrue(measure.getPosTan(0, position, null)); + assertArrayEquals(new float[]{300, 800}, position, 0.1f); + assertTrue(measure.getPosTan(measure.getLength() / 2, position, null)); + assertTrue("Curve must bow above the straight horizontal path", position[1] < 700); + assertTrue(measure.getPosTan(measure.getLength(), position, null)); + assertArrayEquals(new float[]{980, 800}, position, 0.1f); + for (int i = 0; i < phases.size(); i++) + assertEquals(i < 2, phases.get(i).gesture.getStroke(0).willContinue()); + } + + @Test public void curvedDragCompletesHoldMoveDwellAsOneContact() throws Exception { + Future result = execute("curve", curvedDrag()); + for (int i = 0; i < 3; i++) { + awaitInputs(i + 1); + assertEquals(i < 2, InputShadow.inputs.get(i).getStroke(0).willContinue()); + complete(i); + } + JSONObject receipt = result.get(5, TimeUnit.SECONDS); + assertTrue(receipt.getBoolean("success")); + assertTrue(receipt.getBoolean("release_confirmed")); + } + + @Test public void malformedCurveInLaterPhaseNeverDispatchesHold() throws Exception { + String[] invalid = {"[]", "[[1,2]]", "[[1,2],[3,4],[5,6]]", "[[1001,2],[3,4]]", + "[[1.5,2],[3,4]]", "[[true,2],[3,4]]", "[[1,2,3],[4,5]]"}; + for (int i = 0; i < invalid.length; i++) { + JSONArray phases = curvedDrag(); + phases.getJSONObject(1).getJSONArray("pointers").getJSONObject(0) + .put("control_points", new JSONArray(invalid[i])); + assertEquals("invalid_arguments", execute("bad-curve-" + i, phases) + .get(5, TimeUnit.SECONDS).getString("status")); + } + JSONArray phases = curvedDrag(); + phases.getJSONObject(1).getJSONArray("pointers").getJSONObject(0) + .getJSONArray("path").put(new JSONArray("[980,400]")); + assertEquals("invalid_arguments", execute("extra-endpoint", phases) + .get(5, TimeUnit.SECONDS).getString("status")); + assertTrue(InputShadow.inputs.isEmpty()); + } + + @Test public void laterInvalidPhaseCannotCausePartialExecution() throws Exception { + JSONArray phases = drag(); + phases.getJSONObject(2).getJSONArray("pointers").getJSONObject(0) + .put("path", new JSONArray().put(new JSONArray().put(500).put(400))); + JSONObject result = execute("invalid", phases).get(5, TimeUnit.SECONDS); + assertEquals("invalid_arguments", result.getString("status")); + assertTrue(InputShadow.inputs.isEmpty()); + } + + @Test public void cancelHeldPhaseDispatchesOnlyReleaseThenUnlocks() throws Exception { + Future result = execute("cancel", drag()); + awaitInputs(1); + ContinuousGesture.cancel("cancel"); + complete(0); + awaitInputs(2); + GestureDescription release = InputShadow.inputs.get(1); + assertEquals(1, release.getStroke(0).getDuration()); + assertFalse(release.getStroke(0).willContinue()); + complete(1); + JSONObject reply = result.get(5, TimeUnit.SECONDS); + assertEquals("cancelled", reply.getString("status")); + assertTrue(reply.getBoolean("release_confirmed")); + assertFalse(GestureController.INPUT_BUSY.get()); + } + + @Test public void cancelBeforeRequestDoesNotTouchDevice() throws Exception { + ContinuousGesture.cancel("race"); + JSONObject result = execute("race", drag()).get(5, TimeUnit.SECONDS); + assertEquals("cancelled", result.getString("status")); + assertTrue(InputShadow.inputs.isEmpty()); + } + + @Test public void rejectedReleaseDoesNotAllowAnotherGesture() throws Exception { + Future result = execute("reject-release", drag()); + awaitInputs(1); + InputShadow.rejectIndex = 1; + ContinuousGesture.cancel("reject-release"); + complete(0); + JSONObject reply = result.get(5, TimeUnit.SECONDS); + assertFalse(reply.getBoolean("release_confirmed")); + assertTrue(GestureController.INPUT_BUSY.get()); + assertEquals("busy", execute("next", drag()).get(5, TimeUnit.SECONDS).getString("status")); + } + + @Test public void ordinaryTapRetainsSuccessAndCancellationResults() throws Exception { + Future tap = worker.submit(() -> GestureController.tap(service, 300, 400, 5000)); + awaitInputs(1); + assertFalse("Ordinary actions do not own continuous-input state", GestureController.INPUT_BUSY.get()); + complete(0); + assertTrue(tap.get(5, TimeUnit.SECONDS)); + + Future swipe = worker.submit(() -> GestureController.swipe(service, 300, 400, 650, 550, 800, 5000)); + awaitInputs(2); + InputShadow.callbacks.get(1).onCancelled(InputShadow.inputs.get(1)); + assertFalse(swipe.get(5, TimeUnit.SECONDS)); + assertFalse(GestureController.INPUT_BUSY.get()); + } + + @Test public void ordinaryCallbackCannotUnlockAnActiveContinuousGesture() throws Exception { + CompletableFuture tap = CompletableFuture.supplyAsync( + () -> GestureController.tap(service, 300, 400, 5000)); + awaitInputs(1); + Future gesture = execute("legacy-callback", drag()); + awaitInputs(2); + InputShadow.callbacks.get(0).onCompleted(InputShadow.inputs.get(0)); + assertTrue(tap.get(5, TimeUnit.SECONDS)); + assertTrue(GestureController.INPUT_BUSY.get()); + assertFalse(GestureController.tap(service, 600, 500, 100)); + assertFalse(GestureController.swipe(service, 300, 400, 650, 550, 800, 100)); + assertEquals(2, InputShadow.inputs.size()); + + ContinuousGesture.cancel("legacy-callback"); + complete(1); + awaitInputs(3); + complete(2); + assertTrue(gesture.get(5, TimeUnit.SECONDS).getBoolean("release_confirmed")); + assertFalse(GestureController.INPUT_BUSY.get()); + Future next = worker.submit(() -> GestureController.tap(service, 600, 500, 5000)); + awaitInputs(4); + complete(3); + assertTrue(next.get(5, TimeUnit.SECONDS)); + } + + @Test @Config(sdk = 24) public void android7RejectsContinuationButAllowsSinglePhase() throws Exception { + assertEquals(1, ContinuousGesture.parse(new JSONArray().put(phase(200, 100, 200)), 1080, 2400).size()); + assertThrows(UnsupportedOperationException.class, () -> ContinuousGesture.parse(drag(), 1080, 2400)); + } +} diff --git a/packages/artemis-accessibility-helper/gradle/wrapper/gradle-wrapper.jar b/packages/artemis-accessibility-helper/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000..61285a65 Binary files /dev/null and b/packages/artemis-accessibility-helper/gradle/wrapper/gradle-wrapper.jar differ diff --git a/packages/artemis-accessibility-helper/gradle/wrapper/gradle-wrapper.properties b/packages/artemis-accessibility-helper/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..19a6bdeb --- /dev/null +++ b/packages/artemis-accessibility-helper/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-9.3.0-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/packages/artemis-accessibility-helper/gradlew b/packages/artemis-accessibility-helper/gradlew new file mode 100755 index 00000000..adff685a --- /dev/null +++ b/packages/artemis-accessibility-helper/gradlew @@ -0,0 +1,248 @@ +#!/bin/sh + +# +# Copyright © 2015 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/packages/artemis-accessibility-helper/gradlew.bat b/packages/artemis-accessibility-helper/gradlew.bat new file mode 100644 index 00000000..e509b2dd --- /dev/null +++ b/packages/artemis-accessibility-helper/gradlew.bat @@ -0,0 +1,93 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/packages/artemis-accessibility-helper/helper_manifest.json b/packages/artemis-accessibility-helper/helper_manifest.json index f8d979dd..65192926 100644 --- a/packages/artemis-accessibility-helper/helper_manifest.json +++ b/packages/artemis-accessibility-helper/helper_manifest.json @@ -1,7 +1,7 @@ { "package": "com.artemis.helper", - "version_code": 6, - "version_name": "1.2.0", - "sha256": "133e373fe03615c251cd74b03a38539c844d09a074954bca2d2dde6684561969", - "built_at": "2026-09-10T23:13:11+00:00" + "version_code": 10, + "version_name": "1.3.3", + "sha256": "dd625d467f16a432580022bb01b723aa78eac30647611b60c9937c7e8f352f74", + "built_at": "2026-09-29T11:19:03+00:00" } diff --git a/tests/fixtures/action_surfaces/action_server_manifest.json b/tests/fixtures/action_surfaces/action_server_manifest.json index 968aa5df..93649d44 100644 --- a/tests/fixtures/action_surfaces/action_server_manifest.json +++ b/tests/fixtures/action_surfaces/action_server_manifest.json @@ -1439,5 +1439,295 @@ "type": "object" }, "outputSchema": null + }, + "perform_gesture": { + "description": "[ACTION] Perform continuous touch gestures that a normal swipe cannot express: simultaneous multi-finger input, continuous holds and drags, or delayed release. Coordinates use 0-1000 screen space. In explicit phases, pointers move together; a one-point path holds still. Keep the same pointer IDs and join endpoints across phases to maintain contact until the final release. Optional control_points define a cubic Bezier path with two endpoints and two controls. Alternatively, provide one long_press_drag entry with start and end coordinates and optional control_points. The endpoint is freely chosen, including inside the screen. duration_ms is movement time; release_delay_ms is the endpoint hold before lifting (zero releases immediately). Observe the screen afterward to verify the intended effect.", + "inputSchema": { + "$defs": { + "GesturePhase": { + "additionalProperties": false, + "properties": { + "duration_ms": { + "maximum": 5000, + "minimum": 1, + "title": "Duration Ms", + "type": "integer" + }, + "pointers": { + "items": { + "$ref": "#/$defs/GesturePointer" + }, + "maxItems": 10, + "minItems": 1, + "title": "Pointers", + "type": "array" + } + }, + "required": [ + "duration_ms", + "pointers" + ], + "title": "GesturePhase", + "type": "object" + }, + "GesturePointer": { + "additionalProperties": false, + "properties": { + "id": { + "maximum": 9, + "minimum": 0, + "title": "Id", + "type": "integer" + }, + "path": { + "items": { + "items": { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + "maxItems": 128, + "minItems": 1, + "title": "Path", + "type": "array" + }, + "control_points": { + "anyOf": [ + { + "items": { + "items": { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Two cubic Bezier control points. When supplied, path must contain only start and end; Android builds the curve natively.", + "title": "Control Points" + } + }, + "required": [ + "id", + "path" + ], + "title": "GesturePointer", + "type": "object" + }, + "LongPressDrag": { + "additionalProperties": false, + "description": "Hold, move to a caller-selected endpoint, optionally dwell, then release.", + "properties": { + "kind": { + "const": "long_press_drag", + "title": "Kind", + "type": "string" + }, + "start": { + "items": { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "title": "Start", + "type": "array" + }, + "end": { + "items": { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "title": "End", + "type": "array" + }, + "control_points": { + "anyOf": [ + { + "items": { + "items": { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Two cubic Bezier controls between start and end. Omit for a straight path.", + "title": "Control Points" + }, + "duration_ms": { + "default": 800, + "maximum": 5000, + "minimum": 1, + "title": "Duration Ms", + "type": "integer" + }, + "hold_ms": { + "anyOf": [ + { + "maximum": 5000, + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Omit to use the device long-press timeout plus 150ms.", + "title": "Hold Ms" + }, + "release_delay_ms": { + "default": 0, + "description": "Keep the same finger down at the endpoint for this long, then lift. Zero lifts immediately after moving; independent of movement duration.", + "maximum": 5000, + "minimum": 0, + "title": "Release Delay Ms", + "type": "integer" + } + }, + "required": [ + "kind", + "start", + "end" + ], + "title": "LongPressDrag", + "type": "object" + } + }, + "properties": { + "phases": { + "items": { + "anyOf": [ + { + "$ref": "#/$defs/GesturePhase" + }, + { + "$ref": "#/$defs/LongPressDrag" + } + ] + }, + "title": "Phases", + "type": "array" + } + }, + "required": [ + "phases" + ], + "title": "perform_gestureArguments", + "type": "object" + }, + "outputSchema": { + "$defs": { + "ActionCode": { + "description": "Machine-readable outcome classification for a device action.", + "enum": [ + "OK", + "INVALID_ARGS", + "TARGET_NOT_FOUND", + "DEVICE_ERROR", + "PACKAGE_NOT_FOUND", + "TIMEOUT", + "UNSUPPORTED" + ], + "title": "ActionCode", + "type": "string" + } + }, + "description": "Outcome of one device action, independent of any transport or agent.", + "properties": { + "ok": { + "title": "Ok", + "type": "boolean" + }, + "code": { + "$ref": "#/$defs/ActionCode" + }, + "action": { + "title": "Action", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Detail" + }, + "normalized_coordinates": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Normalized Coordinates" + }, + "duration_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Ms" + } + }, + "required": [ + "ok", + "code", + "action", + "message" + ], + "title": "ActionResult", + "type": "object" + } } } diff --git a/tests/fixtures/action_surfaces/operator_shells.json b/tests/fixtures/action_surfaces/operator_shells.json index 6f3e6c81..3be0ba75 100644 --- a/tests/fixtures/action_surfaces/operator_shells.json +++ b/tests/fixtures/action_surfaces/operator_shells.json @@ -347,5 +347,209 @@ "type": "object" } } + }, + "perform_gesture": { + "type": "function", + "function": { + "name": "perform_gesture", + "description": "[ACTION] Perform continuous touch gestures that a normal swipe cannot express: simultaneous multi-finger input, continuous holds and drags, or delayed release. Coordinates use 0-1000 screen space. In explicit phases, pointers move together; a one-point path holds still. Keep the same pointer IDs and join endpoints across phases to maintain contact until the final release. Optional control_points define a cubic Bezier path with two endpoints and two controls. Alternatively, provide one long_press_drag entry with start and end coordinates and optional control_points. The endpoint is freely chosen, including inside the screen. duration_ms is movement time; release_delay_ms is the endpoint hold before lifting (zero releases immediately). Observe the screen afterward to verify the intended effect. target_description states the intended surface/objects and gesture purpose, not a claimed observation.", + "parameters": { + "properties": { + "phases": { + "description": "Explicit continuous phases or a single native long_press_drag object.", + "items": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "duration_ms": { + "maximum": 5000, + "minimum": 1, + "title": "Duration Ms", + "type": "integer" + }, + "pointers": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "maximum": 9, + "minimum": 0, + "title": "Id", + "type": "integer" + }, + "path": { + "items": { + "items": { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + "maxItems": 128, + "minItems": 1, + "title": "Path", + "type": "array" + }, + "control_points": { + "anyOf": [ + { + "items": { + "items": { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Two cubic Bezier control points. When supplied, path must contain only start and end; Android builds the curve natively.", + "title": "Control Points" + } + }, + "required": [ + "id", + "path" + ], + "title": "GesturePointer", + "type": "object" + }, + "maxItems": 10, + "minItems": 1, + "title": "Pointers", + "type": "array" + } + }, + "required": [ + "duration_ms", + "pointers" + ], + "title": "GesturePhase", + "type": "object" + }, + { + "additionalProperties": false, + "description": "Hold, move to a caller-selected endpoint, optionally dwell, then release.", + "properties": { + "kind": { + "const": "long_press_drag", + "title": "Kind", + "type": "string" + }, + "start": { + "items": { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "title": "Start", + "type": "array" + }, + "end": { + "items": { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "title": "End", + "type": "array" + }, + "control_points": { + "anyOf": [ + { + "items": { + "items": { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Two cubic Bezier controls between start and end. Omit for a straight path.", + "title": "Control Points" + }, + "duration_ms": { + "default": 800, + "maximum": 5000, + "minimum": 1, + "title": "Duration Ms", + "type": "integer" + }, + "hold_ms": { + "anyOf": [ + { + "maximum": 5000, + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Omit to use the device long-press timeout plus 150ms.", + "title": "Hold Ms" + }, + "release_delay_ms": { + "default": 0, + "description": "Keep the same finger down at the endpoint for this long, then lift. Zero lifts immediately after moving; independent of movement duration.", + "maximum": 5000, + "minimum": 0, + "title": "Release Delay Ms", + "type": "integer" + } + }, + "required": [ + "kind", + "start", + "end" + ], + "title": "LongPressDrag", + "type": "object" + } + ] + }, + "type": "array" + }, + "target_description": { + "description": "Target surface/objects and gesture purpose.", + "type": "string" + } + }, + "required": [ + "phases", + "target_description" + ], + "type": "object" + } + } } } diff --git a/tests/fixtures/action_surfaces/tool_declarations.json b/tests/fixtures/action_surfaces/tool_declarations.json index 33f5ac85..15a32693 100644 --- a/tests/fixtures/action_surfaces/tool_declarations.json +++ b/tests/fixtures/action_surfaces/tool_declarations.json @@ -310,5 +310,194 @@ ] } } + }, + "PERFORM_GESTURE_TOOL": { + "type": "function", + "function": { + "name": "perform_gesture", + "description": "[ACTION] Perform continuous touch gestures that a normal swipe cannot express: simultaneous multi-finger input, continuous holds and drags, or delayed release. Coordinates use 0-1000 screen space. In explicit phases, pointers move together; a one-point path holds still. Keep the same pointer IDs and join endpoints across phases to maintain contact until the final release. Optional control_points define a cubic Bezier path with two endpoints and two controls. Alternatively, provide one long_press_drag entry with start and end coordinates and optional control_points. The endpoint is freely chosen, including inside the screen. duration_ms is movement time; release_delay_ms is the endpoint hold before lifting (zero releases immediately). Observe the screen afterward to verify the intended effect. target_description states the intended surface/objects and gesture purpose, not a claimed observation.", + "parameters": { + "type": "object", + "properties": { + "phases": { + "type": "array", + "items": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "duration_ms": { + "maximum": 5000, + "minimum": 1, + "type": "integer" + }, + "pointers": { + "items": { + "additionalProperties": false, + "properties": { + "id": { + "maximum": 9, + "minimum": 0, + "type": "integer" + }, + "path": { + "items": { + "items": { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + "maxItems": 128, + "minItems": 1, + "type": "array" + }, + "control_points": { + "anyOf": [ + { + "items": { + "items": { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Two cubic Bezier control points. When supplied, path must contain only start and end; Android builds the curve natively." + } + }, + "required": [ + "id", + "path" + ], + "type": "object" + }, + "maxItems": 10, + "minItems": 1, + "type": "array" + } + }, + "required": [ + "duration_ms", + "pointers" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Hold, move to a caller-selected endpoint, optionally dwell, then release.", + "properties": { + "kind": { + "const": "long_press_drag", + "type": "string" + }, + "start": { + "items": { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + "end": { + "items": { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + "control_points": { + "anyOf": [ + { + "items": { + "items": { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Two cubic Bezier controls between start and end. Omit for a straight path." + }, + "duration_ms": { + "default": 800, + "maximum": 5000, + "minimum": 1, + "type": "integer" + }, + "hold_ms": { + "anyOf": [ + { + "maximum": 5000, + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Omit to use the device long-press timeout plus 150ms." + }, + "release_delay_ms": { + "default": 0, + "description": "Keep the same finger down at the endpoint for this long, then lift. Zero lifts immediately after moving; independent of movement duration.", + "maximum": 5000, + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "start", + "end" + ], + "type": "object" + } + ] + }, + "description": "Explicit continuous phases or a single native long_press_drag object." + }, + "target_description": { + "type": "string", + "description": "Target surface/objects and gesture purpose." + } + }, + "required": [ + "phases", + "target_description" + ] + } + } } } diff --git a/tests/tools/operator/test_operator_tools.py b/tests/tools/operator/test_operator_tools.py index 8fe40927..3f248125 100644 --- a/tests/tools/operator/test_operator_tools.py +++ b/tests/tools/operator/test_operator_tools.py @@ -25,6 +25,10 @@ async def test_operator_shell_tools(): "click": {"target": 1, "times": 1, "delay_ms": 100}, "input_text": {"text": "hello", "target": 1, "clear_exist": True}, "swipe": {"gesture": "up"}, + "perform_gesture": { + "phases": [{"kind": "long_press_drag", "start": [300, 400], "end": [650, 550]}], + "target_description": "move the selected object", + }, "press_key": {"key": "ENTER"}, "manage_app": {"action": "launch", "app_name": "Settings"}, "wait_for_delay": {"time_in_ms": 1000}, diff --git a/tests/unit/agents/test_operator_transcript.py b/tests/unit/agents/test_operator_transcript.py index 0fa9b5b7..076c3e11 100644 --- a/tests/unit/agents/test_operator_transcript.py +++ b/tests/unit/agents/test_operator_transcript.py @@ -53,9 +53,10 @@ ) # SHA-256 snapshots of the legacy system message with the fixed inputs below. +# Updated for perform_gesture in the physical/turn-ending action lists. # Update these when an intentional template change alters the rendered prompt. -GOLDEN_EMPTY_PLAN = "513e0341014b9a2342f608ad4dd173b7dcf5704ce53ee028ba302a9a1b696f96" -GOLDEN_SENTINEL_PLAN = "012fe1b08645266d5a7e518023e20fc6fbfa65d4f2d985847ebb06e0184c67ee" +GOLDEN_EMPTY_PLAN = "5ce954de33ba8c77aa14597b1bd78539a48e3e11009626c70fc04fb98e157715" +GOLDEN_SENTINEL_PLAN = "561f3e0717a6a5e09dee0bb9b79c9b28ed96afa9221258295f02222ac92d3370" SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8") diff --git a/tests/unit/agents/test_prompt_assembly.py b/tests/unit/agents/test_prompt_assembly.py index f924f3d9..f0acb4fa 100644 --- a/tests/unit/agents/test_prompt_assembly.py +++ b/tests/unit/agents/test_prompt_assembly.py @@ -33,6 +33,7 @@ "click", "input_text", "swipe", + "perform_gesture", "press_key", "manage_app", "wait_for_delay", @@ -131,23 +132,25 @@ def test_reduced_enums_stay_well_formed(): available_tools=OPERATOR_PROMPT_TOOLSET - {"manage_app", "wait_for_delay"}, ) assert ( - "Physical device actions (`click`, `input_text`, `swipe`, `press_key`, `long_press`)" in out + "Physical device actions (`click`, `input_text`, `swipe`, `perform_gesture`, `press_key`, `long_press`)" + in out ) assert ( - "Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`, or `press_key`)" in out + "Turn-Ending Action (`click`, `swipe`, `perform_gesture`, `input_text`, `long_press`, or `press_key`)" + in out ) def test_full_set_enum_slots_render_verbatim(): - """With every tool present the historical enumeration wording is reproduced.""" + """The new primitive appears alongside the existing physical actions.""" template = load_operator_prompts()["main_template"] out = apply_operator_prompt_contract(template) assert ( - "Physical device actions (`click`, `input_text`, `swipe`, `press_key`," + "Physical device actions (`click`, `input_text`, `swipe`, `perform_gesture`, `press_key`," " `manage_app`, `wait_for_delay`, `long_press`)" in out ) assert ( - "Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`," + "Turn-Ending Action (`click`, `swipe`, `perform_gesture`, `input_text`, `long_press`," " `press_key`, `manage_app`, or `wait_for_delay`)" in out ) assert "Helper/Subagent tools (`ask_explorer`, `ask_diagnoser`, `video_analyzer`)" in out @@ -162,6 +165,21 @@ def test_full_set_enum_slots_render_verbatim(): assert "analyze_task_output" not in out +def test_without_gesture_the_upstream_action_enumerations_are_unchanged(): + out = apply_operator_prompt_contract( + load_operator_prompts()["main_template"], + available_tools=OPERATOR_PROMPT_TOOLSET - {"perform_gesture"}, + ) + assert ( + "Physical device actions (`click`, `input_text`, `swipe`, `press_key`," + " `manage_app`, `wait_for_delay`, `long_press`)" in out + ) + assert ( + "Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`," + " `press_key`, `manage_app`, or `wait_for_delay`)" in out + ) + + # --- Flash prompt assembly ----------------------------------------------------------- diff --git a/tests/unit/mcp/test_action_specs.py b/tests/unit/mcp/test_action_specs.py index d89c2777..fb635d92 100644 --- a/tests/unit/mcp/test_action_specs.py +++ b/tests/unit/mcp/test_action_specs.py @@ -83,6 +83,7 @@ def test_tool_declarations_match_fixture(): "LONG_PRESS_TOOL": "long_press", "INPUT_TEXT_TOOL": "input_text", "SWIPE_TOOL": "swipe", + "PERFORM_GESTURE_TOOL": "perform_gesture", "PRESS_KEY_TOOL": "press_key", "MANAGE_APP_TOOL": "manage_app", "WAIT_FOR_DELAY_TOOL": "wait_for_delay", @@ -121,6 +122,7 @@ def test_validator_declaration_order_is_stable(): "long_press", "input_text", "swipe", + "perform_gesture", "press_key", "read_note", "list_notes", @@ -138,6 +140,18 @@ def _without_optional_noise(schema: dict) -> dict: ``default`` and the ``null`` alternative say "optional"; the declaration says that through ``required`` alone. """ + + # LangChain retains Pydantic titles inside array-item unions, while the + # declaration projector removes them. Compare semantics, retaining bounds, + # required fields, defaults inside models, and every union alternative. + def without_titles(value): + if isinstance(value, list): + return [without_titles(v) for v in value] + if isinstance(value, dict): + return {k: without_titles(v) for k, v in value.items() if k != "title"} + return value + + schema = without_titles(schema) out = {k: v for k, v in schema.items() if k not in ("anyOf", "default")} if "anyOf" in schema: members = [m for m in schema["anyOf"] if m.get("type") != "null"] diff --git a/tests/unit/mcp/test_gesture_compatibility.py b/tests/unit/mcp/test_gesture_compatibility.py new file mode 100644 index 00000000..b8fc3131 --- /dev/null +++ b/tests/unit/mcp/test_gesture_compatibility.py @@ -0,0 +1,201 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pin existing action contracts and execution behavior while adding gestures.""" + +from copy import deepcopy +import hashlib +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock, patch + +from langchain_core.utils.function_calling import convert_to_openai_tool +import pytest + +from artemis.agents.validator.action_execution import exec_action +from artemis.agents.validator.execution_loop import _process_action +from artemis.mcp.action_server import build_action_server +from artemis.mcp.action_specs import OPERATOR_SHELL_ORDER, operator_shell_tool, tool_declaration +from artemis.mcp.action_types import ActionCode, ActionResult +from artemis.mcp.actuators import MockActuator +from artemis.utils.task_tree import format_action_clean, format_action_intent, format_result_clean + + +# Canonical JSON digests of the upstream 351ca84 fixtures, before perform_gesture. +# These pins are independent of the new gesture fixture and require no Git at test time. +LEGACY_SCHEMAS = { + "operator": "51882b1d081f24fa0dc8e79cd74a94ed13ee3eccda84cdc233bbbd0515cf8ad0", + "declarations": "ba06a25d8587769f9c518329706f6f1f15826a071a4db995f41c85a73c9d796f", + "wire": "ea41842f9a3440dd2eef2ccb5e156c8a136fb13319e5c7751f3d0653daacdd85", +} + + +def digest(value): + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + +@pytest.mark.asyncio +async def test_existing_action_schemas_and_descriptions_match_upstream(): + operator = { + name: convert_to_openai_tool(operator_shell_tool(name)) + for name in OPERATOR_SHELL_ORDER + if name != "perform_gesture" + } + declarations = { + name.upper() + "_TOOL": dict(tool_declaration(name)) + for name in ( + "click", + "click_sequence", + "long_press", + "input_text", + "swipe", + "press_key", + "manage_app", + "wait_for_delay", + ) + } + tools = await build_action_server(MockActuator()).list_tools() + wire = { + t.name: { + "description": t.description, + "inputSchema": t.inputSchema, + "outputSchema": t.outputSchema, + } + for t in tools + if t.name != "perform_gesture" + } + assert digest(operator) == LEGACY_SCHEMAS["operator"] + assert digest(declarations) == LEGACY_SCHEMAS["declarations"] + assert digest(wire) == LEGACY_SCHEMAS["wire"] + + +LEGACY_ACTIONS = [ + ( + {"action": "tap", "normalized_coordinates": [300, 400]}, + "click", + {"target": [300, 400], "times": 1, "delay_ms": 100}, + ), + ( + {"action": "swipe", "normalized_coordinates": [300, 400, 650, 550], "duration": 800}, + "swipe", + {"start": [300, 400], "end": [650, 550], "duration_ms": 800}, + ), + ( + {"action": "focus_and_input_text", "normalized_coordinates": [300, 400], "text": "hello"}, + "input_text", + {"target": [300, 400], "text": "hello", "clear_exist": False}, + ), + ( + {"action": "launch_app", "app_name": "settings"}, + "manage_app", + {"action": "launch", "app_name": "settings"}, + ), + ({"action": "press_key", "keycode": "KEYCODE_HOME"}, "press_key", {"key": "home"}), + ({"action": "back"}, "press_key", {"key": "back"}), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action,name,wire", LEGACY_ACTIONS) +@pytest.mark.parametrize("ok", [True, False]) +async def test_existing_action_results_do_not_add_history_fields(action, name, wire, ok): + original = deepcopy(action) + result = ( + ActionResult.success(name, "completed") + if ok + else ActionResult.failure(name, "outcome unknown", code=ActionCode.TIMEOUT) + ) + session = Mock(call=AsyncMock(return_value=result)) + assert await exec_action(SimpleNamespace(device=None), session, action) == ( + (True, "") if ok else (False, "outcome unknown") + ) + session.call.assert_awaited_once_with(name, wire) + assert action == original + assert format_result_clean({"status": "dispatched", "execution": [action]}) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "action", ["tap", "swipe", "focus_and_input_text", "launch_app", "press_key", "perform_gesture"] +) +@pytest.mark.parametrize("burst", [False, True]) +@pytest.mark.parametrize("throws", [False, True]) +async def test_retry_change_is_scoped_to_perform_gesture(action, burst, throws): + execute = ( + AsyncMock(side_effect=TimeoutError("outcome unknown")) + if throws + else AsyncMock(return_value=(False, "outcome unknown")) + ) + node = SimpleNamespace(_exec_action=execute) + with ( + patch( + "artemis.agents.validator.execution_loop._run_precondition_gate", + AsyncMock(return_value=(True, None, "")), + ), + patch( + "artemis.agents.validator.execution_loop._capture_live_screenshot", + AsyncMock(return_value=None), + ), + patch("artemis.agents.validator.execution_loop.asyncio.sleep", AsyncMock()), + ): + outcome = await _process_action( + node, Mock(), Mock(), {"action": action}, action, "", burst=burst + ) + expected_attempts = 1 if burst or action in ("launch_app", "perform_gesture") else 2 + assert not outcome.success + assert execute.await_count == expected_attempts + assert outcome.action_item == { + "action": action, + "attempts": ["outcome unknown"] * expected_attempts, + } + + +@pytest.mark.parametrize( + "action,past,intent", + [ + ( + {"action": "click", "target": [300, 400], "target_description": "button"}, + "Tapped 'button' (self-described) at [300, 400]", + "tap 'button' (self-described) at [300, 400]", + ), + ( + {"action": "swipe", "coordinates": [300, 400, 650, 550], "duration": 800}, + "Swiped from [300, 400] to [650, 550] over 800ms", + "swipe from [300, 400] to [650, 550] over 800ms", + ), + ( + { + "action": "input_text", + "text": "hello", + "target": [300, 400], + "target_description": "search field", + }, + "Inputted 'hello' into 'search field' (self-described) at [300, 400]", + "type 'hello' into 'search field' (self-described) at [300, 400]", + ), + ({"action": "press_key", "key": "home"}, "Pressed key 'home'", "press key 'home'"), + ], +) +def test_existing_history_rendering_matches_upstream(action, past, intent): + assert format_action_clean(action) == past + assert format_action_intent(action) == intent + assert format_result_clean({"status": "dispatched", "execution": [action]}) is None + assert ( + format_result_clean( + {"status": "failed", "execution": [{**action, "attempts": ["timeout"]}]} + ) + == "Error: timeout" + ) diff --git a/tests/unit/mcp/test_gestures.py b/tests/unit/mcp/test_gestures.py new file mode 100644 index 00000000..21845442 --- /dev/null +++ b/tests/unit/mcp/test_gestures.py @@ -0,0 +1,452 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Multi-touch semantics, model reachability, and no-replay transport regressions.""" + +import asyncio +import json +from unittest.mock import AsyncMock, Mock, patch + +import pytest +from pydantic import ValidationError +from artemis.clients.accessibility_client import AccessibilityClient +from artemis.mcp.gestures import validate_phases +from artemis.mcp.action_names import to_canonical_call +from artemis.mcp.action_specs import operator_shell_tool, tool_declaration +from artemis.mcp.action_executor import McpActionExecutor, _ArgError +from artemis.mcp.action_session import ActionSession +from artemis.mcp.action_server import build_action_server +from artemis.mcp.actuators.mock import MockActuator +from artemis.mcp.action_types import ActionCode + + +def phase(points=None, duration=500): + return { + "duration_ms": duration, + "pointers": points + or [{"id": i, "path": [[x, 300], [x, 700]]} for i, x in enumerate([300, 500, 700])], + } + + +def drag(): + return [ + phase([{"id": 0, "path": [[300, 400]]}], 700), + phase([{"id": 0, "path": [[300, 400], [980, 400]]}], 800), + phase([{"id": 0, "path": [[980, 400]]}], 1000), + ] + + +def curved_drag(): + phases = drag() + phases[1]["pointers"][0]["control_points"] = [[450, 300], [800, 300]] + return phases + + +def long_press_drag(delay=0): + return [ + { + "kind": "long_press_drag", + "start": [300, 400], + "end": [650, 550], + "duration_ms": 800, + "release_delay_ms": delay, + } + ] + + +@pytest.mark.parametrize("delay", [0, 1200, 5000]) +def test_long_press_drag_keeps_endpoint_hold_distinct_from_movement(delay): + from artemis.mcp.gestures import gesture_duration_bound_ms + + plan = long_press_drag(delay) + assert validate_phases(plan) == plan + assert gesture_duration_bound_ms(plan) == 5000 + 800 + delay + plan[0]["hold_ms"] = 700 + assert gesture_duration_bound_ms(validate_phases(plan)) == 700 + 800 + delay + schema = json.dumps(tool_declaration("perform_gesture").parameters) + assert '"release_delay_ms"' in schema and '"long_press_drag"' in schema + + +@pytest.mark.parametrize( + "field,value", + [ + ("release_delay_ms", -1), + ("release_delay_ms", 5001), + ("release_delay_ms", True), + ("release_delay_ms", 0.5), + ("hold_ms", 0), + ("direction", "right"), + ("start", [1001, 0]), + ("end", [-1, 400]), + ("end", [400]), + ("duration_ms", 0), + ("control_points", [[100, 200]]), + ("control_points", [[100, 200], [300, 1001]]), + ], +) +def test_invalid_long_press_drag_rejected(field, value): + plan = long_press_drag() + plan[0][field] = value + with pytest.raises(ValidationError): + validate_phases(plan) + + +def test_long_press_drag_cannot_mix_with_manual_phases(): + with pytest.raises(ValidationError): + validate_phases(long_press_drag() + drag()) + with pytest.raises(ValidationError): + validate_phases(drag() + long_press_drag()) + + +@pytest.mark.parametrize("endpoint", [[650, 550], [400, 200], [20, 400], [980, 400]]) +def test_long_press_drag_keeps_caller_endpoint_and_bezier_controls(endpoint): + plan = long_press_drag(1200) + plan[0]["end"] = endpoint + plan[0]["control_points"] = [[400, 250], [600, 250]] + assert validate_phases(plan) == plan + executor = McpActionExecutor(Mock(), actuator=Mock()) + _, wire, _, _ = executor._translate( + "perform_gesture", {"phases": plan, "target_description": "drag selected object"}, Mock() + ) + assert wire["phases"] == plan + + +def test_long_press_drag_requires_an_explicit_endpoint(): + plan = long_press_drag() + del plan[0]["end"] + with pytest.raises(ValidationError): + validate_phases(plan) + + +def test_curve_controls_are_preserved_without_host_sampling(): + assert validate_phases(curved_drag()) == curved_drag() + schema = json.dumps(tool_declaration("perform_gesture").parameters) + assert "control_points" in schema + assert "Bezier" in operator_shell_tool("perform_gesture").description + + +@pytest.mark.parametrize( + "controls", + [ + [], + [[1, 2]], + [[1, 2]] * 3, + [[-1, 2], [3, 4]], + [[1.5, 2], [3, 4]], + [[True, 2], [3, 4]], + [[1, 2, 3], [3, 4]], + ], +) +def test_invalid_curve_controls_rejected(controls): + phases = curved_drag() + phases[1]["pointers"][0]["control_points"] = controls + with pytest.raises(ValidationError): + validate_phases(phases) + + +@pytest.mark.parametrize("path", [[[300, 400]], [[300, 400], [500, 400], [980, 400]]]) +def test_curve_requires_only_start_and_end(path): + phases = curved_drag() + phases[1]["pointers"][0]["path"] = path + with pytest.raises(ValidationError): + validate_phases(phases) + + +def test_three_fingers_and_continuous_drag_are_valid(): + assert len(validate_phases([phase()])[0]["pointers"]) == 3 + assert validate_phases(drag()) == drag() + + +@pytest.mark.parametrize( + "mutate", + [ + lambda p: p[0]["pointers"][0]["path"][0].__setitem__(0, True), + lambda p: p[0]["pointers"][0]["path"][0].__setitem__(0, 1.5), + lambda p: p[0]["pointers"][0]["path"][0].__setitem__(0, "30"), + lambda p: p[0]["pointers"][0]["path"][0].__setitem__(0, 1001), + lambda p: p[0]["pointers"][0].__setitem__("id", 1), + lambda p: p[0].__setitem__("duration_ms", 0), + lambda p: p[0].__setitem__("duration_ms", 5001), + lambda p: p[0].__setitem__("unexpected", True), + lambda p: p[0]["pointers"][0].__setitem__("path", []), + ], +) +def test_invalid_input_is_rejected(mutate): + p = [phase()] + mutate(p) + with pytest.raises(ValidationError): + validate_phases(p) + + +@pytest.mark.parametrize("change", ["jump", "ids", "budget"]) +def test_continuity_and_total_budget(change): + p = drag() + if change == "jump": + p[1]["pointers"][0]["path"][0][0] = 500 + elif change == "ids": + p[1]["pointers"][0]["id"] = 1 + else: + p = [phase([{"id": 0, "path": [[500, 500]]}], 5000)] * 7 + with pytest.raises(ValidationError): + validate_phases(p) + + +def test_model_schema_has_nested_paths_and_shared_semantics(): + shell = operator_shell_tool("perform_gesture") + args = shell.args_schema(phases=drag(), target_description="launcher icon") + assert validate_phases(args.phases) == drag() + declaration = tool_declaration("perform_gesture") + encoded = json.dumps(declaration.parameters) + assert "$ref" not in encoded and '"duration_ms"' in encoded and '"pointers"' in encoded + assert "maintain contact" in shell.description and "Observe the screen" in shell.description + + +@pytest.mark.parametrize("plan", [drag(), curved_drag(), long_press_drag(), long_press_drag(1200)]) +def test_pro_and_flash_lower_to_the_same_plan_without_early_execution(plan): + from artemis.agents.operator.operator import OperatorNode + + node = OperatorNode(Mock()) + arguments = {"phases": plan, "target_description": "launcher icon"} + decisions, error = node._translate_and_validate_tool( + {"name": "perform_gesture", "args": arguments}, Mock() + ) + assert error is None and decisions[0]["target_description"] == "launcher icon" + pro_name, pro_args = to_canonical_call(decisions[0]) + executor = McpActionExecutor(Mock(), actuator=Mock()) + name, wire, _, recorded = executor._translate("perform_gesture", arguments, Mock()) + assert (name, wire) == (pro_name, pro_args) + assert recorded["target_description"] == "launcher icon" + assert "target_description" not in wire + with pytest.raises(_ArgError): + executor._translate("perform_gesture", {"phases": drag()}, Mock()) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("plan", [drag(), long_press_drag(), long_press_drag(1200)]) +async def test_mcp_executes_all_phases_as_one_action(plan): + actuator = MockActuator() + session = ActionSession(build_action_server(actuator)) + await session.start() + try: + result = await session.call("perform_gesture", {"phases": plan}) + assert result.ok, result + actions = [a for a in actuator.action_history if a["action"] == "perform_gesture"] + assert actions == [{"action": "perform_gesture", "phases": plan}] + finally: + await session.aclose() + + +@pytest.mark.asyncio +async def test_missing_or_incomplete_helper_does_not_succeed(): + actuator = MockActuator() + actuator.controller.perform_gesture = AsyncMock( + return_value={"success": False, "status": "unsupported", "error": "old helper"} + ) + assert (await actuator.perform_gesture([phase()])).code == ActionCode.UNSUPPORTED + actuator.controller.perform_gesture.return_value = { + "success": True, + "status": "completed", + "release_confirmed": False, + } + assert not (await actuator.perform_gesture([phase()])).ok + + +def client_with_capabilities(): + client = AccessibilityClient("test-device") + client._http = Mock( + side_effect=[ + json.dumps( + {"capabilities": ["perform_gesture"], "gesture_continuation": True} + ).encode(), + json.dumps( + {"success": True, "status": "completed", "release_confirmed": True} + ).encode(), + ] + ) + return client + + +def test_helper_receives_one_plan_with_duration_aware_timeout(): + client = client_with_capabilities() + assert client.perform_gesture(drag(), "request-1")["success"] + args, kwargs = client._http.call_args + assert args == ( + "/action", + {"cmd": "perform_gesture", "request_id": "request-1", "phases": drag()}, + ) + assert kwargs["timeout"] == 12.5 + + +def test_helper_probe_prevents_actions_on_old_apk(): + client = client_with_capabilities() + client._http.side_effect = [b"{}"] + assert client.perform_gesture(drag(), "request-1")["status"] == "unsupported" + assert client._http.call_count == 1 + + +def test_old_helper_rejects_curve_before_any_touch(): + client = client_with_capabilities() + assert client.perform_gesture(curved_drag(), "curve")["status"] == "unsupported" + assert client._http.call_count == 1 + + +def test_old_helper_rejects_long_press_drag_without_input(): + client = client_with_capabilities() + assert client.perform_gesture(long_press_drag(), "long-press-drag")["status"] == "unsupported" + assert client._http.call_count == 1 + + +def test_long_press_drag_transmitted_without_host_geometry_and_with_full_timeout(): + client = client_with_capabilities() + client._http.side_effect = [ + b'{"capabilities":["perform_gesture","gesture_long_press_drag"],"gesture_continuation":true}', + b'{"success":true,"status":"completed","release_confirmed":true}', + ] + assert client.perform_gesture(long_press_drag(1200), "long-press-drag")["success"] + assert client._http.call_args.args[1]["phases"] == long_press_drag(1200) + assert client._http.call_args.kwargs["timeout"] == 17 + + +def test_curved_long_press_drag_checks_capability_before_input(): + client = client_with_capabilities() + client._http.side_effect = [ + b'{"capabilities":["perform_gesture","gesture_long_press_drag"],"gesture_continuation":true}', + ] + plan = long_press_drag() + plan[0]["control_points"] = [[400, 250], [600, 250]] + assert client.perform_gesture(plan, "curve")["status"] == "unsupported" + assert client._http.call_count == 1 + + +def test_curved_long_press_drag_reaches_helper_with_interior_endpoint(): + client = client_with_capabilities() + client._http.side_effect = [ + b'{"capabilities":["perform_gesture","gesture_long_press_drag","gesture_cubic_bezier"],"gesture_continuation":true}', + b'{"success":true,"status":"completed","release_confirmed":true}', + ] + plan = long_press_drag(1000) + plan[0]["control_points"] = [[400, 250], [600, 250]] + assert client.perform_gesture(plan, "curve")["success"] + assert client._http.call_args.args[1]["phases"] == plan + + +@pytest.mark.parametrize("plan", [drag(), long_press_drag()]) +def test_helper_without_continuation_rejects_before_input(plan): + client = client_with_capabilities() + client._http.side_effect = [ + b'{"capabilities":["perform_gesture","gesture_long_press_drag"],"gesture_continuation":false}', + ] + assert client.perform_gesture(plan, "android7")["status"] == "unsupported" + assert client._http.call_count == 1 + + +def test_curve_controls_reach_native_helper_unchanged(): + client = client_with_capabilities() + client._http.side_effect = [ + b'{"capabilities":["perform_gesture","gesture_cubic_bezier"],"gesture_continuation":true}', + b'{"success":true,"status":"completed","release_confirmed":true}', + ] + assert client.perform_gesture(curved_drag(), "curve")["success"] + assert client._http.call_args.args[1]["phases"] == curved_drag() + + +def test_lost_action_response_is_not_replayed(): + client = client_with_capabilities() + client._http.side_effect = [ + b'{"capabilities":["perform_gesture"],"gesture_continuation":true}', + TimeoutError(), + ] + with pytest.raises(TimeoutError): + client.perform_gesture(drag(), "request-1") + assert client._http.call_count == 2 + + +@pytest.mark.asyncio +async def test_driver_cancel_targets_the_same_request_and_waits_for_release(): + import threading + from artemis.drivers.android.adb_driver import AndroidAdbDriver + + started, released = threading.Event(), threading.Event() + ids = [] + + def execute(phases, request_id): + ids.append(request_id) + started.set() + assert released.wait(5) + return {"success": False, "status": "cancelled", "release_confirmed": True} + + def cancel(request_id): + assert request_id == ids[0] + released.set() + return {"status": "cancelling"} + + client = Mock() + client.perform_gesture.side_effect = execute + client.cancel_gesture.side_effect = cancel + driver = AndroidAdbDriver("test-device", Mock()) + with patch("artemis.clients.accessibility_client.AccessibilityClient", return_value=client): + task = asyncio.create_task(driver.perform_gesture(drag())) + assert await asyncio.to_thread(started.wait, 5) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert released.is_set() and client.perform_gesture.call_count == 1 + + +@pytest.mark.asyncio +async def test_gesture_uses_existing_result_without_mutating_history(): + from copy import deepcopy + from types import SimpleNamespace + from artemis.agents.validator.action_execution import exec_action + + actuator = MockActuator() + result = await actuator.perform_gesture(drag()) + assert result.ok and result.code == ActionCode.OK + assert result.message == "Gesture completed. Observe the screen to verify the intended effect." + assert "release_confirmed" in result.detail + assert "release_confirmed" not in result.message + session = Mock(call=AsyncMock(return_value=result)) + action = {"action": "perform_gesture", "phases": drag()} + original = deepcopy(action) + assert await exec_action(SimpleNamespace(device=None), session, action) == (True, "") + assert action == original + + +@pytest.mark.asyncio +async def test_continuous_gesture_is_not_automatically_retried_after_unknown_outcome(): + from types import SimpleNamespace + from artemis.agents.validator.execution_loop import _process_action + + node = SimpleNamespace(_exec_action=AsyncMock(return_value=(False, "outcome unknown"))) + with ( + patch( + "artemis.agents.validator.execution_loop._run_precondition_gate", + AsyncMock(return_value=(True, None, "")), + ), + patch( + "artemis.agents.validator.execution_loop._capture_live_screenshot", + AsyncMock(return_value=None), + ), + ): + outcome = await _process_action( + node, + Mock(), + Mock(), + {"action": "perform_gesture", "phases": drag()}, + "perform_gesture", + "", + burst=False, + ) + assert not outcome.success + assert node._exec_action.await_count == 1