Add live device screen streaming to the Screen Capture window

External/UnityLogcatServer/gradle.properties

@@ -1,7 +1,7 @@
 # Version of the wire protocol spoken between the Editor and the on-device server.
 # The Editor refuses to talk to a server whose protocol version it does not know,
 # so bump this whenever the packet layout in Protocol.java changes.
-serverProtocolVersion=1
+serverProtocolVersion=2
 
 # Android platform the server is compiled against (android.jar is taken from
 # $SDK/platforms/android-<androidPlatform>/android.jar).

External/UnityLogcatServer/src/main/java/com/unity/android/logcat/server/Protocol.java

@@ -17,20 +17,25 @@
  *   u32  flags            FLAG_CONTROL_SUPPORTED if touch can be injected
  *   u32  serverPid        this process on the device, so the Editor can name it
  *
- * Frame, repeated, 28 byte header + payload:
+ * Frame, repeated, 32 byte header + payload:
  *   u64  ptsUs            microseconds since the first frame
  *   u32  width            pixels of the streamed image
  *   u32  height           pixels of the streamed image
  *   u32  displayWidth     pixels of the display it was captured from
  *   u32  displayHeight    pixels of the display it was captured from
+ *   u32  rotation         Surface.ROTATION_* the display was at, 0..3
  *   u32  payloadSize      bytes of encoded frame that follow
  *   u8[] payload
  * </pre>
  *
- * All four sizes travel with every frame because they change when the device is
+ * The geometry travels with every frame because it changes when the device is
  * rotated or folded, and the server starts a new capture session without saying so
  * on the socket. The reader is therefore never told out of band that the geometry
  * moved - it just reads the next frame.
+ * <p>
+ * The rotation is sent as well as the sizes because the two can disagree: an app
+ * that pins its orientation keeps the display at its own rotation however the user
+ * rotation is set, and the Editor needs to know which it got.
  */
 public final class Protocol {
     public static final int MAGIC = 0x554C5331;
@@ -60,7 +65,7 @@ public void writeStreamHeader(int codec, int flags, int serverPid) throws IOExce
     }
 
     public void writeFrame(long captureNs, int width, int height, int displayWidth, int displayHeight,
-                           byte[] payload, int payloadSize) throws IOException {
+                           int rotation, byte[] payload, int payloadSize) throws IOException {
         if (firstFrameNs < 0) {
             firstFrameNs = captureNs;
         }
@@ -69,6 +74,7 @@ public void writeFrame(long captureNs, int width, int height, int displayWidth,
         out.writeInt(height);
         out.writeInt(displayWidth);
         out.writeInt(displayHeight);
+        out.writeInt(rotation);
         out.writeInt(payloadSize);
         out.write(payload, 0, payloadSize);
         // Flushed per frame: this is a live stream, buffering a frame to fill the

External/UnityLogcatServer/src/main/java/com/unity/android/logcat/server/ScreenStreamer.java

@@ -76,6 +76,8 @@ public final class ScreenStreamer implements Closeable {
     private volatile IOException streamError;
     /** Read by TouchInjector from the control thread, hence volatile. */
     private volatile Size displaySize;
+    /** Read by the capture thread, written by whoever starts the session. */
+    private volatile int displayRotation;
 
     /**
      * Logical size of the display being captured, or null before capture starts. This
@@ -161,6 +163,7 @@ private void startSession(DisplayInfo info) throws IOException {
             }
 
             displaySize = info.getSize();
+            displayRotation = info.getRotation();
             videoSize = info.getSize().limit(options.getMaxSize());
             int width = videoSize.getWidth();
             int height = videoSize.getHeight();
@@ -338,6 +341,7 @@ private void encodeAndSend(Image image, long captureNs) throws IOException {
         protocol.writeFrame(captureNs, width, height,
             display == null ? 0 : display.getWidth(),
             display == null ? 0 : display.getHeight(),
+            displayRotation,
             jpeg.buffer(), jpeg.size());
 
         statsFrames++;

com.unity.mobile.android-logcat/CHANGELOG.md

@@ -8,6 +8,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Changes & Improvements:
  - Unity 6.0 or later is required.
+ - Screenshots are now saved to `AndroidLogcat/Captures` as `<device_id>_<number>.png`. Previously each capture overwrote a single file in the project's `Temp` folder, so earlier screenshots were lost.
+ - The Screen Capture window can be opened from **Window** > **Analysis** > **Android Screen Capture**, as well as from the Android Logcat window's **Tools** menu.
+ - The Screen Capture window now lists every saved screenshot. Click a row or use the Up and Down keys to cycle through them, or the cross next to a row to delete that screenshot.
+ - Ctrl+Shift+S, or Cmd+Shift+S on macOS, captures a screenshot while the Screen Capture window has focus. It appears in Edit > Shortcuts under "Android Logcat" and can be rebound there.
+ - Capturing a screenshot also writes a `.json` file beside it, recording the device it was captured from and when.
+ - The details beside a selected screenshot show the device, its Android version and display size, along with the image size, the file size and when it was captured. The device details read `Undefined` for a screenshot saved without them.
+ - The screenshot and the live view can be zoomed with Ctrl+Wheel (Cmd+Wheel on macOS), and the zoomed image moved with a Ctrl+middle mouse button drag or with the scrollbars that appear.
+ - The device screen can be viewed live from the Screen Capture window: select the "Live" row at the top of the screenshot list. Clicks, drags, the scroll wheel and typing are sent to the device as touch, scroll and key events, and Ctrl+A, Ctrl+C and Ctrl+V (Cmd on macOS) select all, copy and paste on the device using its own clipboard, and Back / Home / Overview buttons sit beside the image with the stream details. 
  - Tools->Open Terminal is now supported on Linux Editor.
  - Fixed Tools->Open Terminal not opening a window on Windows Editor when running on CoreCLR.
 

com.unity.mobile.android-logcat/Documentation~/TableOfContents.md

@@ -13,7 +13,8 @@
     * [Device Screen Capture window reference](screen-capture-window-reference.md)
     * [Capture a screenshot](screen-capture-screenshot.md)
     * [Capture a video](screen-capture-video.md)
-  * [Stacktace Utility](stacktrace-utility.md)
+    * [View the device screen live](screen-capture-live-stream.md)
+  * [Stacktrace Utility](stacktrace-utility.md)
     * [Stacktrace Utility window reference](stacktrace-utility-window-reference.md)
     * [Set up Stacktrace Utility](stacktrace-utility-set-up.md)
     * [Resolve a stacktrace](stacktrace-utility-resolve.md) 

com.unity.mobile.android-logcat/Documentation~/android-logcat-settings.md

@@ -7,6 +7,8 @@ To open the Android Logcat Settings window in the Unity Editor, go to **Edit** >
 * [Message Colors](#message-colors)
 * [Memory Window](#memory-window)
 * [Packages](#packages)
+* [Capture Settings](#capture-settings)
+    * [Live Stream](#live-stream)
 * [Stacktrace Regex](#stacktrace-regex)
 * [Symbol Extensions](#symbol-extensions)
 
@@ -48,6 +50,34 @@ Use the **Request Interval ms** setting to specify a time interval to request me
 
 Use the **Max Exited Packages** setting to specify the maximum number for the applications selected in the Package Selector that are now closed. This allows you to restrict the number of entries in the Package Selector dropdown for closed applications to prevent overpopulating the dropdown.
 
+## Capture Settings
+
+Use **Captures Folder** to choose where Unity saves screenshots. Unity names each screenshot after the ID of the device that captured it. The note under the field displays the full path of the folder.
+
+The **Captures Folder** setting accepts a relative or an absolute path:
+* A relative path starts at your project folder. The default value `AndroidLogcat/Captures` is a relative path. This path remains valid when you move the project or open it on another computer.
+* An absolute path is used for screenshots saved outside the project.
+
+To choose a folder, select **Browse...**. If the folder is inside the project, Unity stores it as a relative path. If the folder is outside the project, Unity stores it as an absolute path.
+
+The default, `AndroidLogcat/Captures`, is a folder in your project, beside `Assets`. It is outside `Assets`, so Unity doesn't import captures or include them in a build. Deleting the `Library` folder doesn't remove the captures folder. Captures in this folder remain until you delete them. Unity's own `.gitignore` file doesn't cover the captures folder. If you don't want captures in source control, add the captures folder to your own `.gitignore` file.
+
+Changing the folder takes effect immediately. Unity doesn't move existing captures to the new folder: they stay in the old folder, and the [Capture list](screen-capture-window-reference.md#capture-list) displays only the captures in the new folder. To keep using earlier captures, move them to the new folder yourself, together with their `.json` files.
+
+### Live Stream
+
+Use the **Live Stream** settings to control the [live view of the device's screen](screen-capture-live-stream.md). Streaming a display uses the device's CPU to compress each frame and the connection to your computer to carry it, so these settings trade image quality for both.
+
+|**Setting**|**Description**|
+|---|---|
+|**Max Size**|Specifies the longest side of the streamed image in pixels, between 256 and 2048. The device's display is scaled down to fit. The default value is 1024.|
+|**JPEG Quality**|Specifies the quality of each streamed frame, between 1 and 100. Lower values produce smaller frames and use less bandwidth. The default value is 70.|
+|**Max Frame Rate**|Specifies the highest number of frames per second the device sends, between 1 and 120. The device only sends a frame when its screen changes, so this is a limit rather than a rate. The default value is 30.|
+
+These settings apply when a stream starts. To apply them to a stream that is already running, turn **Live** off and on again in the [Device Screen Capture window](screen-capture-window-reference.md#capture-list).
+
+Use the **Reset** button in this section to restore the three Live Stream settings without changing any other setting.
+
 ## Stacktrace Regex
 
 ![Stacktrace Regex](images/logcat-settings-stacktrace-regex.png)
@@ -66,4 +96,4 @@ For more information, refer to [configure stacktrace regular expressions](stackt
 
 Use the **Symbol Extensions** setting to specify file extensions which are appended to symbol file names during stacktrace resolving.
 
-For example, if the symbol file name is `libunity` and the specified symbol extensions are `.so.sym`, `.so.dbg`, the Stacktrace Utility tool looks for `libunity.so.sym` and `libunity.so.dbg` symbols.
+For example, if the symbol file name is `libunity` and the specified symbol extensions are `.so.sym`, `.so.dbg`, the Stacktrace Utility tool looks for `libunity.so.sym` and `libunity.so.dbg` symbols.

com.unity.mobile.android-logcat/Documentation~/screen-capture-live-stream.md

@@ -0,0 +1,61 @@
+# View the device screen live
+
+This page explains how to use the [Screen Capture tool](screen-capture.md) to view the screen of the connected device as it changes, and how to control the device from the Unity Editor.
+
+The live view mirrors the device's display into the Unity Editor. Unlike a screenshot or a video, Unity doesn't save a live view. The view is live and stops when you select something else.
+
+## View the screen
+
+To view the device screen live:
+
+1. Open the [Device Screen Capture window](screen-capture-window-reference.md).
+2. In the [Toolbar](screen-capture-window-reference.md#toolbar), use **Device Selector** to specify the device to view.
+3. In the [Capture list](screen-capture-window-reference.md#capture-list), select **Live**. The Screen Capture tool starts streaming and displays the device's screen in the [Capture preview](screen-capture-window-reference.md#capture-preview).
+
+![The Device Screen Capture window showing a device's screen live, with the device controls beside it](images/device-screen-capture-live-view.png)
+
+The stream stops when you select another row in the list, close the window, or disconnect the device. Selecting a different device restarts the stream against the newly selected one.
+
+The device only sends a frame when its screen changes, so **Frame Rate** in the [Live view details](screen-capture-window-reference.md#live-view-details) drops to almost nothing while the device shows a still screen. This is expected: the last frame stays on display.
+
+If the stream stops on its own, for example because the device restarted, the [Capture preview](screen-capture-window-reference.md#capture-preview) displays a **Reconnect** button. Select **Reconnect** to restart the stream, or turn **Live** off and on again.
+
+## Control the device
+
+While the live view is streaming, the Screen Capture tool sends your input to the device. **Input** in the [Live view details](screen-capture-window-reference.md#live-view-details) shows whether the device accepts input.
+
+| **Input**                                  | **Result on the device**                                     |
+| ------------------------------------------ | ------------------------------------------------------------ |
+| Click or drag the image                    | Sends a tap or a swipe at the same place on the device's screen. |
+| Scroll the wheel over the image            | Scrolls the content under the pointer.                       |
+| Click the image, then press keys           | Sends the keys you press, including **Backspace**, **Enter**, **Tab**, and the arrow keys. |
+| **Shift** and an arrow key                 | Extends the selection in a text field on the device, as it would on the device's own keyboard. |
+| **Escape**                                 | Sends the Back key.                                          |
+| **Ctrl+A**, **Ctrl+C**, **Ctrl+V** (macOS: **Cmd+A**, **Cmd+C**, **Cmd+V**) | Selects all, copies, and pastes on the device, using the device's clipboard. |
+| The **◄**, **●**, and **■** buttons         | Sends the Back, Home, and Overview keys. Useful on a device that uses gesture navigation, where the mirrored image has no navigation bar to tap. |
+| The **Power**, **Vol +**, and **Vol -** buttons | Sends the Power, Volume Up, and Volume Down keys, which you can't reach through the mirrored screen. The device sends no frames while its screen is off. |
+| The **Device Rotation** buttons            | Rotates the device's screen. **Auto** returns rotation control to the device's accelerometer. |
+| The **Device Fold** buttons                | Folds and unfolds a foldable device, so that you can view its other display. These buttons appear only for a foldable device. |
+
+The Unity Editor handles other **Ctrl** and **Cmd** combinations, so its own shortcuts continue to work while the image has focus. To zoom the view instead of scrolling the device, press **Ctrl** (macOS: **Cmd**) and scroll over the image. For more information, refer to [Zoom into the image](screen-capture-window-reference.md#zoom-into-the-image).
+
+An app that locks its orientation doesn't rotate with the device. In this case, the live view rotates the image instead, so the rotation is still visible. The device's screen keeps the app's rotation.
+
+> [!NOTE]
+> The device's clipboard and your computer's clipboard don't exchange content. **Ctrl+C** copies content on the device and **Ctrl+V** pastes that content on the device.
+
+## Change the size, quality, and frame rate
+
+Streaming a display uses the device's CPU to compress each frame, and the connection to your computer to transfer it. To reduce CPU usage or bandwidth, use the [Live Stream](android-logcat-settings.md#live-stream) settings. To open these settings, go to **Edit** > **Preferences** > **Analysis** > **Android Logcat Settings** (macOS: **Unity** > **Settings** > **Analysis** > **Android Logcat Settings**).
+
+The settings apply when a stream starts. To apply them to a stream that is already running, select **Reconnect** in the live view, or turn **Live** off and on again.
+
+> [!NOTE]
+> If you connect the device with `adb connect` instead of USB, the stream shares the device's Wi-Fi connection with other adb tasks, including the message log. If the connection is slow, lower **Max Size** and **Max Frame Rate**, or connect the device by USB.
+
+## Additional resources
+
+* [Device Screen Capture window reference](screen-capture-window-reference.md)
+* [Capture a screenshot](screen-capture-screenshot.md)
+* [Capture a video](screen-capture-video.md)
+* [Connect to a device](connect-to-a-device.md)

com.unity.mobile.android-logcat/Documentation~/screen-capture-screenshot.md

@@ -3,12 +3,19 @@
 This page explains how to use the [Screen Capture tool](screen-capture.md) to capture a screenshot from the connected device and save it as a file on your computer.
 
 1. Open the [Device Screen Capture window](screen-capture-window-reference.md).
-2. In the [Toolbar](screen-capture-window-reference.md#toolbar), use **Device Selector** to specify to device to take a screenshot of.
+2. In the [Toolbar](screen-capture-window-reference.md#toolbar), use **Device Selector** to specify the device to take a screenshot of.
 3. Set **Screen Capture Mode** to **Screenshot**.
-4. Select **Capture**. The Screen Capture tool takes a screenshot of the connected device and displays it in the [Capture preview](screen-capture-window-reference.md#capture-preview).
-5. Select **Save As** and use the file explorer to save the image file to your computer.
+4. In the [Capture list](screen-capture-window-reference.md#capture-list), select **Live**, then select **Take Screenshot** in the [Live view details](screen-capture-window-reference.md#live-view-details). Alternatively, press **Ctrl+Shift+S** (macOS: **Cmd+Shift+S**) while the live view displays. The Screen Capture tool takes a screenshot of the connected device, displays it in the [Capture preview](screen-capture-window-reference.md#capture-preview), and adds it to the list.
+5. To save a copy of the screenshot elsewhere on your computer, right-click its row in the Capture list, select **Copy To...**, and use the file explorer to choose the location. To copy several screenshots into one folder, select them first and then right-click.
+
+Every screenshot you capture is kept, so you don't have to save one before taking the next. Screenshots are named `<device id>_<number>.png` and are stored in your project, in `AndroidLogcat/Captures`, unless you set [Captures Folder](android-logcat-settings.md#capture-settings) to another folder. The default folder is outside `Assets` and isn't part of your build, so use **Copy To...**, or a captures folder of your own, for a screenshot you want to share.
+
+Each screenshot is saved with a `.json` file of the same name beside it. The `.json` file records the device the screenshot came from and when it was captured. The device information includes the device name, ID, Android version, API level, ABI, and display size. The [Screenshot details](screen-capture-window-reference.md#screenshot-details) beside the image display this information. When you rename, delete, or copy a screenshot, the Screen Capture tool renames, deletes, or copies its `.json` file too, so a copy you save elsewhere still records the device it came from. A screenshot without a `.json` file still opens, and its device details display `Undefined`.
+
+To rename or delete a screenshot, or to show it in Explorer or Finder, use the [Capture list](screen-capture-window-reference.md#capture-list). To look at part of a screenshot more closely, [zoom into it](screen-capture-window-reference.md#zoom-into-the-image).
 
 ## Additional resources
 
 * [Device Screen Capture window reference](screen-capture-window-reference.md)
-* [Capture a video](screen-capture-video.md)
+* [Capture a video](screen-capture-video.md)
+* [View the device screen live](screen-capture-live-stream.md)