From 6b23d43eec74ef0d65f959517b513de674368ca6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ru=CC=84dolfs=20Bundulis?= Date: Wed, 30 Sep 2026 11:32:35 -0400 Subject: [PATCH] Read the active display, and let a client name one The framebuffer stream picked the first display reporting `displayClass == 0`, a heuristic written to skip tvOS TVOut rather than to choose between panels. A foldable reports an integrated panel per fold state and lights one of them, and which of those two comes first is not the one that is lit -- so the stream could attach to a dark panel and copy black frames for the life of the boot. It now reads the active integrated display, which is what the screenshot commands already do. A runtime whose provider cannot report displays keeps the old heuristic, since that is all that was ever available there. `Configure` lets a client name a display instead, and fixes the scale, for the life of the stream: geometry a client has allocated against cannot change under it, so the message is accepted at most once and only before the first copy. It is answered with the chosen display's geometry, so it doubles as a probe. Streams that never send one behave exactly as before. `list_displays` reports what there is to choose from, since a unique id is not something a client can otherwise discover. Co-Authored-By: Claude Opus 5 --- .../CompanionServiceProvider.swift | 6 + .../FramebufferStreamMethodHandler.swift | 120 ++++++++++++++---- .../ListDisplaysMethodHandler.swift | 43 +++++++ .../Commands/SimulatorDisplayCommands.swift | 2 +- .../Framebuffer/Framebuffer.swift | 5 + proto/idb.proto | 40 ++++++ 6 files changed, 188 insertions(+), 28 deletions(-) create mode 100644 CompanionLib/SwiftServer/MethodHandlers/ListDisplaysMethodHandler.swift diff --git a/CompanionLib/SwiftServer/CompanionServiceProvider.swift b/CompanionLib/SwiftServer/CompanionServiceProvider.swift index 77e4dd9bd3..a5bd7184ac 100644 --- a/CompanionLib/SwiftServer/CompanionServiceProvider.swift +++ b/CompanionLib/SwiftServer/CompanionServiceProvider.swift @@ -396,6 +396,12 @@ final class CompanionServiceProvider: Idb_CompanionService.SimpleServiceProtocol } } + func list_displays(request: Idb_ListDisplaysRequest, context: ServerContext) async throws -> Idb_ListDisplaysResponse { + return try await trackedUnaryCall(context, request: request) { + try await ListDisplaysMethodHandler(target: target).handle(request: request, context: context) + } + } + func crash_delete(request: Idb_CrashLogQuery, context: ServerContext) async throws -> Idb_CrashLogResponse { return try await trackedUnaryCall(context, request: request) { try await CrashDeleteMethodHandler(commandExecutor: commandExecutor) diff --git a/CompanionLib/SwiftServer/MethodHandlers/FramebufferStreamMethodHandler.swift b/CompanionLib/SwiftServer/MethodHandlers/FramebufferStreamMethodHandler.swift index fae457dcc0..cd1355c90d 100644 --- a/CompanionLib/SwiftServer/MethodHandlers/FramebufferStreamMethodHandler.swift +++ b/CompanionLib/SwiftServer/MethodHandlers/FramebufferStreamMethodHandler.swift @@ -25,6 +25,9 @@ enum FramebufferStreamError: Error, LocalizedError { case sharedMemoryTooSmall(name: String, needed: Int, available: UInt64) case scaleFailed(status: Int) case scaleChanged(latched: Float?, requested: Float) + case configuredLate + case unknownDisplay(uniqueID: String) + case displaysUnavailable var errorDescription: String? { switch self { @@ -42,6 +45,12 @@ enum FramebufferStreamError: Error, LocalizedError { return "Shared memory \(name) holds \(available) bytes, the frame needs \(needed)" case let .scaleFailed(status): return "Scaling the framebuffer failed (vImage status \(status))" + case .configuredLate: + return "The stream is already reading a display; Configure has to arrive before the first copy" + case let .unknownDisplay(uniqueID): + return "No display with unique id \(uniqueID); ask list_displays for the ones this simulator has" + case .displaysUnavailable: + return "This runtime cannot report its displays, so a display cannot be selected by id" case let .scaleChanged(latched, requested): let was = latched.map { "\($0)" } ?? "none" return "The stream was opened at scale \(was); a later request asked for \(requested). The scale is fixed for the life of the stream" @@ -84,39 +93,50 @@ struct FramebufferStreamMethodHandler: @unchecked Sendable { throw RPCError(code: .failedPrecondition, message: FramebufferStreamError.simulatorRequired(targetDescription: String(describing: target)).localizedDescription) } - let framebuffer = try Framebuffer.mainScreenSurface(for: simulator, logger: targetLogger) - let attachment = try framebuffer.attach() - defer { attachment.cancel() } - - let state = FramebufferStreamState(surface: attachment.initialSurface) - - // The request loop returns on `stop`; the frame loop runs until the attachment finishes. Whichever - // ends first tears the other down, so a client disconnect and a display teardown both land here. - let requests = Task { - try await readRequests(requestStream, into: state, responseStream: responseStream) - } - let frames = Task { - try await serviceFrames(attachment, state: state) - } + // The display is not known until a Configure names one, so the attachment is made on the first + // request that needs it rather than up front. A stream that never configures attaches to the + // active integrated display, which is what every client got before Configure existed. + var attached: Attached? + var frames: Task? defer { - requests.cancel() - frames.cancel() + frames?.cancel() + attached?.attachment.cancel() } - _ = try await Task.select(requests, frames).value - } - - // MARK: - Loops - /// Reads control frames. A zero-length copy request is a geometry probe and is answered at once, - /// so a client can learn the frame size without waiting for the display to render. - private func readRequests( - _ requestStream: RequestStreamReader, - into state: FramebufferStreamState, - responseStream: RPCWriter - ) async throws { for try await request in requestStream { switch request.control { + case let .configure(configure): + guard attached == nil else { + try await responseStream.send( + Self.failure(FramebufferStreamError.configuredLate, sharedMemoryName: "")) + continue + } + do { + let established = try await attach( + simulator: simulator, + displayUniqueID: configure.hasDisplayUniqueID ? configure.displayUniqueID : nil, + scaleFactor: configure.hasScaleFactor ? configure.scaleFactor : nil) + attached = established + frames = Task { try await serviceFrames(established.attachment, state: established.state) } + // Answered with the chosen display's geometry, so Configure doubles as a probe. + try await responseStream.send( + await probeResponse(for: .init(), state: established.state)) + } catch { + try await responseStream.send(Self.failure(error, sharedMemoryName: "")) + } + case let .copyFramebuffer(copy): + if attached == nil { + do { + let established = try await attach(simulator: simulator, displayUniqueID: nil, scaleFactor: nil) + attached = established + frames = Task { try await serviceFrames(established.attachment, state: established.state) } + } catch { + try await responseStream.send(Self.failure(error, sharedMemoryName: copy.sharedMemoryName)) + continue + } + } + guard let state = attached?.state else { continue } guard copy.sharedMemoryLength > 0 else { try await responseStream.send(await probeResponse(for: copy, state: state)) continue @@ -124,12 +144,58 @@ struct FramebufferStreamMethodHandler: @unchecked Sendable { // Answered from the current surface rather than on the next rendered frame: a client pulls // when it wants a frame, and a still display renders none to wait for. try await responseStream.send(await copyResponse(for: copy, state: state)) + case .stop, .none: return } } } + /// A stream's display, once chosen. + private struct Attached { + let attachment: FramebufferAttachment + let state: FramebufferStreamState + } + + /// Resolves the display and attaches to it. + /// + /// Without an id this is the active integrated display -- the lit panel, which on a foldable is the + /// one the user is looking at. A runtime too old to report displays falls back to the main-screen + /// heuristic, which is all that was ever available there. + private func attach( + simulator: Simulator, displayUniqueID: String?, scaleFactor: Float? + ) async throws -> Attached { + let framebuffer: Framebuffer + if let displayUniqueID { + guard let displays = try? await simulator.displays.list() else { + throw FramebufferStreamError.displaysUnavailable + } + guard let display = displays.first(where: { $0.uniqueID == displayUniqueID }) else { + throw FramebufferStreamError.unknownDisplay(uniqueID: displayUniqueID) + } + framebuffer = try await Framebuffer.surface(for: display, simulator: simulator) + } else if let active = try await simulator.displays.activeIntegratedDisplayIfSupported() { + framebuffer = try await Framebuffer.surface(for: active, simulator: simulator) + } else { + framebuffer = try Framebuffer.mainScreenSurface(for: simulator, logger: targetLogger) + } + let attachment = try framebuffer.attach() + let state = FramebufferStreamState(surface: attachment.initialSurface) + if let scaleFactor { + _ = try state.scale(requesting: scaleFactor) + } + return Attached(attachment: attachment, state: state) + } + + private static func failure(_ error: any Error, sharedMemoryName: String) -> Idb_FramebufferStreamResponse { + Idb_FramebufferStreamResponse.with { + $0.sharedMemoryName = sharedMemoryName + $0.error = error.localizedDescription + } + } + + // MARK: - Loops + /// Tracks the surface the display currently holds, and ends when the display tears down. private func serviceFrames( _ attachment: FramebufferAttachment, diff --git a/CompanionLib/SwiftServer/MethodHandlers/ListDisplaysMethodHandler.swift b/CompanionLib/SwiftServer/MethodHandlers/ListDisplaysMethodHandler.swift new file mode 100644 index 0000000000..580951abf3 --- /dev/null +++ b/CompanionLib/SwiftServer/MethodHandlers/ListDisplaysMethodHandler.swift @@ -0,0 +1,43 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import CompanionUtilities +import FBControlCore +import FBSimulatorControl +import Foundation +import GRPCCore +import IDBGRPCSwift + +/// Reports the simulator's displays, so a client can name one when it opens a framebuffer stream. +/// +/// A runtime whose provider cannot report displays answers with none rather than guessing, which is +/// the same signal `framebuffer_stream` uses to fall back to its main-screen heuristic. +struct ListDisplaysMethodHandler { + + let target: any Target + + func handle(request: Idb_ListDisplaysRequest, context: ServerContext) async throws -> Idb_ListDisplaysResponse { + guard let simulator = target as? Simulator else { + throw RPCError(code: .failedPrecondition, message: "Displays can only be listed for a simulator") + } + let displays = (try? await simulator.displays.list()) ?? [] + return Idb_ListDisplaysResponse.with { + $0.displays = displays.map { display in + Idb_Display.with { + $0.uniqueID = display.uniqueID + $0.name = display.name + $0.isActive = display.isActive + $0.isPrimary = display.isPrimary + $0.isIntegrated = display.isIntegrated + $0.width = UInt32(display.bounds.width) + $0.height = UInt32(display.bounds.height) + $0.scale = display.scale + } + } + } + } +} diff --git a/FBSimulatorControl/Commands/SimulatorDisplayCommands.swift b/FBSimulatorControl/Commands/SimulatorDisplayCommands.swift index c7a8ff2aea..e13b26ee80 100644 --- a/FBSimulatorControl/Commands/SimulatorDisplayCommands.swift +++ b/FBSimulatorControl/Commands/SimulatorDisplayCommands.swift @@ -203,7 +203,7 @@ public final class SimulatorDisplayCommands: DisplayCommands, @unchecked Sendabl /// Returns nil only when the runtime lacks the feature, or the provider the fields, needed to /// select a display. - func activeIntegratedDisplayIfSupported() async throws -> SimulatorDisplay? { + public func activeIntegratedDisplayIfSupported() async throws -> SimulatorDisplay? { let snapshot = try await target().coreDevice.performIfSupported( action: SimulatorDisplayProtocol.action, service: SimulatorDisplayProtocol.service, input: CoreDeviceEmptyInput(), decode: SimulatorDisplayProtocol.snapshot) diff --git a/FBSimulatorControl/Framebuffer/Framebuffer.swift b/FBSimulatorControl/Framebuffer/Framebuffer.swift index 374ee9dd17..abdeee5aed 100644 --- a/FBSimulatorControl/Framebuffer/Framebuffer.swift +++ b/FBSimulatorControl/Framebuffer/Framebuffer.swift @@ -83,6 +83,11 @@ public final class Framebuffer: @unchecked Sendable { private let statsRecorder: FramebufferStatsRecorder private let logger: any ControlCoreLogger + /// The framebuffer of one named display, for a caller that has chosen from `displays.list()`. + public class func surface(for display: SimulatorDisplay, simulator: Simulator) async throws -> Framebuffer { + try await FramebufferSurfaceLocator.framebuffer(for: display, simulator: simulator) + } + public class func mainScreenSurface(for simulator: Simulator, logger: any ControlCoreLogger) throws -> Framebuffer { let surface = try FramebufferSurfaceLocator.mainDisplaySurface(for: simulator, logger: logger) return Framebuffer(surface: surface, logger: logger) diff --git a/proto/idb.proto b/proto/idb.proto index 55c015f8f4..bec648e428 100644 --- a/proto/idb.proto +++ b/proto/idb.proto @@ -70,6 +70,7 @@ service CompanionService { returns (stream VideoStreamResponse) {} rpc framebuffer_stream(stream FramebufferStreamRequest) returns (stream FramebufferStreamResponse) {} + rpc list_displays(ListDisplaysRequest) returns (ListDisplaysResponse) {} // Crash Operations rpc crash_delete(CrashLogQuery) returns (CrashLogResponse) {} rpc crash_list(CrashLogQuery) returns (CrashLogResponse) {} @@ -898,17 +899,33 @@ message VideoStreamResponse { } message FramebufferStreamRequest { + // Fixes what the stream reads for its lifetime. Optional, at most once, and only + // before the first copy: a client allocates against the geometry it is told, so + // neither the display nor the scale can change under it afterwards. Answered with + // the selected display's geometry, so a Configure doubles as a probe. + // + // Omitted entirely, the stream reads the active integrated display unscaled -- + // what every client did before this message existed. + message Configure { + // From `list_displays`. Unset selects the active integrated display. + optional string display_unique_id = 1; + // In (0, 1]. Unset copies at the display's own size. + optional float scale_factor = 2; + } message CopyFramebuffer { string shared_memory_name = 1; // A zero length asks only for the framebuffer's geometry, answered from the // current surface without waiting for a frame. uint64 shared_memory_length = 2; + // Superseded by Configure.scale_factor. Still honoured when no Configure was + // sent, where the first request naming a scale fixes it for the stream. optional float scale_factor = 3; } message Stop {} oneof control { CopyFramebuffer copy_framebuffer = 1; Stop stop = 2; + Configure configure = 3; } } @@ -920,6 +937,29 @@ message FramebufferInfo { string format = 5; } +message ListDisplaysRequest {} + +// One of the simulator's displays. A foldable reports an integrated panel per +// fold state, of which one is active; a runtime that cannot report displays +// answers with none rather than guessing. +message Display { + string unique_id = 1; + string name = 2; + // Lit and rendering. Exactly one integrated display is active at a time. + bool is_active = 3; + bool is_primary = 4; + // False for TVOut, CarPlay and the resizable scene display. + bool is_integrated = 5; + // Bounds in the display's own unrotated pixel space. + uint32 width = 6; + uint32 height = 7; + double scale = 8; +} + +message ListDisplaysResponse { + repeated Display displays = 1; +} + message FramebufferStreamResponse { string shared_memory_name = 1; optional uint64 bytes_written = 2;