From 61a04fed55c32aa7227727a9940a61e91f339cec Mon Sep 17 00:00:00 2001 From: Siya <57148521+siyabendoezdemir@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:01:40 +0000 Subject: [PATCH 01/27] WIP: media lane (previous agent's uncommitted work, as found) Co-Authored-By: Claude Opus 5.5 --- FEATURES.md | 48 +- docs/feature-status.json | 24 +- src/main/context.ts | 10 +- src/main/media/agent.ts | 348 +++++++++++ src/main/media/index.ts | 131 ++++- src/main/media/manager.ts | 360 ++++++++++++ src/main/persist.ts | 3 +- src/main/tabs/context-menu.ts | 5 +- src/main/tabs/model.ts | 6 +- src/main/tabs/ops.ts | 7 +- .../src/shell/media/CaptureIndicator.tsx | 18 + src/renderer/src/shell/media/MediaTray.tsx | 221 +++++++ src/renderer/src/shell/media/media.css | 197 +++++++ src/renderer/src/shell/sidebar/ItemList.tsx | 3 + src/renderer/src/shell/sidebar/Sidebar.tsx | 3 + src/shared/actions.ts | 13 + src/shared/commands/media.ts | 24 + src/shared/media.ts | 225 +++++++ src/shared/rpc.ts | 4 +- src/shared/state.ts | 40 ++ tests/e2e/media.spec.ts | 548 ++++++++++++++++++ tests/unit/media.test.ts | 227 ++++++++ 22 files changed, 2428 insertions(+), 37 deletions(-) create mode 100644 src/main/media/agent.ts create mode 100644 src/main/media/manager.ts create mode 100644 src/renderer/src/shell/media/CaptureIndicator.tsx create mode 100644 src/renderer/src/shell/media/MediaTray.tsx create mode 100644 src/renderer/src/shell/media/media.css create mode 100644 src/shared/commands/media.ts create mode 100644 src/shared/media.ts create mode 100644 tests/e2e/media.spec.ts create mode 100644 tests/unit/media.test.ts diff --git a/FEATURES.md b/FEATURES.md index 33920811..febe128f 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -4,13 +4,13 @@ Every row is one behaviour from the clean-room spec. Status per ID: **spec’d** (every row) → **built** → **tested** (a unit or e2e test names the ID) → **feels right** (lead sign-off after a hands-on review against `feel.md`). -**Totals:** 485 spec’d · 46 built (21 partial) · 69 tested · 0 feel right. +**Totals:** 485 spec’d · 53 built (32 partial) · 82 tested · 0 feel right. | Priority | Spec’d | Built | Tested | |---|---|---|---| | ? | 7 | 0 | 0 | -| P0 | 174 | 30 | 40 | -| P1 | 221 | 15 | 27 | +| P0 | 174 | 35 | 49 | +| P1 | 221 | 17 | 31 | | P2 | 83 | 1 | 2 | P0 = milestone-1 core experience, P1 = parity, P2 = later. @@ -30,7 +30,7 @@ Spec: `spec/sidebar.md` | SB-07 | "+ New Tab" row | P0 | ✅ | ✅ e2e | – | + New Tab row with press animation opens the command bar | | SB-08 | Empty sidebar area: double-click, drag, right-click | P1 | – | – | – | | | SB-09 | Bottom bar: Library, Space switcher, "+" menu | P0 | 🟡 partial | ✅ e2e | – | scaffold | -| SB-10 | Transient trays above the bottom bar | P1 | – | – | – | | +| SB-10 | Transient trays above the bottom bar | P1 | 🟡 partial | – | – | media tray and call controls (media lane); downloads and update banner pending | | SB-11 | Show / hide the sidebar | P0 | 🟡 partial | – | – | scaffold | | SB-12 | Hover to reveal the hidden sidebar | P0 | – | – | – | | | SB-13 | Feedback when things happen while the sidebar is hidden | P1 | – | – | – | | @@ -70,8 +70,8 @@ Spec: `spec/tabs.md` | TB-12 | Rename a tab and set a custom icon | P1 | ✅ | ✅ e2e | – | double-click/menu rename, empty reverts; emoji icon picker | | TB-13 | Multi-select tabs | P1 | 🟡 partial | ✅ e2e | – | Cmd-click, Shift-click ranges, Escape/plain click clears; Cmd+W, Copy URL, menu pin/unpin/move/new folder and group drag act on the selection; open-as-split is the split lane's | | TB-14 | Tab context menu | P0 | 🟡 partial | ✅ e2e | – | native menus with every tab, pinned, Favorite, folder and multi-selection item; not theme-coloured; split/share items belong to other lanes | -| TB-15 | Audio, mute, microphone/camera and screen-share indicators | P1 | 🟡 partial | ✅ e2e | – | audio icon toggles mute; no mic/camera/screen-share indicators | -| TB-16 | Tab sleeping and background resource limits | P1 | 🟡 partial | ✅ e2e+unit | – | sleep timer (settings.sleepAfterMs), Favorites warm 4x, audible/selected never sleep | +| TB-15 | Audio, mute, microphone/camera and screen-share indicators | P1 | 🟡 partial | ✅ e2e | – | audio icon toggles mute; camera/microphone indicator (media lane); no screen-share indicator or Favorites notes | +| TB-16 | Tab sleeping and background resource limits | P1 | 🟡 partial | ✅ e2e+unit | – | sleep timer (settings.sleepAfterMs), Favorites warm 4x; selected, audible, playing (muted too), PiP and camera/mic tabs never sleep | | TB-17 | Tabs across multiple windows | P1 | 🟡 partial | ✅ e2e+unit | – | tab handoff; no two windows ever show the same tab (close, Clear, archive, moves, undo, drag-out); no blank window yet | | TB-18 | Undo sidebar actions | P1 | ✅ | ✅ e2e+unit | – | Cmd+Z undoes moves, close, Clear, archive below, renames, icons and folder changes (page text fields keep their own undo); Cmd+Shift+Z redoes moves, closes and renames | | TB-19 | Back on a fresh child tab returns to its opener | P2 | 🟡 partial | ✅ e2e | – | Back with no history closes a link-opened Today tab and selects its opener (shortcut/command) | @@ -187,7 +187,7 @@ Spec: `spec/commandbar.md` | CB-19 | Space and profile awareness | P0 | – | – | – | | | CB-20 | Sidebar stays interactive while the bar is open | P1 | – | – | – | | | CB-21 | Closing the bar and what happens to typed text | P0 | – | – | – | | -| CB-22 | Feel: open/close animation, latency and live updates | P0 | – | – | – | | +| CB-22 | Feel: open/close animation, latency and live updates | P0 | – | ✅ e2e | – | | | CB-23 | The bar in other contexts (split pane, private window, Mini Window) | P1 | – | – | – | | ## Split View (prefix SV) @@ -306,22 +306,22 @@ Spec: `spec/media.md` | ID | Feature | P | Built | Tested | Feels right | Notes | |---|---|---|---|---|---|---| -| MD-01 | Automatic picture-in-picture when leaving a video tab | P0 | – | – | – | | -| MD-02 | Open picture-in-picture manually | P1 | – | – | – | | -| MD-03 | The floating player's look and placement | P0 | – | – | – | | -| MD-04 | Floating player controls | P0 | – | – | – | | -| MD-05 | Keyboard and media keys for the floating player | P1 | – | – | – | | -| MD-06 | Move, resize and zoom the floating player | P1 | – | – | – | | -| MD-07 | Ending picture-in-picture | P0 | – | – | – | | -| MD-08 | Turning automatic picture-in-picture off | P0 | – | – | – | | -| MD-09 | Sidebar media controls for background audio | P0 | – | – | – | | -| MD-10 | Several players: the media tray | P1 | – | – | – | | -| MD-11 | Audio and recording indicators, mute | P0 | – | – | – | | -| MD-12 | Media and the tab lifecycle | P0 | – | – | – | | -| MD-13 | Video calls: automatic call picture-in-picture | P1 | – | – | – | | -| MD-14 | Call controls in the sidebar | P1 | – | – | – | | +| MD-01 | Automatic picture-in-picture when leaving a video tab | P0 | ✅ | ✅ e2e+unit | – | Chromium's own PiP, requested by Orbit's isolated page script when a tab with a playing, unmuted video leaves the screen (tab, Space or split change); ends on return, still playing | +| MD-02 | Open picture-in-picture manually | P1 | ✅ | ✅ e2e | – | page right-click menu Picture in Picture on a video, Picture in Picture action, and the tray controller's PiP button | +| MD-03 | The floating player's look and placement | P0 | ✅ | ✅ e2e | – | the engine's native PiP window: bottom-right, 320x180, borderless, always on top, keeps aspect ratio, controls on hover; corners/shadow follow the platform | +| MD-04 | Floating player controls | P0 | 🟡 partial | ✅ e2e | – | native player: play/pause, Back to tab (Orbit selects the tab), x keeps audio playing in the sidebar, previous/next for Media Session pages, progress bar; no +/-15 s buttons, no ... menu (its choices are in the tray controller menu), no double-click | +| MD-05 | Keyboard and media keys for the floating player | P1 | 🟡 partial | – | – | Chromium hardware media key handling switched on; Space/arrow keys are the native player's; not verifiable under Xvfb | +| MD-06 | Move, resize and zoom the floating player | P1 | 🟡 partial | – | – | native window moves and resizes from its corners; no corner snap, zoom or remembered position | +| MD-07 | Ending picture-in-picture | P0 | ✅ | ✅ e2e | – | returning ends PiP still playing; Back to tab selects the tab; x keeps playing in the sidebar; tab close/unload or navigation closes it; an ended video leaves PiP without replay | +| MD-08 | Turning automatic picture-in-picture off | P0 | 🟡 partial | ✅ e2e+unit | – | global switch (settings.autoPip + action); per-site Always off / Off for today (until midnight) via media.setSitePip and the tray controller menu; muted tabs never trigger; site controls checkbox waits for the SB-04 panel | +| MD-09 | Sidebar media controls for background audio | P0 | ✅ | ✅ e2e+unit | – | controller above the bottom bar: favicon, Media Session title/artist or tab title (marquee), play/pause, next (plays to the track end), previous on hover, x pauses and hides; click jumps to the tab (Space switch); audible-only pages get mute; no artwork yet | +| MD-10 | Several players: the media tray | P1 | ✅ | ✅ e2e+unit | – | one controller per background tab, newest first, shows/hides by itself, PiP button sends a video back to PiP, works from every window; no travel animation | +| MD-11 | Audio and recording indicators, mute | P0 | 🟡 partial | ✅ e2e | – | pulsing camera/microphone indicator on tab rows (live camera/mic tracks the page shows); speaker/mute is TB-15's; no screen-share indicator, split-pane mute or switcher indicators | +| MD-12 | Media and the tab lifecycle | P0 | ✅ | ✅ e2e+unit | – | playing (muted too), PiP and camera/mic tabs never sleep, auto-archive, Clear or Archive Below; closing stops media, PiP and controller; isAnyMediaPlaying() is ready for the updates lane | +| MD-13 | Video calls: automatic call picture-in-picture | P1 | 🟡 partial | ✅ e2e | – | leaving a call tab puts the largest participant video (never the self view) in PiP; the native player shows mic/camera/hang-up only if the site registers them; no document PiP | +| MD-14 | Call controls in the sidebar | P1 | 🟡 partial | ✅ e2e+unit | – | background call tab gets a call controller with mic/camera toggles (track.enabled on the tracks the page shows) and jump; detection needs a self-view element | | MD-15 | Switch which tab is being screen-shared | P2 | – | – | – | | -| MD-16 | Feel: entering and leaving picture-in-picture | P1 | – | – | – | | +| MD-16 | Feel: entering and leaving picture-in-picture | P1 | 🟡 partial | – | – | tray slides up/down with the snappy spring (fade with reduced motion), play/pause flips at once; PiP open/close motion is the native window's; no Back to PiP travel animation | ## Archive, recently closed, history and sidebar backups (prefix AR) @@ -357,7 +357,7 @@ Spec: `spec/shortcuts.md` | KS-08 | Page navigation and page tools | P0 | – | – | – | | | KS-09 | Split view | P0 | – | – | – | | | KS-10 | Peek and Mini Window | P1 | – | – | – | | -| KS-11 | Media | P1 | – | – | – | | +| KS-11 | Media | P1 | 🟡 partial | – | – | hardware media keys via Chromium; Space/arrows in the native player; Mute Tab action is assignable | | KS-12 | Library, history, downloads, settings, extensions, help | P1 | – | – | – | | | KS-13 | Developer tools | P1 | – | – | – | | | KS-14 | Canvases, notes, capture and Tweaks | P2 | – | – | – | | @@ -405,7 +405,7 @@ Spec: `spec/feel.md` | FL-29 | Responsiveness budgets (input to visual) | P0 | – | – | – | | | FL-30 | What makes it feel native and smooth | P0 | – | – | – | | | FL-31 | Haptics (macOS trackpads) | P2 | – | – | – | | -| FL-32 | Picture-in-picture and media tray motion | P2 | – | – | – | | +| FL-32 | Picture-in-picture and media tray motion | P2 | 🟡 partial | – | – | media tray slides up with the snappy spring; PiP drag/fling physics are the native window's | ## Appearance (global) diff --git a/docs/feature-status.json b/docs/feature-status.json index 64703131..093c36c7 100644 --- a/docs/feature-status.json +++ b/docs/feature-status.json @@ -16,7 +16,7 @@ "TB-10": { "built": "yes", "note": "menu/action Duplicate (selected, below the original) and Alt-drag copy, both with history; Cmd-click Reload is the top bar's" }, "TB-12": { "built": "yes", "note": "double-click/menu rename, empty reverts; emoji icon picker" }, "TB-14": { "built": "partial", "note": "native menus with every tab, pinned, Favorite, folder and multi-selection item; not theme-coloured; split/share items belong to other lanes" }, - "TB-16": { "built": "partial", "note": "sleep timer (settings.sleepAfterMs), Favorites warm 4x, audible/selected never sleep" }, + "TB-16": { "built": "partial", "note": "sleep timer (settings.sleepAfterMs), Favorites warm 4x; selected, audible, playing (muted too), PiP and camera/mic tabs never sleep" }, "TB-22": { "built": "yes", "note": "Spaces, Favorites, pinned, Today, folders, selection restored; only the selected tab loads; Linux bound saving is core's" }, "PN-02": { "built": "yes", "note": "Cmd+D, drag above/below divider, menu, Pin to [Space] command" }, "PN-04": { "built": "yes", "note": "favicon click resets, Cmd/Ctrl-click keeps the page as a Today tab, Reset Tab action" }, @@ -49,7 +49,7 @@ "SB-22": { "built": "yes", "note": "Cmd/Ctrl+Shift+E focuses the sidebar; arrows, Enter, Escape, F2" }, "SB-23": { "built": "yes", "note": "selected tab scrolled into view (smooth, then settles); pill points to it while it is out of view; rows use content-visibility" }, "TB-13": { "built": "partial", "note": "Cmd-click, Shift-click ranges, Escape/plain click clears; Cmd+W, Copy URL, menu pin/unpin/move/new folder and group drag act on the selection; open-as-split is the split lane's" }, - "TB-15": { "built": "partial", "note": "audio icon toggles mute; no mic/camera/screen-share indicators" }, + "TB-15": { "built": "partial", "note": "audio icon toggles mute; camera/microphone indicator (media lane); no screen-share indicator or Favorites notes" }, "TB-18": { "built": "yes", "note": "Cmd+Z undoes moves, close, Clear, archive below, renames, icons and folder changes (page text fields keep their own undo); Cmd+Shift+Z redoes moves, closes and renames" }, "TB-19": { "built": "partial", "note": "Back with no history closes a link-opened Today tab and selects its opener (shortcut/command)" }, "TB-20": { "built": "yes", "note": "Cmd+Shift+C clean URL, Cmd+Shift+Alt+C Markdown, multi-select and folder Copy Links" }, @@ -65,5 +65,23 @@ "FD-09": { "built": "partial", "note": "Rename, Change Icon, Duplicate, Copy All Links, Paste URL, Move to, Delete; no Open [Folder] command" }, "TB-05": { "built": "partial", "note": "Ctrl+Tab / Ctrl+Shift+Tab step through the 5 most recent tabs while Ctrl is held (keyboard only); no overlay list yet" }, "FD-08": { "built": "yes", "note": "magnifier on folder hover opens a search popover (custom names, first match selected, Enter opens and peeks)" }, - "TB-17": { "built": "partial", "note": "tab handoff; no two windows ever show the same tab (close, Clear, archive, moves, undo, drag-out); no blank window yet" } + "TB-17": { "built": "partial", "note": "tab handoff; no two windows ever show the same tab (close, Clear, archive, moves, undo, drag-out); no blank window yet" }, + "MD-01": { "built": "yes", "note": "Chromium's own PiP, requested by Orbit's isolated page script when a tab with a playing, unmuted video leaves the screen (tab, Space or split change); ends on return, still playing" }, + "MD-02": { "built": "yes", "note": "page right-click menu Picture in Picture on a video, Picture in Picture action, and the tray controller's PiP button" }, + "MD-03": { "built": "yes", "note": "the engine's native PiP window: bottom-right, 320x180, borderless, always on top, keeps aspect ratio, controls on hover; corners/shadow follow the platform" }, + "MD-04": { "built": "partial", "note": "native player: play/pause, Back to tab (Orbit selects the tab), x keeps audio playing in the sidebar, previous/next for Media Session pages, progress bar; no +/-15 s buttons, no ... menu (its choices are in the tray controller menu), no double-click" }, + "MD-05": { "built": "partial", "note": "Chromium hardware media key handling switched on; Space/arrow keys are the native player's; not verifiable under Xvfb" }, + "MD-06": { "built": "partial", "note": "native window moves and resizes from its corners; no corner snap, zoom or remembered position" }, + "MD-07": { "built": "yes", "note": "returning ends PiP still playing; Back to tab selects the tab; x keeps playing in the sidebar; tab close/unload or navigation closes it; an ended video leaves PiP without replay" }, + "MD-08": { "built": "partial", "note": "global switch (settings.autoPip + action); per-site Always off / Off for today (until midnight) via media.setSitePip and the tray controller menu; muted tabs never trigger; site controls checkbox waits for the SB-04 panel" }, + "MD-09": { "built": "yes", "note": "controller above the bottom bar: favicon, Media Session title/artist or tab title (marquee), play/pause, next (plays to the track end), previous on hover, x pauses and hides; click jumps to the tab (Space switch); audible-only pages get mute; no artwork yet" }, + "MD-10": { "built": "yes", "note": "one controller per background tab, newest first, shows/hides by itself, PiP button sends a video back to PiP, works from every window; no travel animation" }, + "MD-11": { "built": "partial", "note": "pulsing camera/microphone indicator on tab rows (live camera/mic tracks the page shows); speaker/mute is TB-15's; no screen-share indicator, split-pane mute or switcher indicators" }, + "MD-12": { "built": "yes", "note": "playing (muted too), PiP and camera/mic tabs never sleep, auto-archive, Clear or Archive Below; closing stops media, PiP and controller; isAnyMediaPlaying() is ready for the updates lane" }, + "MD-13": { "built": "partial", "note": "leaving a call tab puts the largest participant video (never the self view) in PiP; the native player shows mic/camera/hang-up only if the site registers them; no document PiP" }, + "MD-14": { "built": "partial", "note": "background call tab gets a call controller with mic/camera toggles (track.enabled on the tracks the page shows) and jump; detection needs a self-view element" }, + "MD-16": { "built": "partial", "note": "tray slides up/down with the snappy spring (fade with reduced motion), play/pause flips at once; PiP open/close motion is the native window's; no Back to PiP travel animation" }, + "FL-32": { "built": "partial", "note": "media tray slides up with the snappy spring; PiP drag/fling physics are the native window's" }, + "KS-11": { "built": "partial", "note": "hardware media keys via Chromium; Space/arrows in the native player; Mute Tab action is assignable" }, + "SB-10": { "built": "partial", "note": "media tray and call controls (media lane); downloads and update banner pending" } } diff --git a/src/main/context.ts b/src/main/context.ts index cc8485d0..862216c9 100644 --- a/src/main/context.ts +++ b/src/main/context.ts @@ -1,4 +1,10 @@ -import type { HandlerDetails, WebContents, WindowOpenHandlerResponse } from 'electron' +import type { + ContextMenuParams, + HandlerDetails, + MenuItemConstructorOptions, + WebContents, + WindowOpenHandlerResponse, +} from 'electron' import type { TabId } from '../shared/state' import type { Perf } from './perf' import type { ActionRegistry, CommandRegistry, EventRegistry } from './registry' @@ -17,6 +23,8 @@ export interface Hooks { tabCreated: ((tabId: TabId, contents: WebContents) => void)[] /** Called when a tab's main frame finished a navigation (url committed). */ tabNavigated: ((tabId: TabId, contents: WebContents, url: string) => void)[] + /** Extra items at the top of a page's right-click menu (e.g. Picture in Picture on a video, MD-02). */ + pageContextMenu?: ((tabId: TabId, contents: WebContents, params: ContextMenuParams) => MenuItemConstructorOptions[])[] } /** Everything a feature module needs. Passed to each module's register(). */ diff --git a/src/main/media/agent.ts b/src/main/media/agent.ts new file mode 100644 index 00000000..186e1d5f --- /dev/null +++ b/src/main/media/agent.ts @@ -0,0 +1,348 @@ +/** + * Orbit's media page script (media lane). It runs in an isolated world, so the page can't see or + * change it, and only in tabs that started playing media (installed by MediaManager on + * `media-started-playing` / audible). It watches the page's media elements with capture-phase + * listeners (no timers, no polling) and reports changes through a long-poll: the main process + * calls `__orbitMedia.next(v)`, which resolves with a fresh report when something changed after + * report `v`. Commands (play, pause, seek, picture-in-picture, call toggles) run here too. + * + * Reports follow `AgentSnapshot` in src/shared/media.ts. + */ + +/** Isolated world for the media script (the tabs runtime uses 999, the fullscreen fix 1001). */ +export const MEDIA_WORLD = 1010 + +/** Installs the script once per document; returns the first report either way. */ +export const AGENT_SOURCE = `(() => { + const w = window + if (w.__orbitMedia) return w.__orbitMedia.state() + let version = 1 + let waiter = null + let timer = 0 + let lastPlayed = null + let pausedByOrbit = [] + let orbitExit = false + let pipEnd = null + let pipEnds = 0 + const watchedTracks = new WeakSet() + + const list = () => Array.from(document.querySelectorAll('video, audio')) + const running = (m) => !m.paused && !m.ended + const unmuted = (m) => !m.muted && m.volume > 0 + const frames = (m) => m.tagName === 'VIDEO' && m.videoWidth > 0 && m.videoHeight > 0 + const area = (m) => { + const r = m.getBoundingClientRect() + return r.width * r.height + } + const seekable = (m) => Number.isFinite(m.duration) && m.duration > 0 && m.seekable.length > 0 + const seekEnd = (m) => Math.min(m.duration, m.seekable.end(m.seekable.length - 1)) + + // The media the controls act on: playing first, then audible, then the last one played, + // then the largest video. + function main() { + let best = null + let bestScore = -1 + for (const m of list()) { + const started = running(m) || m === lastPlayed || (m.currentTime > 0 && !m.ended) + if (!started) continue + let s = 0 + if (running(m)) s += 8 + if (unmuted(m)) s += 4 + if (m === lastPlayed) s += 2 + if (frames(m)) s += 1 + (Math.min(area(m), 4e6) / 4e6) * 0.9 + if (s > bestScore) { + best = m + bestScore = s + } + } + return best + } + + // The page's main video for automatic picture-in-picture (MD-01): the largest playing, unmuted, + // visible video. Muted decorative loops and hidden players don't count. In a call (the page + // uses the camera or microphone), the largest participant video counts even when muted (call + // apps often play voices through separate audio elements), never the self view (MD-13). + function pipVideo() { + const inCall = localTracks().length > 0 + let best = null + let bestArea = 64 * 36 - 1 + for (const m of list()) { + if (m.tagName !== 'VIDEO' || !running(m) || !frames(m) || m.disablePictureInPicture) continue + if (!unmuted(m) && !(inCall && remoteVideo(m))) continue + const a = area(m) + if (a > bestArea) { + best = m + bestArea = a + } + } + return best + } + + // A video showing a stream from elsewhere (a call participant), not this device's camera. + function remoteVideo(m) { + const s = m.srcObject + if (!s || typeof s.getVideoTracks !== 'function') return false + const tracks = s.getVideoTracks() + return tracks.length > 0 && tracks.every((t) => { + let device = '' + try { + device = t.getSettings().deviceId || '' + } catch {} + return !inputs.has(device) + }) + } + + // The system's camera and microphone inputs. A page's own canvas or Web Audio streams have + // device ids too, so a track counts as the camera or microphone only if its device is one of these. + let inputs = new Set() + const askedAbout = new Set() + let enumerating = false + function refreshInputs() { + if (enumerating || !navigator.mediaDevices) return + enumerating = true + navigator.mediaDevices.enumerateDevices().then( + (list) => { + enumerating = false + const ids = list.filter((d) => d.kind === 'audioinput' || d.kind === 'videoinput').map((d) => d.deviceId) + inputs = new Set(ids.filter(Boolean)) + bump() + }, + () => { + enumerating = false + }, + ) + } + if (navigator.mediaDevices) navigator.mediaDevices.addEventListener('devicechange', refreshInputs) + + // Live camera and microphone tracks the page shows (a call's self view), best effort (MD-11). + function localTracks() { + const out = [] + let ask = false + for (const m of list()) { + const s = m.srcObject + if (!s || typeof s.getTracks !== 'function') continue + for (const t of s.getTracks()) { + if (t.readyState !== 'live') continue + let device = '' + try { + device = t.getSettings().deviceId || '' + } catch {} + if (!device) continue + if (!inputs.has(device)) { + // Look the device up once (a canvas or Web Audio stream never matches). + if (!askedAbout.has(device)) { + askedAbout.add(device) + ask = true + } + continue + } + if (!watchedTracks.has(t)) { + watchedTracks.add(t) + t.addEventListener('ended', bump) + } + if (!out.includes(t)) out.push(t) + } + } + if (ask) refreshInputs() + return out + } + + function state() { + const m = main() + let playing = false + let active = false + for (const x of list()) { + if (!running(x)) continue + active = true + if (unmuted(x)) playing = true + } + let title = '' + let artist = '' + try { + const md = navigator.mediaSession && navigator.mediaSession.metadata + if (md) { + title = String(md.title || '').slice(0, 200) + artist = String(md.artist || '').slice(0, 200) + } + } catch {} + const tracks = localTracks() + const mics = tracks.filter((t) => t.kind === 'audio') + const cams = tracks.filter((t) => t.kind === 'video') + return { + v: version, + playing, + active, + controllable: !!m, + video: !!(m && frames(m)), + pip: !!document.pictureInPictureElement, + pipCandidate: !!pipVideo(), + title, + artist, + seekable: !!(m && seekable(m)), + ended: !!(m && m.ended), + pipEnd, + mic: mics.length > 0, + micMuted: mics.length > 0 && mics.every((t) => !t.enabled), + camera: cams.length > 0, + cameraOff: cams.length > 0 && cams.every((t) => !t.enabled), + } + } + + function bump() { + version++ + if (waiter && !timer) timer = setTimeout(flush, 40) + } + function flush() { + timer = 0 + const resolve = waiter + waiter = null + if (resolve) resolve(state()) + } + + for (const type of ['play', 'playing', 'pause', 'ended', 'emptied', 'loadedmetadata', 'durationchange', 'volumechange', 'resize', 'enterpictureinpicture']) { + document.addEventListener( + type, + (e) => { + if (type === 'play' && e.target instanceof HTMLMediaElement) lastPlayed = e.target + bump() + // Track details (Media Session) often follow a track change by a moment. + if (type === 'loadedmetadata' || type === 'play') setTimeout(bump, 1000) + }, + true, + ) + } + + // A finished video leaves picture-in-picture and doesn't replay (MD-07). + document.addEventListener( + 'ended', + (e) => { + if (e.target === document.pictureInPictureElement && !e.target.loop) exitPip() + }, + true, + ) + + function endPip(reason) { + pipEnd = { n: ++pipEnds, reason } + bump() + } + + // How picture-in-picture ended (MD-07). The engine's × pauses a file-backed video right after + // leaving and its "Back to tab" doesn't, which tells the two apart. × keeps the media playing + // as audio (it then shows in the sidebar). A live stream can't tell them apart: treated as ×. + document.addEventListener( + 'leavepictureinpicture', + (e) => { + const v = e.target + if (orbitExit) { + orbitExit = false + endPip('orbit') + return + } + if (v.ended) return endPip('ended') + if (v.paused || v.srcObject) return endPip('close') + let done = false + const onPause = () => { + if (done) return + done = true + clearTimeout(timeout) + v.play().catch(() => {}) + endPip('close') + } + const timeout = setTimeout(() => { + if (done) return + done = true + v.removeEventListener('pause', onPause) + endPip('back') + }, 300) + v.addEventListener('pause', onPause, { once: true }) + }, + true, + ) + + async function exitPip() { + if (!document.pictureInPictureElement) return false + orbitExit = true + try { + await document.exitPictureInPicture() + } catch { + orbitExit = false + } + return false + } + + // auto: only the page's main playing, unmuted video (MD-01); otherwise any video with frames (MD-02). + async function pip(on, auto) { + if (!on) return exitPip() + const m = main() + const v = pipVideo() || (!auto && m && frames(m) && !m.disablePictureInPicture ? m : null) + if (!v) return false + if (document.pictureInPictureElement === v) return true + try { + await v.requestPictureInPicture() + return true + } catch { + return false + } + } + + function control(action, seconds) { + const all = list() + if (action === 'pause' || (action === 'toggle' && all.some(running))) { + pausedByOrbit = all.filter(running) + for (const x of pausedByOrbit) x.pause() + return pausedByOrbit.length > 0 + } + const m = main() + if (action === 'play' || action === 'toggle') { + const again = pausedByOrbit.filter((x) => x.isConnected) + pausedByOrbit = [] + const targets = again.length > 0 ? again : m ? [m] : [] + for (const x of targets) x.play().catch(() => {}) + return targets.length > 0 + } + if (!m) return false + if (action === 'seek' && seekable(m)) { + m.currentTime = Math.min(Math.max(m.currentTime + (Number(seconds) || 0), 0), seekEnd(m)) + return true + } + // Next: play to the end of the track, so the page moves on to its next one. + if (action === 'next' && seekable(m)) { + m.currentTime = seekEnd(m) + return true + } + if (action === 'previous' && seekable(m)) { + m.currentTime = 0 + if (m.paused) m.play().catch(() => {}) + return true + } + return false + } + + // Call controls (MD-14): turn the page's own camera or microphone tracks off or on again. + function call(action) { + const kind = action === 'mic' ? 'audio' : action === 'camera' ? 'video' : '' + const tracks = localTracks().filter((t) => t.kind === kind) + if (tracks.length === 0) return false + const enable = tracks.every((t) => !t.enabled) + for (const t of tracks) t.enabled = enable + bump() + return true + } + + w.__orbitMedia = { + state, + refresh: bump, + next(v) { + if (v !== version) return state() + // One waiting report per document: an older loop (say, from before a restart) ends. + if (waiter) waiter(null) + return new Promise((resolve) => { + waiter = resolve + }) + }, + control, + pip, + call, + } + return state() +})()` diff --git a/src/main/media/index.ts b/src/main/media/index.ts index 17cc8e2d..88b330a5 100644 --- a/src/main/media/index.ts +++ b/src/main/media/index.ts @@ -1,4 +1,131 @@ +import { app, Menu, type MenuItemConstructorOptions } from 'electron' +import type { TabId } from '../../shared/state' import type { MainContext } from '../context' +import { activeTabOf } from '../windows' +import { MEDIA_WORLD } from './agent' +import { MediaManager } from './manager' -/** Media feature (main process). Owned by its build lane; see docs/ARCHITECTURE.md. */ -export function registerMedia(_ctx: MainContext): void {} +/** + * Hardware media keys (play/pause, next, previous) control the playing media and the PiP video, as + * in other browsers (MD-05, KS-11). Electron leaves Chromium's handling off; it has to be switched + * on before the app is ready, so this runs when the module loads. Merged into any features other + * modules enabled. + */ +function enableFeatures(...names: string[]): void { + const current = app.commandLine.getSwitchValue('enable-features') + const list = current ? current.split(',').filter(Boolean) : [] + for (const name of names) if (!list.includes(name)) list.push(name) + app.commandLine.appendSwitch('enable-features', list.join(',')) +} +enableFeatures('HardwareMediaKeyHandling') + +/** Test-only handle (ORBIT_E2E=1): globalThis.__orbitMedia. */ +export interface MediaTestHooks { + manager: MediaManager + lastMenu: () => string[] + clickMenu: (path: string) => boolean +} + +/** Picture-in-picture, sidebar media controls and call controls (media lane; spec media.md). */ +export function registerMedia(ctx: MainContext): MediaManager { + const media = new MediaManager(ctx) + media.start() + const { commands, actions, store, windows } = ctx + const windowOf = (windowId: string | undefined) => windowId ?? windows.focusedWindowId() + + commands.register('media.control', ({ tabId, action, seconds }) => media.control(tabId, action, seconds)) + commands.register('media.pip', async ({ tabId, on = true }, c) => { + const windowId = windowOf(c.windowId) + const id = tabId ?? (windowId ? activeTabOf(store.state, windowId) : null) + return id ? media.setPip(id, on) : false + }) + commands.register('media.dismiss', ({ tabId }) => media.dismiss(tabId)) + commands.register('media.focus', ({ tabId }, c) => { + const windowId = windowOf(c.windowId) + if (windowId) media.focusTab(tabId, windowId) + }) + commands.register('media.menu', ({ tabId }, c) => { + const windowId = windowOf(c.windowId) + const items = media.menu(tabId, windowId) + if (items.length === 0) return + media.lastMenu = items + if (ctx.e2e) return + const win = windowId ? windows.get(windowId) : undefined + Menu.buildFromTemplate(items).popup(win ? { window: win } : undefined) + }) + commands.register('media.sitePip', ({ url }) => media.sitePip(url)) + commands.register('media.setSitePip', ({ url, mode }) => media.setSitePip(url, mode)) + commands.register('media.call', ({ tabId, action }) => media.callControl(tabId, action)) + + // Picture in Picture for the selected tab's video (MD-02), and the global switch (MD-08, ST-10). + actions.register('media.togglePip', ({ windowId }) => { + const id = windowId ? activeTabOf(store.state, windowId) : null + if (!id) return + void media.setPip(id, !store.state.tabs[id]?.media?.pip) + }) + actions.register('media.toggleAutoPip', ({ windowId }) => { + const on = !store.state.settings.autoPip + store.update((d) => { + d.settings.autoPip = on + }) + ctx.toast(windowId, on ? 'Automatic Picture in Picture is on' : 'Automatic Picture in Picture is off') + }) + + // The page's right-click menu on a video offers Picture in Picture (MD-02). + ctx.hooks.pageContextMenu ??= [] + ctx.hooks.pageContextMenu.push((tabId, contents, params): MenuItemConstructorOptions[] => { + if (params.mediaType !== 'video' || !params.mediaFlags?.canShowPictureInPicture) return [] + const showing = params.mediaFlags.isShowingPictureInPicture + const zoom = contents.getZoomFactor() || 1 + const x = Math.round(params.x / zoom) + const y = Math.round(params.y / zoom) + // The video under the pointer, not the page's main one. + const code = showing + ? 'document.exitPictureInPicture().then(() => true, () => false)' + : `(() => { const el = document.elementFromPoint(${x}, ${y}); const v = el && (el.tagName === 'VIDEO' ? el : el.closest('video')); return v ? v.requestPictureInPicture().then(() => true, () => false) : false })()` + return [ + { + label: 'Picture in Picture', + type: 'checkbox', + checked: showing, + click: () => { + void contents + .executeJavaScriptInIsolatedWorld(MEDIA_WORLD, [{ code }], true) + .catch(() => false) + // The page script follows picture-in-picture from now on (tray, return to the tab). + .then(() => media.watchTab(tabId as TabId)) + }, + }, + { type: 'separator' }, + ] + }) + + if (ctx.e2e) { + const hooks: MediaTestHooks = { + manager: media, + lastMenu: () => flatLabels(media.lastMenu), + clickMenu: (path) => clickMenuPath(media.lastMenu, path), + } + ;(globalThis as unknown as { __orbitMedia: MediaTestHooks }).__orbitMedia = hooks + } + return media +} + +function flatLabels(items: MenuItemConstructorOptions[], prefix = ''): string[] { + const out: string[] = [] + for (const item of items) { + if (!item.label) continue + out.push(`${prefix}${item.label}`) + if (Array.isArray(item.submenu)) out.push(...flatLabels(item.submenu, `${prefix}${item.label} > `)) + } + return out +} + +function clickMenuPath(items: MenuItemConstructorOptions[], path: string): boolean { + const [head, ...rest] = path.split(' > ') + const item = items.find((i) => i.label === head) + if (!item) return false + if (rest.length > 0) return Array.isArray(item.submenu) ? clickMenuPath(item.submenu, rest.join(' > ')) : false + ;(item.click as (() => void) | undefined)?.() + return !!item.click +} diff --git a/src/main/media/manager.ts b/src/main/media/manager.ts new file mode 100644 index 00000000..f4c3a062 --- /dev/null +++ b/src/main/media/manager.ts @@ -0,0 +1,360 @@ +/** + * Media in the main process (media lane): picture-in-picture, the sidebar media tray and call + * controls. Event driven, no polling: + * - a tab gets Orbit's page script (agent.ts) only once it plays media (`media-started-playing`, + * or it turns audible); the script reports changes through a long-poll; + * - tab switches arrive as store changes: a tab leaving the screen with a playing video goes to + * Chromium's own picture-in-picture (MD-01), and coming back ends it; + * - each tab's `media` record (shared/state.ts) holds what the sidebar needs, and `tray` says + * whether its controller shows (shared/media.ts settleTray). + */ +import type { MenuItemConstructorOptions, WebContents } from 'electron' +import { + type AgentSnapshot, + applySitePip, + type MediaAction, + mediaFromSnapshot, + type PipEndReason, + type SitePipMode, + sameMedia, + settleTray, + shouldAutoPip, + siteOf, + sitePipMode, +} from '../../shared/media' +import type { OrbitState, TabId, TabMedia, WindowId } from '../../shared/state' +import type { MainContext } from '../context' +import { displayedTabs } from '../tabs/model' +import { AGENT_SOURCE, MEDIA_WORLD } from './agent' + +interface Agent { + contents: WebContents + /** The last report. */ + snap?: AgentSnapshot + /** The last picture-in-picture ending already handled (reports repeat it). */ + pipEndSeen: number + /** A long-poll is running for this document. */ + looping: boolean +} + +const call = (fn: string, ...args: unknown[]) => + `window.__orbitMedia ? window.__orbitMedia.${fn}(${args.map((a) => JSON.stringify(a)).join(', ')}) : null` + +export class MediaManager { + private agents = new Map() + /** Tabs on screen in any window at the last state change. */ + private shown = new Set() + /** Last menu template (tests read it instead of a native popup). */ + lastMenu: MenuItemConstructorOptions[] = [] + + constructor( + private ctx: MainContext, + private now: () => number = Date.now, + ) {} + + start(): void { + this.shown = displayedTabs(this.ctx.store.state) + this.ctx.hooks.tabCreated.push((tabId, contents) => this.watch(tabId, contents)) + this.ctx.store.subscribe((next, prev) => this.onState(next, prev)) + } + + /** The page script's last report for a tab (tests and diagnostics). */ + snapshot(tabId: TabId): AgentSnapshot | undefined { + return this.agents.get(tabId)?.snap + } + + // ---- Page script lifecycle ---- + + private watch(tabId: TabId, contents: WebContents): void { + contents.on('media-started-playing', () => void this.install(tabId, contents)) + contents.on('audio-state-changed', (e) => { + if (e.audible) void this.install(tabId, contents) + // Media removed from the page pauses without an event the script can see: ask again. + else void this.run(tabId, call('refresh')) + }) + // A new document: its media (and picture-in-picture) went with the old one. + contents.on('did-navigate', () => this.drop(tabId, contents)) + contents.once('destroyed', () => this.drop(tabId, contents)) + } + + /** Installs the page script in a tab that plays media (once per document) and follows its reports. */ + private async install(tabId: TabId, contents: WebContents): Promise { + const existing = this.agents.get(tabId) + if (existing?.contents === contents && existing.looping) return + const agent: Agent = { contents, pipEndSeen: 0, looping: true } + this.agents.set(tabId, agent) + let snap: AgentSnapshot | null = null + try { + snap = await contents.executeJavaScriptInIsolatedWorld(MEDIA_WORLD, [{ code: AGENT_SOURCE }]) + } catch { + // The page went away meanwhile. + } + if (this.agents.get(tabId) !== agent) return + if (!snap) { + agent.looping = false + return + } + this.apply(tabId, agent, snap) + void this.follow(tabId, agent) + } + + private async follow(tabId: TabId, agent: Agent): Promise { + while (this.agents.get(tabId) === agent && !agent.contents.isDestroyed()) { + let snap: AgentSnapshot | null = null + try { + snap = await agent.contents.executeJavaScriptInIsolatedWorld(MEDIA_WORLD, [ + { code: call('next', agent.snap?.v ?? 0) }, + ]) + } catch { + break + } + if (this.agents.get(tabId) !== agent) return + if (!snap) break + this.apply(tabId, agent, snap) + } + if (this.agents.get(tabId) === agent) agent.looping = false + } + + /** Forgets a tab's media when its document or contents go away (MD-07, MD-12). */ + private drop(tabId: TabId, contents: WebContents): void { + const agent = this.agents.get(tabId) + if (agent && agent.contents !== contents) return + this.agents.delete(tabId) + this.ctx.store.update((d) => { + const tab = d.tabs[tabId] + if (tab?.media) delete tab.media + }) + } + + /** Runs a script function in the tab's page script; undefined if the tab has none. */ + private async run(tabId: TabId, code: string, gesture = false): Promise { + const agent = this.agents.get(tabId) + if (!agent || agent.contents.isDestroyed()) return undefined + try { + return (await agent.contents.executeJavaScriptInIsolatedWorld(MEDIA_WORLD, [{ code }], gesture)) as T + } catch { + return undefined + } + } + + /** Makes sure the tab has the page script (e.g. picture-in-picture for a video that never played). */ + watchTab(tabId: TabId): Promise { + return this.ensureAgent(tabId) + } + + private async ensureAgent(tabId: TabId): Promise { + if (this.agents.get(tabId)?.snap) return true + const contents = this.ctx.tabs.contents(tabId) + if (!contents) return false + await this.install(tabId, contents) + return !!this.agents.get(tabId)?.snap + } + + // ---- State ---- + + private apply(tabId: TabId, agent: Agent, snap: AgentSnapshot): void { + const first = !agent.snap + agent.snap = snap + if (first) agent.pipEndSeen = snap.pipEnd?.n ?? 0 + else if (snap.pipEnd && snap.pipEnd.n > agent.pipEndSeen) { + agent.pipEndSeen = snap.pipEnd.n + this.onPipEnd(tabId, snap.pipEnd.reason) + } + this.patch(tabId, (prev) => { + const next = mediaFromSnapshot(prev, snap) + // Keep the "opened automatically" mark while the same picture-in-picture lasts. + if (next.pip && prev?.autoPip) next.autoPip = true + return next + }) + } + + /** Updates a tab's media record and settles its tray membership; writes only real changes. */ + private patch(tabId: TabId, fn: (prev: TabMedia | undefined) => TabMedia | undefined): void { + const state = this.ctx.store.state + const tab = state.tabs[tabId] + if (!tab) return + const changed = fn(tab.media ? { ...tab.media } : undefined) + const settled = changed && settleTray(changed, tab, this.shown.has(tabId), this.now()) + if (sameMedia(tab.media, settled)) return + this.ctx.store.update((d) => { + const t = d.tabs[tabId] + if (!t) return + if (settled) t.media = settled + else delete t.media + }) + } + + private onState(next: OrbitState, prev: OrbitState): void { + if ( + next.tabs === prev.tabs && + next.windows === prev.windows && + next.splits === prev.splits && + next.spaces === prev.spaces && + next.settings === prev.settings + ) { + return + } + const before = this.shown + const shown = displayedTabs(next) + this.shown = shown + for (const id of before) if (!shown.has(id) && next.tabs[id]) this.left(id, next) + for (const id of shown) if (!before.has(id)) this.returned(id, next) + // Tray membership follows the screen, mute and audibility (MD-08, MD-09). + const now = this.now() + const updates: [TabId, TabMedia][] = [] + for (const id of this.agents.keys()) { + const tab = next.tabs[id] + if (!tab?.media) continue + let media = tab.media + if (shown.has(id) && media.dismissed) media = { ...media, dismissed: undefined } + const settled = settleTray(media, tab, shown.has(id), now) + if (settled !== tab.media) updates.push([id, settled]) + } + if (updates.length === 0) return + this.ctx.store.update((d) => { + for (const [id, media] of updates) { + const t = d.tabs[id] + if (!t?.media) continue + if (media.dismissed === undefined) delete media.dismissed + t.media = media + } + }) + } + + // ---- Picture-in-picture ---- + + /** A tab left the screen: a playing video continues in picture-in-picture (MD-01). */ + private left(tabId: TabId, state: OrbitState): void { + const agent = this.agents.get(tabId) + const tab = state.tabs[tabId] + if (!agent?.snap || !tab?.media || agent.contents.isDestroyed()) return + // The page script checks again when it runs; the report may be a moment old. + if (!shouldAutoPip(state.settings, tab, agent.snap.pipCandidate || agent.snap.video, this.now())) return + const optimistic = agent.snap.pipCandidate + if (optimistic) this.markPip(tabId, true, true) + void this.run(tabId, call('pip', true, true), true).then((ok) => { + if (ok) this.markPip(tabId, true, true) + else if (optimistic) this.markPip(tabId, false) + }) + } + + /** A tab came back on screen: its picture-in-picture ends and the video plays in the page (MD-01, MD-07). */ + private returned(tabId: TabId, state: OrbitState): void { + if (!state.tabs[tabId]?.media?.pip) return + void this.run(tabId, call('pip', false, false)) + } + + private markPip(tabId: TabId, pip: boolean, auto = false): void { + this.patch(tabId, (m) => { + if (!m) return m + const next = { ...m, pip } + if (pip && auto) next.autoPip = true + else if (!pip) delete next.autoPip + return next + }) + } + + /** The user ended picture-in-picture from the player (MD-04, MD-07). */ + private onPipEnd(tabId: TabId, reason: PipEndReason): void { + if (reason !== 'back') return + // "Back to tab": select the source tab in its window (and Space) and bring the window forward. + const windowId = this.ctx.windows.windowOfTab(tabId) ?? this.ctx.windows.focusedWindowId() + if (windowId) this.focusTab(tabId, windowId) + } + + /** Opens or closes picture-in-picture for a tab (MD-02, MD-10 "Back to PiP"). */ + async setPip(tabId: TabId, on: boolean): Promise { + if (!(await this.ensureAgent(tabId))) return false + const ok = await this.run(tabId, call('pip', on, false), true) + if (on && ok) this.markPip(tabId, true) + return !!ok + } + + // ---- Controls ---- + + async control(tabId: TabId, action: MediaAction, seconds?: number): Promise { + const gesture = action === 'play' || action === 'toggle' || action === 'previous' + const ok = await this.run(tabId, call('control', action, seconds ?? 0), gesture) + return !!ok + } + + /** The controller's ×: pause and hide until it plays again (MD-09). */ + async dismiss(tabId: TabId): Promise { + this.patch(tabId, (m) => (m ? { ...m, dismissed: true } : m)) + await this.run(tabId, call('control', 'pause', 0)) + } + + async callControl(tabId: TabId, action: 'mic' | 'camera'): Promise { + return !!(await this.run(tabId, call('call', action))) + } + + /** Selects a tab in a window (switching Space if needed) and brings that window forward (MD-09). */ + focusTab(tabId: TabId, windowId: WindowId): void { + void this.ctx.commands.run('tabs.activate', { tabId }, { windowId }).then(() => { + const win = this.ctx.windows.get(windowId) + if (!win || win.isDestroyed()) return + if (win.isMinimized()) win.restore() + win.focus() + }) + } + + // ---- Per-site choices (MD-08) ---- + + sitePip(url: string): SitePipMode { + return sitePipMode(this.ctx.store.state.settings, url, this.now()) + } + + setSitePip(url: string, mode: SitePipMode): void { + if (!siteOf(url)) return + const now = this.now() + this.ctx.store.update((d) => applySitePip(d.settings, url, mode, now)) + } + + // ---- Menus ---- + + /** The controller's "…" menu (MD-08, MD-10). */ + menu(tabId: TabId, windowId: WindowId | undefined): MenuItemConstructorOptions[] { + const tab = this.ctx.store.state.tabs[tabId] + if (!tab) return [] + const media = tab.media + const site = siteOf(tab.url) + const mode = this.sitePip(tab.url) + const items: MenuItemConstructorOptions[] = [ + { label: 'Go to Tab', click: () => windowId && this.focusTab(tabId, windowId) }, + ] + if (media?.video) { + items.push({ + label: media.pip ? 'Close Picture in Picture' : 'Picture in Picture', + click: () => void this.setPip(tabId, !media.pip), + }) + } + items.push({ + label: tab.muted ? 'Unmute Tab' : 'Mute Tab', + click: () => void this.ctx.commands.run('tabs.setMuted', { tabId, muted: !tab.muted }, { windowId }), + }) + if (site) { + items.push( + { type: 'separator' }, + { + label: `Automatic Picture in Picture on ${site}`, + submenu: [ + { label: 'On', type: 'radio', checked: mode === 'on', click: () => this.setSitePip(tab.url, 'on') }, + { + label: 'Turn Off for Today', + type: 'radio', + checked: mode === 'today', + click: () => this.setSitePip(tab.url, 'today'), + }, + { + label: 'Always Turn Off for This Site', + type: 'radio', + checked: mode === 'always', + click: () => this.setSitePip(tab.url, 'always'), + }, + ], + }, + ) + } + return items + } +} diff --git a/src/main/persist.ts b/src/main/persist.ts index 68a66c1c..6e46a150 100644 --- a/src/main/persist.ts +++ b/src/main/persist.ts @@ -115,6 +115,7 @@ export function normalizeState(input: Partial): OrbitState { tab.canGoBack = tab.canGoBack ?? false tab.canGoForward = tab.canGoForward ?? false tab.muted = tab.muted ?? false + delete tab.media } for (const win of Object.values(state.windows)) { win.focused = false @@ -337,7 +338,7 @@ function repairReferences(state: OrbitState): void { * touch nothing else (loading flips, focus, audio, download progress…) don't cause a write. */ const RUNTIME_ONLY: Record> = { - tabs: new Set(['loading', 'audible', 'asleep', 'progress', 'error', 'canGoBack', 'canGoForward']), + tabs: new Set(['loading', 'audible', 'asleep', 'progress', 'error', 'canGoBack', 'canGoForward', 'media']), windows: new Set(['focused', 'fullscreen', 'overlay']), downloads: new Set(['receivedBytes']), } diff --git a/src/main/tabs/context-menu.ts b/src/main/tabs/context-menu.ts index 4ea568ae..80074fa2 100644 --- a/src/main/tabs/context-menu.ts +++ b/src/main/tabs/context-menu.ts @@ -4,7 +4,10 @@ import type { MainContext } from '../context' /** Native right-click menu for web pages. */ export function showPageContextMenu(ctx: MainContext, tabId: TabId, contents: WebContents, params: ContextMenuParams) { - const items: MenuItemConstructorOptions[] = [] + // Items other features add (media: Picture in Picture on a video). + const items: MenuItemConstructorOptions[] = (ctx.hooks.pageContextMenu ?? []).flatMap((hook) => + hook(tabId, contents, params), + ) const openInNewTab = (url: string, activate: boolean) => { const tab = ctx.store.state.tabs[tabId] void ctx.commands.run('tabs.create', { url, spaceId: tab?.spaceId, activate }, {}) diff --git a/src/main/tabs/model.ts b/src/main/tabs/model.ts index b2eb244c..a8827cd2 100644 --- a/src/main/tabs/model.ts +++ b/src/main/tabs/model.ts @@ -3,6 +3,7 @@ */ import { DEFAULT_PROFILE_ID } from '../../shared/defaults' import { newId } from '../../shared/ids' +import { isMediaBusy } from '../../shared/media' import type { ItemId, OrbitState, Space, SpaceId, Tab, TabId, WindowId } from '../../shared/state' import { containerItems, @@ -252,7 +253,8 @@ export function autoArchiveCandidates(state: OrbitState, now: number): ItemId[] .map((t) => state.tabs[t]) .filter((t): t is Tab => !!t) if (tabs.length === 0) continue - if (tabs.some((t) => shown.has(t.id) || t.audible)) continue + // Tabs playing media, in picture-in-picture or using camera/microphone stay (MD-12). + if (tabs.some((t) => shown.has(t.id) || isMediaBusy(t))) continue const last = Math.max(...tabs.map((t) => t.lastActiveAt)) if (now - last >= after) out.push(id) } @@ -274,7 +276,7 @@ export function sleepCandidates(state: OrbitState, now: number, awake: Iterable< const out: TabId[] = [] for (const id of awake) { const tab = state.tabs[id] - if (!tab || shown.has(id) || tab.audible) continue + if (!tab || shown.has(id) || isMediaBusy(tab)) continue const limit = sectionOf(state, id) === 'favorite' ? after * FAVORITE_WARM_FACTOR : after if (now - tab.lastActiveAt >= limit) out.push(id) } diff --git a/src/main/tabs/ops.ts b/src/main/tabs/ops.ts index f83bf829..0e69cc10 100644 --- a/src/main/tabs/ops.ts +++ b/src/main/tabs/ops.ts @@ -5,6 +5,7 @@ */ import { createWindowState, MAX_FAVORITES } from '../../shared/defaults' import { isFolderId, isTabId, newId } from '../../shared/ids' +import { isMediaBusy } from '../../shared/media' import type { ArchivedTab, ContainerRef, @@ -357,7 +358,9 @@ export class TabOps { clearToday(spaceId: SpaceId): void { const space = this.state.spaces[spaceId] if (!space) return - const tabs = space.today.flatMap((id) => tabsInItem(this.state, id)).filter((id) => !this.state.tabs[id]?.audible) + const tabs = space.today + .flatMap((id) => tabsInItem(this.state, id)) + .filter((id) => !isMediaBusy(this.state.tabs[id])) this.closeTabs(tabs, 'cleared', { reselect: 'pinned', undoLabel: 'Clear Today' }) } @@ -372,7 +375,7 @@ export class TabOps { const tabs = space.today .slice(loc.index + 1) .flatMap((id) => tabsInItem(state, id)) - .filter((id) => !state.tabs[id]?.audible) + .filter((id) => !isMediaBusy(state.tabs[id])) this.closeTabs(tabs, 'cleared', { undoLabel: 'Archive Tabs Below' }) } diff --git a/src/renderer/src/shell/media/CaptureIndicator.tsx b/src/renderer/src/shell/media/CaptureIndicator.tsx new file mode 100644 index 00000000..126eca3f --- /dev/null +++ b/src/renderer/src/shell/media/CaptureIndicator.tsx @@ -0,0 +1,18 @@ +import { Mic, Video } from 'lucide-react' +import type { TabMedia } from '../../../../shared/state' +import './media.css' + +/** + * A tab using the camera or microphone shows a pulsing red indicator on its row (TB-15, MD-11). + * Media lane; drawn by the tab row next to the speaker icon. + */ +export function CaptureIndicator({ media }: { media: TabMedia | undefined }) { + if (!media?.mic && !media?.camera) return null + const label = + media.camera && media.mic ? 'Using camera and microphone' : media.camera ? 'Using camera' : 'Using microphone' + return ( + + {media.camera ? + ) +} diff --git a/src/renderer/src/shell/media/MediaTray.tsx b/src/renderer/src/shell/media/MediaTray.tsx new file mode 100644 index 00000000..4567803a --- /dev/null +++ b/src/renderer/src/shell/media/MediaTray.tsx @@ -0,0 +1,221 @@ +import { Mic, MicOff, Pause, PictureInPicture2, Play, SkipBack, SkipForward, Video, VideoOff, X } from 'lucide-react' +import { AnimatePresence, motion, useReducedMotion } from 'motion/react' +import { type CSSProperties, memo, type ReactNode, useEffect, useLayoutEffect, useRef, useState } from 'react' +import { inCall, mediaSubtitle, mediaTitle, trayTabIds } from '../../../../shared/media' +import type { TabId } from '../../../../shared/state' +import { invoke } from '../../api' +import { useOrbit, useOrbitShallow } from '../../store' +import { Favicon } from '../../ui/Favicon' +import { QUICK_EASE, SNAPPY } from '../sidebar/motion-tokens' +import './media.css' + +/** + * The sidebar media tray (MD-09, MD-10, MD-14; sidebar.md SB-10): one controller per tab playing + * in the background, newest on top, just above the bottom bar. It slides up when a playing tab is + * left and down when it goes (FL-32, MD-16); with reduced motion it only fades. + */ +export const MediaTray = memo(function MediaTray() { + const ids = useOrbitShallow(trayTabIds) + const reduce = useReducedMotion() + const motionProps = reduce + ? { + initial: { opacity: 0 }, + animate: { opacity: 1 }, + exit: { opacity: 0 }, + transition: { duration: 0.12, ease: QUICK_EASE }, + } + : { + initial: { opacity: 0, height: 0, y: 16 }, + animate: { opacity: 1, height: 'auto', y: 0 }, + exit: { opacity: 0, height: 0, y: 16 }, + transition: SNAPPY, + } + return ( +
+ + {ids.map((id) => ( + + + + ))} + +
+ ) +}) + +/** One background tab's controls: source (click to go there), play/pause, next, PiP, close. */ +const MediaController = memo(function MediaController({ tabId }: { tabId: TabId }) { + const tab = useOrbit((s) => s.tabs[tabId]) + const media = tab?.media + // Controls answer at once: the icon flips on click, before the page confirms (MD-16). + const [optimistic, setOptimistic] = useState(null) + const confirmed = media?.playing + useEffect(() => { + if (confirmed !== undefined) setOptimistic(null) + }, [confirmed]) + if (!tab || !media) return null + const playing = optimistic ?? media.playing + const call = inCall(tab) + const toggle = () => { + setOptimistic(!playing) + void invoke('media.control', { tabId, action: playing ? 'pause' : 'play' }) + } + return ( +
{ + e.preventDefault() + void invoke('media.menu', { tabId }) + }} + > +