From 3c8ee296ccf2645e75d3b65504ca8537eb92c00c Mon Sep 17 00:00:00 2001 From: Mat4m0 Date: Sat, 29 Aug 2026 16:14:57 +0200 Subject: [PATCH 1/5] feat(dictation): add optional on-device voice input --- docs/process-model.md | 19 + docs/user-guide.md | 19 + forge.config.ts | 4 + packaging/entitlements.development.plist | 2 + packaging/entitlements.preview.plist | 2 + packaging/entitlements.release.plist | 2 + scripts/apple-signing.ts | 1 + scripts/build.mjs | 77 +++- scripts/verify-stable-beta-roundtrip.ts | 1 + src/companion-kernel/abi.rs | 7 +- src/companion-kernel/lib.rs | 4 +- src/companion-kernel/skill_slots.rs | 61 +++- .../certification/enhancement-build-model.ts | 9 + src/main/certification/enhancement-builds.ts | 15 +- src/main/certification/enhancement-policy.ts | 6 +- .../enhancement-skill-slot-geometry-proof.ts | 51 ++- .../enhancement-skill-transform.ts | 80 ++++- .../certification/enhancement-transform.ts | 48 ++- src/main/core/settings.ts | 1 + src/main/dictation.ts | 233 ++++++++++++ src/main/game-text-editing.ts | 20 +- src/main/ipc.ts | 12 + src/main/main.ts | 8 + src/main/native-host.ts | 25 ++ src/main/settings-actions.ts | 22 +- src/native/host/dictation-host.hpp | 9 + src/native/host/dictation-host.mm | 245 +++++++++++++ src/native/host/dictation-modern.swift | 339 ++++++++++++++++++ src/native/host/host.mm | 11 + src/preload/preload.body.cjs | 7 + .../certified-companion-core-installation.ts | 35 +- src/renderer/certified-companion-extension.ts | 3 + .../certified-companion-tools-installation.ts | 73 ++-- .../companion-interface-geometry-snapshot.ts | 164 +++++++++ src/renderer/companion-observer.ts | 8 +- src/renderer/companion-skill-snapshot.ts | 120 +------ .../companion-tools-observer-readers.ts | 1 - src/renderer/dictation.ts | 220 ++++++++++++ src/renderer/gw-native.d.ts | 6 + src/renderer/harness.css | 83 +++++ src/renderer/harness.ts | 6 + src/renderer/index.html | 27 ++ src/renderer/settings.ts | 46 +++ .../skill-cooldown-overlay-consumer.ts | 6 +- src/renderer/skill-key-overlay-consumer.ts | 2 +- src/renderer/skill-overlays-installation.ts | 8 +- .../skill-slot-geometry-installation.ts | 2 +- src/renderer/skill-slot-projection.ts | 52 ++- src/shared/companion-abi.ts | 2 +- src/shared/contracts.ts | 40 +++ src/shared/enhancement-contracts.ts | 6 +- tests/electron/input-clipboard.spec.ts | 48 +++ tests/electron/sandbox.spec.ts | 1 + tests/electron/settings-tools-updates.spec.ts | 82 ++++- tests/fixtures/enhancements.ts | 25 +- .../skill-slot-geometry-kernel.test.ts | 56 ++- .../policy/distribution-entitlements.test.ts | 3 +- tests/policy/source-native-keychain.test.ts | 42 ++- tests/policy/source-release-pipeline.test.ts | 4 +- tests/release/preload-behaviour.test.ts | 9 + ...e-agrees-with-the-shared-contracts.test.ts | 10 +- tests/unit/apple-signing.test.ts | 1 + .../client-recertification-evidence.test.ts | 11 + tests/unit/companion-kernel-loader.test.ts | 3 +- tests/unit/companion-observer-pollers.test.ts | 78 ++++ tests/unit/dictation.test.ts | 157 ++++++++ .../unit/enhancement-skill-transform.test.ts | 43 ++- tests/unit/local-client-verifier.test.ts | 8 + tests/unit/settings.test.ts | 12 + tests/unit/skill-cooldown-overlay.test.ts | 3 + tests/unit/skill-key-overlay.test.ts | 11 +- .../unit/skill-overlays-installation.test.ts | 9 + .../skill-slot-geometry-installation.test.ts | 5 +- tests/unit/skill-slot-projection.test.ts | 3 + 74 files changed, 2563 insertions(+), 311 deletions(-) create mode 100644 src/main/dictation.ts create mode 100644 src/native/host/dictation-host.hpp create mode 100644 src/native/host/dictation-host.mm create mode 100644 src/native/host/dictation-modern.swift create mode 100644 src/renderer/companion-interface-geometry-snapshot.ts create mode 100644 src/renderer/dictation.ts create mode 100644 tests/unit/dictation.test.ts diff --git a/docs/process-model.md b/docs/process-model.md index d7b6cb4b..90f92bf4 100644 --- a/docs/process-model.md +++ b/docs/process-model.md @@ -217,6 +217,25 @@ sends Guild Wars Control-A, so a following Cut can export the same selection. Password text never crosses the renderer bridge. Physical Control stays available to Guild Wars unchanged. +Optional dictation stays outside Chromium's renderer media permissions. Main +owns recognition and commits a final transcript to the still-focused Guild +Wars editor with complete trusted key-down, character, and key-up sequences. +The mic is eligible only while ArenaNet owns that exact text proxy. Dictation +never reads or changes the pasteboard, never simulates Paste, and never sends +Return or another control character. The renderer owns eligibility, placement, +and visible state. Main +requests microphone access only after the player selects the visible mic and +owns one native Apple Speech session across all game windows. On macOS 26 or +newer, an explicit Settings setup uses AssetInventory to install the selected +language's system-managed model before enabling the feature. Older macOS does +not load a fallback recognizer. SpeechAnalyzer sends bounded partial/final +transcripts only to the owning renderer; it never logs them. The renderer shows +the control only for the ordinary text and multiline Guild Wars proxies, and +cancels when that proxy closes or changes. Password, email, and number proxies are never +eligible. A certified `EditMessage` frame rectangle anchors the mic to the +movable Guild Wars chat input; missing or uncertified geometry hides it. +Insertion edits the draft only: no layer presses Return or sends chat. + Before the first window exists, Electron writes the bundle-specific persistent `ApplePressAndHoldEnabled = false` preference. This makes macOS send physical repeat keydowns to hidden text proxies for printable characters, Backspace, diff --git a/docs/user-guide.md b/docs/user-guide.md index 2da9b013..355ae4ef 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -109,6 +109,25 @@ Main letters, the number row, and ANSI punctuation keep the same physical game binding when the macOS input source changes. Text fields still use the active input source. An old custom binding can need one manual rebind. +Choose **Settings → Tools → In-game dictation → Set up** on macOS 26 or +newer. GWonMac asks Apple to prepare the on-device model for your current +language, then asks you to restart once so it can attach dictation to the +certified Guild Wars chat frame. This is optional; when it is off, GWonMac does +not download a model or request microphone access. + +After setup, a small microphone appears beside the movable chat input only when +Guild Wars opens a normal text field. Select the microphone, speak, then select +it again to finish. The recognized +words are added to the current draft. Review them and press Return yourself. +GWonMac never sends chat automatically. + +The first recording asks for macOS Microphone access. Apple manages and may +share the downloaded language model with other apps. Audio and transcripts are +not sent to GWonMac services. The mic does not appear for email, password, or number fields, +and GWonMac does not save or include transcripts in diagnostics. If access was +denied, allow **Guild Wars Reforged** under **System Settings → Privacy & +Security → Microphone**. + The official client includes browser gamepad support. This project does not have a dedicated controller release test. diff --git a/forge.config.ts b/forge.config.ts index 8c24452a..bbeb8f40 100644 --- a/forge.config.ts +++ b/forge.config.ts @@ -85,6 +85,10 @@ const config: ForgeConfig = { ...(releaseNotarization ? { osxNotarize: releaseNotarization } : {}), extendInfo: { NSAppTransportSecurity: { NSAllowsArbitraryLoads: false }, + NSMicrophoneUsageDescription: + "Guild Wars Reforged uses the microphone only when you start in-game dictation.", + NSSpeechRecognitionUsageDescription: + "Guild Wars Reforged converts speech to a local Guild Wars text draft.", }, // Forge's own packaged output is out/; compiled JS lives in build/. // The explicit filter below is the package boundary. Forge's npm pruner diff --git a/packaging/entitlements.development.plist b/packaging/entitlements.development.plist index 9ba0181a..07443495 100644 --- a/packaging/entitlements.development.plist +++ b/packaging/entitlements.development.plist @@ -4,6 +4,8 @@ com.apple.security.cs.allow-jit + com.apple.security.device.audio-input + com.apple.application-identifier 9NN976MFZ4.io.github.mat4m0.gwonmac.dev com.apple.developer.team-identifier diff --git a/packaging/entitlements.preview.plist b/packaging/entitlements.preview.plist index 6cbfcffd..4554cbd8 100644 --- a/packaging/entitlements.preview.plist +++ b/packaging/entitlements.preview.plist @@ -4,6 +4,8 @@ com.apple.security.cs.allow-jit + com.apple.security.device.audio-input + com.apple.application-identifier 9NN976MFZ4.io.github.mat4m0.gwonmac.preview com.apple.developer.team-identifier diff --git a/packaging/entitlements.release.plist b/packaging/entitlements.release.plist index 804e9e72..283d8a6a 100644 --- a/packaging/entitlements.release.plist +++ b/packaging/entitlements.release.plist @@ -4,6 +4,8 @@ com.apple.security.cs.allow-jit + com.apple.security.device.audio-input + com.apple.application-identifier 9NN976MFZ4.io.github.mat4m0.gwonmac com.apple.developer.team-identifier diff --git a/scripts/apple-signing.ts b/scripts/apple-signing.ts index 978948f7..acd8a957 100644 --- a/scripts/apple-signing.ts +++ b/scripts/apple-signing.ts @@ -124,6 +124,7 @@ export function approvedDistributionEntitlements( ): Readonly> { return { "com.apple.security.cs.allow-jit": true, + "com.apple.security.device.audio-input": true, "com.apple.application-identifier": applicationIdentifier(channel), "com.apple.developer.team-identifier": APPLE_TEAM_ID, }; diff --git a/scripts/build.mjs b/scripts/build.mjs index 8c47a22a..c6badb1f 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -96,19 +96,33 @@ export const BUILD_STEPS = [ // directory, so it cannot disturb the emits above and its position here is // for readability rather than correctness. ["pnpm", ["--filter", "@gwonmac/tools-ui", "build:embedded"]], - // The only native addon. It uses raw Node-API version 8, whose ABI remains - // stable across the Node and Electron upgrades this project takes. The - // framework APIs resolve at runtime from the Electron host, so the bundle - // deliberately leaves Node-API symbols undefined here. + // The only native addon. Swift owns Apple's modern, on-device transcription + // API; Objective-C++ owns the Node-API boundary. They meet at a + // four-function C ABI, then swiftc links both objects while deliberately + // leaving Node-API symbols for the Electron host to resolve at runtime. + [ + "xcrun", + [ + "swiftc", + "-module-cache-path", + "build/.swift-module-cache", + "-parse-as-library", + "-target", + `${nativeArchitecture}-apple-macosx12.0`, + "-O", + "-c", + "src/native/host/dictation-modern.swift", + "-o", + "build/native/dictation-modern.o", + ], + ], [ "xcrun", [ "clang++", "-std=c++20", "-fobjc-arc", - "-bundle", - "-undefined", - "dynamic_lookup", + "-c", "-DNAPI_VERSION=8", "-I", "node_modules/node-api-headers/include", @@ -121,6 +135,51 @@ export const BUILD_STEPS = [ "-Werror", "-fvisibility=hidden", "src/native/host/host.mm", + "-o", + "build/native/host.o", + ], + ], + [ + "xcrun", + [ + "clang++", + "-std=c++20", + "-fobjc-arc", + "-c", + "-DNAPI_VERSION=8", + "-I", + "node_modules/node-api-headers/include", + "-mmacosx-version-min=12.0", + "-arch", + nativeArchitecture, + "-O2", + "-Wall", + "-Wextra", + "-Werror", + "-fvisibility=hidden", + "src/native/host/dictation-host.mm", + "-o", + "build/native/dictation-host.o", + ], + ], + [ + "xcrun", + [ + "swiftc", + "-module-cache-path", + "build/.swift-module-cache", + "-target", + `${nativeArchitecture}-apple-macosx12.0`, + "-emit-library", + "build/native/host.o", + "build/native/dictation-host.o", + "build/native/dictation-modern.o", + "-Xlinker", + "-undefined", + "-Xlinker", + "dynamic_lookup", + "-Xlinker", + "-lc++", "-framework", "AppKit", "-framework", @@ -129,6 +188,10 @@ export const BUILD_STEPS = [ "LocalAuthentication", "-framework", "Security", + "-framework", + "AVFoundation", + "-framework", + "Speech", "-o", "build/native/host.node", ], diff --git a/scripts/verify-stable-beta-roundtrip.ts b/scripts/verify-stable-beta-roundtrip.ts index c7ffa43a..7478b1b0 100644 --- a/scripts/verify-stable-beta-roundtrip.ts +++ b/scripts/verify-stable-beta-roundtrip.ts @@ -256,6 +256,7 @@ const candidateSettingsDomains = Array.from( uiCustomTheme: cycle(customThemeValues, index), uiFont: cycle(UI_FONTS, index), controllerPromptStyle: cycle(CONTROLLER_PROMPT_STYLES, index), + dictationEnabled: cycle(booleanValues, index), uiPanelOpacity: cycle(opacityValues, index), cartographyOverlayEnabled: cycle(booleanValues, index), cartographyGridEnabled: cycle(booleanValues, index + 1), diff --git a/src/companion-kernel/abi.rs b/src/companion-kernel/abi.rs index 3f887ab7..0726bd21 100644 --- a/src/companion-kernel/abi.rs +++ b/src/companion-kernel/abi.rs @@ -51,7 +51,7 @@ pub(crate) const FLAG_PLAY_REGION_UNLOCKS: u32 = 1 << 3; pub(crate) const SKILL_SLOT_BYTES: u32 = size_of::() as u32; pub(crate) const SKILL_SLOT_MAGIC: u32 = 0x534b_5747; -pub(crate) const SKILL_SLOT_ABI_AND_SIZE: u32 = (SKILL_SLOT_BYTES << 16) | 2; +pub(crate) const SKILL_SLOT_ABI_AND_SIZE: u32 = (SKILL_SLOT_BYTES << 16) | 3; pub(crate) const FLAG_SKILL_SLOTS_READY: u32 = 1 << 0; pub(crate) const SKILL_COOLDOWN_BYTES: u32 = size_of::() as u32; @@ -482,6 +482,9 @@ pub(crate) struct SkillSlotSnapshot { pub(crate) viewport_width: f32, pub(crate) viewport_height: f32, pub(crate) slots: [SkillSlotRect; SKILL_SLOTS], + pub(crate) chat_frame_id: u32, + pub(crate) chat_outcome: u32, + pub(crate) chat_input: SkillSlotRect, } #[repr(C)] @@ -568,5 +571,5 @@ const _: [(); 148] = [(); size_of::()]; const _: [(); 4160] = [(); size_of::()]; const _: [(); 64] = [(); size_of::()]; const _: [(); 16] = [(); size_of::()]; -const _: [(); 164] = [(); size_of::()]; +const _: [(); 188] = [(); size_of::()]; const _: [(); 60] = [(); size_of::()]; diff --git a/src/companion-kernel/lib.rs b/src/companion-kernel/lib.rs index 23bb8700..2c9df534 100644 --- a/src/companion-kernel/lib.rs +++ b/src/companion-kernel/lib.rs @@ -809,7 +809,7 @@ pub unsafe extern "C" fn companion_init( } #[no_mangle] -pub unsafe extern "C" fn companion_dispatch(kind: u32, a: u32, b: u32, c: u32, _d: u32, _e: u32) { +pub unsafe extern "C" fn companion_dispatch(kind: u32, a: u32, b: u32, c: u32, d: u32, _e: u32) { match kind { DISPATCH_TICK => { if !unsafe { INITIALIZED } { @@ -835,7 +835,7 @@ pub unsafe extern "C" fn companion_dispatch(kind: u32, a: u32, b: u32, c: u32, _ unsafe { cursor::tick(layout) }; } if active & FEATURE_SKILL_SLOT_GEOMETRY != 0 { - unsafe { skill_slots::tick(layout, b) }; + unsafe { skill_slots::tick(layout, b, d) }; } if active & FEATURE_SKILL_COOLDOWN_OBSERVATION != 0 { unsafe { skill_cooldowns::tick(layout, c) }; diff --git a/src/companion-kernel/skill_slots.rs b/src/companion-kernel/skill_slots.rs index 002eb99b..96f8485a 100644 --- a/src/companion-kernel/skill_slots.rs +++ b/src/companion-kernel/skill_slots.rs @@ -18,6 +18,7 @@ const MAX_FRAMES: u32 = 16_384; const CACHE_AUDIT_TICKS: u32 = 30; const CREATED: u32 = 0x4; const HIDDEN: u32 = 0x200; +const FLAG_CHAT_INPUT_READY: u32 = 1 << 1; #[derive(Clone, Copy)] #[repr(u32)] enum Outcome { @@ -279,10 +280,32 @@ unsafe fn collect(layout: Layout, skill_bar_id: u32) -> Result) { +unsafe fn collect_chat( + layout: Layout, + chat_frame_id: u32, +) -> Result<(SkillSlotRect, f32, f32), Refusal> { + if chat_frame_id == 0 || !valid_layout(layout) { + return Err(refuse(Outcome::InvalidInput)); + } + let (array, count) = unsafe { frame_table(layout) } + .ok_or_else(|| refuse(Outcome::FrameTable))?; + let frame = unsafe { frame_at(layout, array, count, chat_frame_id) } + .ok_or_else(|| refuse(Outcome::ParentMissing))?; + if !unsafe { visible(layout, frame) } { + return Err(refuse(Outcome::ParentHidden)); + } + unsafe { rect(layout, frame) }.map_err(refuse) +} + +unsafe fn publish( + frame_id: u32, + observed: Result, + chat_frame_id: u32, + chat_observed: Result<(SkillSlotRect, f32, f32), Refusal>, +) { let next = unsafe { SEQUENCE }.wrapping_add(2) & !1; let snapshot = unsafe { POINTER as *mut SkillSlotSnapshot }; - let (flags, outcome, candidate_count, viewport_width, viewport_height, slots) = match observed { + let (mut flags, outcome, candidate_count, viewport_width, viewport_height, slots) = match observed { Ok((width, height, slots)) => (FLAG_SKILL_SLOTS_READY, 0, 0, width, height, slots), Err(refusal) => ( 0, @@ -293,17 +316,35 @@ unsafe fn publish(frame_id: u32, observed: Result) { [EMPTY_RECT; 8], ), }; + // Chat frames can carry a local clipping viewport while their screen + // bounds remain in the global interface coordinate space. The certified + // skill viewport is the projection source; requiring the local sizes to + // match discarded the real movable chat editor. + let (published_chat_id, chat_outcome, chat_input) = match chat_observed { + Ok((bounds, _, _)) if flags & FLAG_SKILL_SLOTS_READY != 0 => { + flags |= FLAG_CHAT_INPUT_READY; + (chat_frame_id, 0, bounds) + } + Ok(_) => (0, Outcome::ViewportMismatch as u32, EMPTY_RECT), + Err(refusal) => (0, refusal.outcome as u32, EMPTY_RECT), + }; unsafe { write_volatile(&mut (*snapshot).sequence, next.wrapping_sub(1)); write_volatile(&mut (*snapshot).magic, SKILL_SLOT_MAGIC); write_volatile(&mut (*snapshot).abi_and_size, SKILL_SLOT_ABI_AND_SIZE); write_volatile(&mut (*snapshot).flags, flags); - write_volatile(&mut (*snapshot).frame_id, if flags == 0 { 0 } else { frame_id }); + write_volatile( + &mut (*snapshot).frame_id, + if flags & FLAG_SKILL_SLOTS_READY == 0 { 0 } else { frame_id }, + ); write_volatile(&mut (*snapshot).outcome, outcome); write_volatile(&mut (*snapshot).candidate_count, candidate_count); write_volatile(&mut (*snapshot).viewport_width, viewport_width); write_volatile(&mut (*snapshot).viewport_height, viewport_height); write_volatile(&mut (*snapshot).slots, slots); + write_volatile(&mut (*snapshot).chat_frame_id, published_chat_id); + write_volatile(&mut (*snapshot).chat_outcome, chat_outcome); + write_volatile(&mut (*snapshot).chat_input, chat_input); write_volatile(&mut (*snapshot).sequence, next); SEQUENCE = next; } @@ -321,10 +362,18 @@ pub(crate) unsafe fn initialize(pointer: u32) { for index in 0..SKILL_SLOT_BYTES / 4 { unsafe { write_volatile((pointer + index * 4) as *mut u32, 0) }; } - unsafe { publish(0, Err(refuse(Outcome::Inactive))) }; + unsafe { + publish( + 0, + Err(refuse(Outcome::Inactive)), + 0, + Err(refuse(Outcome::Inactive)), + ) + }; } -pub(crate) unsafe fn tick(layout: Layout, skill_bar_id: u32) { +pub(crate) unsafe fn tick(layout: Layout, skill_bar_id: u32, chat_frame_id: u32) { let observed = unsafe { collect(layout, skill_bar_id) }; - unsafe { publish(skill_bar_id, observed) }; + let chat_observed = unsafe { collect_chat(layout, chat_frame_id) }; + unsafe { publish(skill_bar_id, observed, chat_frame_id, chat_observed) }; } diff --git a/src/main/certification/enhancement-build-model.ts b/src/main/certification/enhancement-build-model.ts index f1d8ea6f..3a9bf382 100644 --- a/src/main/certification/enhancement-build-model.ts +++ b/src/main/certification/enhancement-build-model.ts @@ -375,6 +375,15 @@ export interface KnownEnhancementBuild { bodySha256: string; }>; labelAddress: number; + /** The movable Guild Wars chat editor captured through the same frame constructor. */ + chatInitializer: Readonly<{ + functionIndex: number; + params: readonly ["i32", "i32"]; + results: readonly []; + bodySha256: string; + constructorCallOperand: number; + }>; + chatLabelAddress: number; layout: EnhancementSkillSlotGeometryLayout; }>; /** Exact label-hash authority and frame layout used by reload automation. */ diff --git a/src/main/certification/enhancement-builds.ts b/src/main/certification/enhancement-builds.ts index f5887a4d..0e2da225 100644 --- a/src/main/certification/enhancement-builds.ts +++ b/src/main/certification/enhancement-builds.ts @@ -36,9 +36,11 @@ export const ENHANCEMENT_BUILDS: readonly KnownEnhancementBuild[] = // ENHANCEMENT_TRANSFORM_ABI or any config word changes. outputSha256: Object.freeze({ "features-601": - "cfdc58bd7a024fe6cdc4ae631e6edfcb23cb54a8885d3f264243c0fcc2ba64cf", + "4da16e7d0814f5957acd3b6b7f0015b498012d1b94c41fd1867ca969d3dd8774", + "features-681": + "17d30b20bee572b992516c5c9b1604796264c3dc6758ab68f9c6eb8ec5de5ec1", "features-7ff": - "74b75630edff0cb9e9b68cc719a43d496b72c1b1aa9720afb40d7c1ed9aac0f5", + "2dbfd05a25a80d2fd312f0df0a0cf7d7edda1975ecd168bbf0cfcdf8ffab3256", }), programId: 1, // The verifier derives this bounded identity from the exact module; it is @@ -492,6 +494,15 @@ export const ENHANCEMENT_BUILDS: readonly KnownEnhancementBuild[] = "a29fca1d30e5fa7dea1ca30f6453acbb8a099e4423c1f05ee43b01cfc3045c41", }), labelAddress: 0x186e1a, + chatInitializer: Object.freeze({ + functionIndex: 13736, + params: Object.freeze(["i32", "i32"] as const), + results: Object.freeze([] as const), + bodySha256: + "5097f3592ca8c5978cf6586ea78cb9f4852857143280138f98e801ffe365573e", + constructorCallOperand: 139, + }), + chatLabelAddress: 0x1813dc, layout: Object.freeze({ frameArray: 0x5a1fdc, frameCount: 0x5a1fe4, diff --git a/src/main/certification/enhancement-policy.ts b/src/main/certification/enhancement-policy.ts index e3a3451f..cb41e91d 100644 --- a/src/main/certification/enhancement-policy.ts +++ b/src/main/certification/enhancement-policy.ts @@ -57,6 +57,10 @@ export function requestedEnhancementCapabilities( if (program !== "none") { return enhancementCapabilitiesFor(enhancementSelectionFor(settings), program); } - if (!settings.gwonmacTools) return ENHANCEMENT_CAPABILITY_PRESETS.core; + if (!settings.gwonmacTools) { + return settings.dictationEnabled + ? ENHANCEMENT_CAPABILITY_PRESETS.dictation + : ENHANCEMENT_CAPABILITY_PRESETS.core; + } return ENHANCEMENT_CAPABILITY_PRESETS.all; } diff --git a/src/main/certification/enhancement-skill-slot-geometry-proof.ts b/src/main/certification/enhancement-skill-slot-geometry-proof.ts index 81ff50a8..4deb39d4 100644 --- a/src/main/certification/enhancement-skill-slot-geometry-proof.ts +++ b/src/main/certification/enhancement-skill-slot-geometry-proof.ts @@ -119,10 +119,11 @@ export function isSkillSlotGeometryProof( if (!value || typeof value !== "object") return false; const proof = value as Partial; const initializer = proof.initializer; + const chatInitializer = proof.chatInitializer; const constructor = proof.constructor; const layout = proof.layout; return Object.keys(proof).sort().join() === - "constructor,initializer,labelAddress,layout" + "chatInitializer,chatLabelAddress,constructor,initializer,labelAddress,layout" && initializer !== undefined && Object.keys(initializer).sort().join() === "bodySha256,constructorCallOperand,functionIndex,params,results" @@ -133,6 +134,16 @@ export function isSkillSlotGeometryProof( && isDeepStrictEqual(initializer.params, ["i32", "i32"]) && isDeepStrictEqual(initializer.results, []) && /^[0-9a-f]{64}$/.test(initializer.bodySha256 ?? "") + && chatInitializer !== undefined + && Object.keys(chatInitializer).sort().join() === + "bodySha256,constructorCallOperand,functionIndex,params,results" + && Number.isSafeInteger(chatInitializer.functionIndex) + && chatInitializer.functionIndex >= 0 + && Number.isSafeInteger(chatInitializer.constructorCallOperand) + && chatInitializer.constructorCallOperand >= 0 + && isDeepStrictEqual(chatInitializer.params, ["i32", "i32"]) + && isDeepStrictEqual(chatInitializer.results, []) + && /^[0-9a-f]{64}$/.test(chatInitializer.bodySha256 ?? "") && constructor !== undefined && Object.keys(constructor).sort().join() === "bodySha256,functionIndex,params,results" @@ -147,6 +158,9 @@ export function isSkillSlotGeometryProof( && typeof proof.labelAddress === "number" && Number.isSafeInteger(proof.labelAddress) && proof.labelAddress > 0 + && typeof proof.chatLabelAddress === "number" + && Number.isSafeInteger(proof.chatLabelAddress) + && proof.chatLabelAddress > 0 && layout !== undefined && Number.isSafeInteger(layout.frameArray) && Number.isSafeInteger(layout.frameCount) @@ -202,6 +216,33 @@ export function deriveSkillSlotGeometry( ) ) return null; + // The chat editor is the `EditMessage` child in the unique GmChat frame + // definition. Its generic frame constructor is the same one already proven + // above, so a changed label block, call site, function, or signature refuses + // the entire all-or-nothing geometry capability. + const chatBlock = utf16Le("GmChat\0BtnStateSwitch\0EditMessage"); + const chatBlockAddress = uniqueStaticAddress(context, chatBlock); + if (chatBlockAddress === null) return null; + const chatLabelAddress = chatBlockAddress + + "GmChat\0BtnStateSwitch\0".length * 2; + if (codeOperandOccurrences(module, chatLabelAddress) !== 1) return null; + const encodedChatLabel = paddedOperand(chatLabelAddress); + const chatCandidates = module.bodies.flatMap((body, localIndex) => { + const operand = indexOfBytes(body, encodedChatLabel); + return operand < 0 + ? [] + : [{ body, operand, functionIndex: module.functionImportCount + localIndex }]; + }); + if (chatCandidates.length !== 1) return null; + const chatInitializer = chatCandidates[0]!; + const chatCallOperand = chatInitializer.operand + encodedChatLabel.byteLength + 1; + if ( + chatInitializer.body[chatInitializer.operand - 1] !== 0x41 + || chatInitializer.body[chatCallOperand - 1] !== 0x10 + || unsignedOperand(chatInitializer.body, chatCallOperand) !== constructorFunction + || !signatureMatches(module, chatInitializer.functionIndex, ["i32", "i32"], []) + ) return null; + const allocator = uniqueRoleFunction(module, SKILL_SLOT_ALLOCATOR_ROLE); const registrar = uniqueRoleFunction(module, SKILL_SLOT_REGISTRAR_ROLE); const resolver = uniqueRoleFunction(module, SKILL_SLOT_ID_RESOLVER_ROLE); @@ -259,6 +300,14 @@ export function deriveSkillSlotGeometry( bodySha256: functionBodySha256(module, constructorFunction), }), labelAddress, + chatInitializer: Object.freeze({ + functionIndex: chatInitializer.functionIndex, + params: Object.freeze(["i32", "i32"] as const), + results: Object.freeze([] as const), + bodySha256: functionBodySha256(module, chatInitializer.functionIndex), + constructorCallOperand: chatCallOperand, + }), + chatLabelAddress, layout, }); } diff --git a/src/main/certification/enhancement-skill-transform.ts b/src/main/certification/enhancement-skill-transform.ts index 3ad85a17..8094b434 100644 --- a/src/main/certification/enhancement-skill-transform.ts +++ b/src/main/certification/enhancement-skill-transform.ts @@ -36,6 +36,7 @@ export type EnhancementSkillTransformResolution = Readonly<{ geometry: Readonly<{ certificate: SkillSlotGeometryCertificate; initializer: ResolvedSkillFunction; + chatInitializer: ResolvedSkillFunction; constructor: ResolvedSkillFunction; }> | null; cooldown: Readonly<{ @@ -89,20 +90,43 @@ export function resolveEnhancementSkillTransform(options: Readonly<{ certificate.constructor.params, certificate.constructor.results, ); + const chatInitializer = resolveFunction( + "chat editor initializer", + certificate.chatInitializer.functionIndex, + certificate.chatInitializer.params, + certificate.chatInitializer.results, + ); if ( bodyHash(bodies, importCount, certificate.initializer.functionIndex, fail) !== certificate.initializer.bodySha256 || bodyHash(bodies, importCount, certificate.constructor.functionIndex, fail) !== certificate.constructor.bodySha256 + || bodyHash(bodies, importCount, certificate.chatInitializer.functionIndex, fail) + !== certificate.chatInitializer.bodySha256 ) fail("SkillBar frame capture bodies do not match their certificates"); - const operand = certificate.initializer.constructorCallOperand; - const body = bodies[initializer.localIndex]!; const expected = paddedIndex(certificate.constructor.functionIndex); - if ( - body[operand - 1] !== 0x10 - || expected.some((byte, index) => body[operand + index] !== byte) - ) fail("SkillBar constructor call site does not match its certificate"); - geometry = { certificate, initializer, constructor }; + const certifiedCall = ( + resolved: ResolvedSkillFunction, + operand: number, + label: string, + ) => { + const body = bodies[resolved.localIndex]!; + if ( + body[operand - 1] !== 0x10 + || expected.some((byte, index) => body[operand + index] !== byte) + ) fail(`${label} constructor call site does not match its certificate`); + }; + certifiedCall( + initializer, + certificate.initializer.constructorCallOperand, + "SkillBar", + ); + certifiedCall( + chatInitializer, + certificate.chatInitializer.constructorCallOperand, + "chat editor", + ); + geometry = { certificate, initializer, chatInitializer, constructor }; } let cooldown: EnhancementSkillTransformResolution["cooldown"] = null; @@ -141,16 +165,18 @@ export function resolveEnhancementSkillTransform(options: Readonly<{ } /** Append the capture wrapper and replace only the certified call operand. */ -export function rewriteSkillBarConstructorCapture(options: Readonly<{ +export function rewriteInterfaceFrameCaptures(options: Readonly<{ resolution: EnhancementSkillTransformResolution; nextBodies: Uint8Array[]; skillBarFrameGlobalIndex: number; + chatInputFrameGlobalIndex: number; appendFunction: (typeIndex: number, body: Uint8Array) => number; }>): void { const { resolution, nextBodies, skillBarFrameGlobalIndex, + chatInputFrameGlobalIndex, appendFunction, } = options; const geometry = resolution.geometry; @@ -158,29 +184,47 @@ export function rewriteSkillBarConstructorCapture(options: Readonly<{ const { certificate: skillSlotGeometry, initializer: skillInitializer, + chatInitializer, constructor: skillConstructor, } = geometry; - const wrapperIndex = appendFunction( + const captureWrapper = (globalIndex: number) => appendFunction( skillConstructor.typeIndex, concat( - // One i32 local holds the constructor result while it is published. + // Zero is constructor refusal, not a frame. Retain the last successful + // frame so a later optional UI initialization cannot erase it; the + // kernel still validates that retained id against the live frame table. uleb(1), uleb(1), Uint8Array.of(0x7f), ...Array.from({ length: 6 }, (_, index) => concat(Uint8Array.of(0x20), uleb(index))), Uint8Array.of(0x10), uleb(skillSlotGeometry.constructor.functionIndex), - Uint8Array.of(0x22), uleb(6), - Uint8Array.of(0x24), uleb(skillBarFrameGlobalIndex), + Uint8Array.of(0x21), uleb(6), + Uint8Array.of(0x20), uleb(6), + Uint8Array.of(0x04, 0x40), + Uint8Array.of(0x20), uleb(6), + Uint8Array.of(0x24), uleb(globalIndex), + Uint8Array.of(0x0b), Uint8Array.of(0x20), uleb(6), Uint8Array.of(0x0b), ), ); - const rewrittenInitializer = new Uint8Array( - nextBodies[skillInitializer.localIndex]!, - ); - rewrittenInitializer.set( - paddedIndex(wrapperIndex), + const rewriteCapture = ( + resolved: ResolvedSkillFunction, + operand: number, + globalIndex: number, + ) => { + const rewritten = new Uint8Array(nextBodies[resolved.localIndex]!); + rewritten.set(paddedIndex(captureWrapper(globalIndex)), operand); + nextBodies[resolved.localIndex] = rewritten; + }; + rewriteCapture( + skillInitializer, skillSlotGeometry.initializer.constructorCallOperand, + skillBarFrameGlobalIndex, + ); + rewriteCapture( + chatInitializer, + skillSlotGeometry.chatInitializer.constructorCallOperand, + chatInputFrameGlobalIndex, ); - nextBodies[skillInitializer.localIndex] = rewrittenInitializer; } diff --git a/src/main/certification/enhancement-transform.ts b/src/main/certification/enhancement-transform.ts index 8e866837..110afa59 100644 --- a/src/main/certification/enhancement-transform.ts +++ b/src/main/certification/enhancement-transform.ts @@ -34,7 +34,7 @@ import { import { applyFeatureContributions } from "./enhancement-transform-features.js"; import { resolveEnhancementSkillTransform, - rewriteSkillBarConstructorCapture, + rewriteInterfaceFrameCaptures, } from "./enhancement-skill-transform.js"; import { preGameDiagnosticReader, @@ -104,20 +104,22 @@ function dispatcher( dispatchTypeIndex: number, originalIndex: number, hookGlobalIndex: number, - extraArgumentGlobal: number | null = null, - extraArgumentFunction: number | null = null, + extraArguments: readonly (Readonly<{ + source: "global" | "function"; + index: number; + }> | null)[] = [], ): Uint8Array { const args = Array.from({ length: paramCount }, (_, index) => concat(Uint8Array.of(0x20), uleb(index)), ); const dispatchArgs = [ ...args, - ...(extraArgumentGlobal === null - ? [] - : [concat(Uint8Array.of(0x23), uleb(extraArgumentGlobal))]), - ...(extraArgumentFunction === null - ? [] - : [concat(Uint8Array.of(0x10), uleb(extraArgumentFunction))]), + ...extraArguments.map((argument) => argument === null + ? Uint8Array.of(0x41, 0x00) + : concat( + Uint8Array.of(argument.source === "global" ? 0x23 : 0x10), + uleb(argument.index), + )), ]; return concat( uleb(0), @@ -779,6 +781,9 @@ function assembleEnhancementTransform( const skillBarFrameGlobalIndex = capabilities.skillSlotGeometry ? allocateGlobals(1) : 0; + const chatInputFrameGlobalIndex = capabilities.skillSlotGeometry + ? allocateGlobals(1) + : 0; const commandPendingGlobalIndex = hasActionQueue ? allocateGlobals(1) : 0; const commandArgumentGlobalBase = hasActionQueue ? allocateGlobals(COMMAND_ARGS) @@ -858,27 +863,34 @@ function assembleEnhancementTransform( ? selectedOriginalIndices[uiHookIndex]! : null; selected.forEach((hook, index) => { + const tickArguments = hook.dispatchKind === COMPANION_DISPATCH_KINDS.tick + ? [ + capabilities.skillSlotGeometry + ? { source: "global" as const, index: skillBarFrameGlobalIndex } + : null, + capabilities.skillCooldownObservation + ? { source: "function" as const, index: skillTimerFunctionIndex! } + : null, + capabilities.skillSlotGeometry + ? { source: "global" as const, index: chatInputFrameGlobalIndex } + : null, + ] + : []; nextBodies[hook.localIndex] = dispatcher( hook.type.params.length, hook.dispatchKind, dispatchTypeIndex, selectedOriginalIndices[index]!, hookGlobalIndex, - capabilities.skillSlotGeometry - && hook.dispatchKind === COMPANION_DISPATCH_KINDS.tick - ? skillBarFrameGlobalIndex - : null, - capabilities.skillCooldownObservation - && hook.dispatchKind === COMPANION_DISPATCH_KINDS.tick - ? skillTimerFunctionIndex - : null, + tickArguments, ); }); - rewriteSkillBarConstructorCapture({ + rewriteInterfaceFrameCaptures({ resolution: skillResolution, nextBodies, skillBarFrameGlobalIndex, + chatInputFrameGlobalIndex, appendFunction, }); diff --git a/src/main/core/settings.ts b/src/main/core/settings.ts index e9de8423..dd530038 100644 --- a/src/main/core/settings.ts +++ b/src/main/core/settings.ts @@ -242,6 +242,7 @@ export function parseSettings(raw: unknown): AppSettings { out.buildLibrary = asBool(src.teamManagement, "teamManagement"); } for (const setting of [ + "dictationEnabled", "gwonmacTools", "cartographyOverlayEnabled", "cartographyGridEnabled", diff --git a/src/main/dictation.ts b/src/main/dictation.ts new file mode 100644 index 00000000..08f122e5 --- /dev/null +++ b/src/main/dictation.ts @@ -0,0 +1,233 @@ +/** + * The one app-wide macOS dictation session. Main owns microphone permission, + * the native recognizer, the active game window, and trusted text insertion. + * Transcripts are forwarded only to that window and are never logged. + */ + +import type { BrowserWindow, WebContents } from 'electron'; +import { DICTATION_TEXT_CEILING, IPC, type DictationEvent } from '../shared/contracts.js'; +import type { NativeDictationEvent, NativeHost } from './native-host.js'; + +type ActiveDictation = Readonly<{ + generation: number; + window: BrowserWindow; + onClosed: () => void; +} & ( + | { kind: 'capture' } + | { kind: 'committing' } + | { kind: 'setup'; resolve(locale: string): void; reject(reason: Error): void } +)>; + +export function safeDictationText(transcript: string): string { + return Array.from(transcript, (character) => { + const codePoint = character.codePointAt(0) ?? 0; + return codePoint <= 0x1f + || (codePoint >= 0x7f && codePoint <= 0x9f) + || codePoint === 0x2028 + || codePoint === 0x2029 + ? ' ' + : character; + }).join(''); +} + +export function sendDictationText(contents: WebContents, transcript: string): void { + for (const keyCode of safeDictationText(transcript)) { + contents.sendInputEvent({ type: 'keyDown', keyCode }); + contents.sendInputEvent({ type: 'char', keyCode }); + contents.sendInputEvent({ type: 'keyUp', keyCode }); + } +} + +export class DictationController { + private active: ActiveDictation | null = null; + private generation = 0; + private readonly nativeHost: NativeHost; + private readonly requestMicrophone: () => Promise; + private readonly permissionHostReady: boolean; + + constructor( + nativeHost: NativeHost, + requestMicrophone: () => Promise, + permissionHostReady = true, + ) { + this.nativeHost = nativeHost; + this.requestMicrophone = requestMicrophone; + this.permissionHostReady = permissionHostReady; + } + + async start(window: BrowserWindow): Promise { + this.cancel(); + // Forge's unpackaged development host is the generic Electron.app. It has + // no speech privacy description, and macOS terminates it instead of + // returning a permission error. Packaged builds carry both declarations. + if (!this.permissionHostReady) { + this.send(window, { state: 'error', reason: 'unavailable' }); + return; + } + const generation = ++this.generation; + const onClosed = () => this.cancel(window); + this.active = { kind: 'capture', generation, window, onClosed }; + window.once('closed', onClosed); + this.send(window, { state: 'requesting' }); + + const microphoneAllowed = await this.requestMicrophone(); + if (!this.isActive(generation, window)) return; + if (!microphoneAllowed) { + this.send(window, { state: 'error', reason: 'permission-denied' }); + this.releaseActive(); + return; + } + + try { + this.nativeHost.startDictation((event) => { + this.receive(generation, window, event); + }); + } catch { + if (!this.isActive(generation, window)) return; + this.send(window, { state: 'error', reason: 'unavailable' }); + this.releaseActive(); + } + } + + prepare(window: BrowserWindow): Promise { + this.cancel(); + const generation = ++this.generation; + const onClosed = () => this.cancel(window); + return new Promise((resolve, reject) => { + this.active = { kind: 'setup', generation, window, onClosed, resolve, reject }; + window.once('closed', onClosed); + try { + this.nativeHost.prepareDictation((event) => { + this.receive(generation, window, event); + }); + } catch (cause) { + this.releaseActive(); + reject(new Error('On-device dictation setup is unavailable', { cause })); + } + }); + } + + finish(window: BrowserWindow): void { + if (this.active?.window !== window) return; + this.nativeHost.finishDictation(); + } + + cancel(window?: BrowserWindow): void { + if (!this.active || (window && this.active.window !== window)) return; + if (this.active.kind === 'setup') { + this.active.reject(new Error('On-device dictation setup was cancelled')); + } + this.generation += 1; + this.releaseActive(); + this.nativeHost.cancelDictation(); + } + + dispose(): void { + this.cancel(); + } + + private receive( + generation: number, + window: BrowserWindow, + event: NativeDictationEvent, + ): void { + if (!this.isActive(generation, window)) return; + if (this.active?.kind === 'committing') return; + if (event.type === 'ready') { + if (this.active?.kind !== 'setup') return; + this.active.resolve(event.locale); + this.releaseActive(); + return; + } + if (this.active?.kind === 'setup' && event.type !== 'error') { + if (event.type === 'preparing') return; + this.active.reject(new Error('On-device dictation setup returned an invalid event')); + this.releaseActive(); + return; + } + if (event.type === 'preparing') { + this.send(window, event.progress === undefined + ? { state: 'preparing' } + : { state: 'preparing', progress: event.progress }); + return; + } + if (event.type === 'listening') { + this.send(window, { state: 'listening', transcript: '' }); + return; + } + if (event.type === 'result') { + const transcript = event.transcript.slice(0, DICTATION_TEXT_CEILING); + if (event.final) { + if (this.active?.kind !== 'capture') return; + this.active = { ...this.active, kind: 'committing' }; + this.commitFinal(generation, window, transcript); + } else { + this.send(window, { state: 'listening', transcript }); + } + return; + } + const reason: Extract['reason'] = + event.reason === 'permission-denied' + ? 'permission-denied' + : event.reason === 'model-unavailable' + ? 'model-unavailable' + : event.reason === 'model-download-failed' + ? 'model-download-failed' + : event.reason === 'setup-required' + ? 'setup-required' + : event.reason === 'audio-unavailable' + ? 'audio-unavailable' + : event.reason === 'unavailable' + ? 'unavailable' + : 'recognition-failed'; + if (this.active?.kind === 'setup') { + this.active.reject(new Error(`On-device dictation setup failed: ${reason}`)); + } else { + this.send(window, { state: 'error', reason }); + } + this.releaseActive(); + } + + private commitFinal( + generation: number, + window: BrowserWindow, + transcript: string, + ): void { + try { + if (!window.isFocused()) throw new Error('owning window lost focus'); + sendDictationText(window.webContents, transcript); + if (!this.isActive(generation, window)) return; + this.send(window, { state: 'final', transcript }); + } catch { + if (!this.isActive(generation, window)) return; + this.send(window, { state: 'error', reason: 'insertion-failed' }); + } finally { + if (this.isActive(generation, window)) { + this.releaseActive(); + } + } + } + + private isActive(generation: number, window: BrowserWindow): boolean { + return this.active?.generation === generation + && this.active.window === window + && !window.isDestroyed(); + } + + private send(window: BrowserWindow, event: DictationEvent): void { + if (window.isDestroyed() || window.webContents.isDestroyed()) return; + try { + window.webContents.send(IPC.dictationEvent, event); + } catch { + // The renderer can disappear between the liveness check and delivery. + } + } + + private releaseActive(): void { + const active = this.active; + this.active = null; + if (active && !active.window.isDestroyed()) { + active.window.removeListener('closed', active.onClosed); + } + } +} diff --git a/src/main/game-text-editing.ts b/src/main/game-text-editing.ts index 5b461093..66108b7b 100644 --- a/src/main/game-text-editing.ts +++ b/src/main/game-text-editing.ts @@ -10,32 +10,32 @@ import { type GameTextEditRequest, } from '../shared/contracts.js'; -const sendControlChord = (contents: WebContents, keyCode: 'A' | 'X'): void => { +const sendControlPaste = (contents: WebContents): void => { contents.sendInputEvent({ type: 'keyDown', keyCode: 'Control', modifiers: ['control'], }); contents.sendInputEvent({ - type: 'keyDown', keyCode, modifiers: ['control'], + type: 'keyDown', keyCode: 'V', modifiers: ['control'], }); + // sendInputEvent reproduces the key transition but does not run Chromium's + // default Paste action. Invoke it while Control-V is down, matching the + // ordering of a physical Control-V gesture. + contents.paste(); contents.sendInputEvent({ - type: 'keyUp', keyCode, modifiers: ['control'], + type: 'keyUp', keyCode: 'V', modifiers: ['control'], }); contents.sendInputEvent({ type: 'keyUp', keyCode: 'Control' }); }; -const sendControlPaste = (contents: WebContents): void => { +const sendControlChord = (contents: WebContents, keyCode: 'A' | 'X'): void => { contents.sendInputEvent({ type: 'keyDown', keyCode: 'Control', modifiers: ['control'], }); contents.sendInputEvent({ - type: 'keyDown', keyCode: 'V', modifiers: ['control'], + type: 'keyDown', keyCode, modifiers: ['control'], }); - // sendInputEvent reproduces the key transition but does not run Chromium's - // default Paste action. Invoke it while Control-V is down, matching the - // ordering of a physical Control-V gesture. - contents.paste(); contents.sendInputEvent({ - type: 'keyUp', keyCode: 'V', modifiers: ['control'], + type: 'keyUp', keyCode, modifiers: ['control'], }); contents.sendInputEvent({ type: 'keyUp', keyCode: 'Control' }); }; diff --git a/src/main/ipc.ts b/src/main/ipc.ts index c1b7e6f8..cd5f66d4 100644 --- a/src/main/ipc.ts +++ b/src/main/ipc.ts @@ -134,6 +134,7 @@ import { submitVisualCapture, } from "./visual-capture.js"; import { parseDiagnosticProfile } from "./core/diagnostic-profile.js"; +import type { DictationController } from './dictation.js'; export interface IpcContext { sockets: SocketManager; @@ -146,6 +147,7 @@ export interface IpcContext { getCacheInfo: () => Promise; getSettings: () => Promise; updateSettings: (patch: RendererSettingsPatch) => Promise; + dictation: DictationController; setDiagnosticProfile: (profile: DiagnosticProfile) => Promise; resetSettings: () => Promise; /** Whether this process started with every certified Tools capability prepared. */ @@ -517,6 +519,16 @@ export function registerIpcHandlers(ctx: IpcContext): { settingsRestartForTools: channel(nothing, (win) => requestToolsUnloadRestart(win)), + dictationPrepare: channel(nothing, (win) => ctx.dictation.prepare(win)), + dictationStart: channel(nothing, async (win) => { + if (!(await ctx.getSettings()).dictationEnabled) { + throw new ValidationError('dictation is disabled'); + } + await ctx.dictation.start(win); + }), + dictationFinish: channel(nothing, (win) => ctx.dictation.finish(win)), + dictationCancel: channel(nothing, (win) => ctx.dictation.cancel(win)), + shortcutCapture: channel(nothing, (win) => captureWindowShortcut(win)), shortcutCaptureCancel: channel(nothing, (win) => { cancelWindowShortcutCapture(win); diff --git a/src/main/main.ts b/src/main/main.ts index 659d5523..61a0c73b 100644 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -139,6 +139,7 @@ import { import { MultipleAccountsController } from "./multiple-accounts-controller.js"; import { GameReloader } from "./game-reload.js"; import { updateToolsMenuItems } from "./window-menu.js"; +import { DictationController } from './dictation.js'; // The public app name changed after alpha profiles already existed. Keep that // one profile as the canonical home so the rename cannot strand saved login, @@ -447,6 +448,11 @@ if (primaryInstance) void app.whenReady().then(async () => { appPath: app.getAppPath(), resourcesPath: process.resourcesPath, }); + const dictation = new DictationController( + nativeHost, + () => systemPreferences.askForMediaAccess('microphone'), + app.isPackaged, + ); const stopCommandKeyUps = installMacosCommandKeyUps(nativeHost, { focusedGameTarget() { return windowRegistry.focusedGameWindow(); @@ -467,6 +473,7 @@ if (primaryInstance) void app.whenReady().then(async () => { }, }); app.once("will-quit", () => stopCommandKeyUps()); + app.once('will-quit', () => dictation.dispose()); const paths = gamePaths(); try { activeAccountMode = await loadAccountMode(paths.launcherMode); @@ -722,6 +729,7 @@ if (primaryInstance) void app.whenReady().then(async () => { getCacheInfo: () => clientRuntime.cacheInfo(), getSettings: () => preferences.getSettings(), updateSettings: (patch) => preferences.updateRendererSettings(patch), + dictation, resetSettings: () => toolsRuntime?.resetSettings() ?? preferences.resetCoreSettings(), setDiagnosticProfile: async (profile) => { diff --git a/src/main/native-host.ts b/src/main/native-host.ts index 4069205c..837ece17 100644 --- a/src/main/native-host.ts +++ b/src/main/native-host.ts @@ -9,6 +9,23 @@ import { createRequire } from "node:module"; import type { NativeKeychain } from "./core/native-keychain.js"; import { unpackedPath, type BundleLayout } from "./core/paths.js"; +export type NativeDictationEvent = + | Readonly<{ type: "preparing"; progress?: number }> + | Readonly<{ type: "ready"; locale: string }> + | Readonly<{ type: "listening" }> + | Readonly<{ type: "result"; transcript: string; final: boolean }> + | Readonly<{ + type: "error"; + reason: + | "permission-denied" + | "unavailable" + | "model-unavailable" + | "model-download-failed" + | "setup-required" + | "audio-unavailable" + | "recognition-failed"; + }>; + export interface NativeHost extends NativeKeychain { /** * Observe app-local key releases owned by a Command chord. Returning true @@ -16,6 +33,10 @@ export interface NativeHost extends NativeKeychain { * owns its replacement. */ monitorCommandKeyUps(handler: (keyCode: number) => boolean): () => void; + startDictation(handler: (event: NativeDictationEvent) => void): void; + prepareDictation(handler: (event: NativeDictationEvent) => void): void; + finishDictation(): void; + cancelDictation(): void; } export type NativeHostLayout = BundleLayout; @@ -32,6 +53,10 @@ function isNativeHost(value: unknown): value is NativeHost { typeof candidate.save === "function" && typeof candidate.clear === "function" && typeof candidate.monitorCommandKeyUps === "function" + && typeof candidate.startDictation === "function" + && typeof candidate.prepareDictation === "function" + && typeof candidate.finishDictation === "function" + && typeof candidate.cancelDictation === "function" ); } diff --git a/src/main/settings-actions.ts b/src/main/settings-actions.ts index 596d2065..3acc39eb 100644 --- a/src/main/settings-actions.ts +++ b/src/main/settings-actions.ts @@ -92,21 +92,29 @@ export async function applySettingsChange( ): Promise { try { const previous = await read(); - const restartForCapability = patch.gwonmacTools === true - && !toolsEnabledAtLaunch; + const capabilityToEnable = patch.gwonmacTools === true && !toolsEnabledAtLaunch + ? "tools" + : patch.dictationEnabled === true + && !previous.dictationEnabled + && !toolsEnabledAtLaunch + ? "dictation" + : null; if ( - restartForCapability + capabilityToEnable !== null && !(await confirmAction(win, { confirmLabel: "Enable and Restart", - message: "Restart to enable Tools Beta?", - detail: - "GWonMac prepares every certified Tools capability together. Restart once to use the saved change. This closes Guild Wars if it is running.", + message: capabilityToEnable === "tools" + ? "Restart to enable Tools Beta?" + : "Restart to enable in-game dictation?", + detail: capabilityToEnable === "tools" + ? "GWonMac prepares every certified Tools capability together. Restart once to use the saved change. This closes Guild Wars if it is running." + : "The Apple language model is ready. Restart once so GWonMac can attach the microphone to the certified Guild Wars chat frame. This closes Guild Wars if it is running.", })) ) { return previous; } const saved = await write(patch); - if (restartForCapability) requestRelaunch(win, "capabilityEnable"); + if (capabilityToEnable !== null) requestRelaunch(win, "capabilityEnable"); return saved; } catch (error) { logEvent({ k: "settings.saveFailed", code: errorCode(error) }); diff --git a/src/native/host/dictation-host.hpp b/src/native/host/dictation-host.hpp new file mode 100644 index 00000000..dbfd20c8 --- /dev/null +++ b/src/native/host/dictation-host.hpp @@ -0,0 +1,9 @@ +#pragma once + +#include + +napi_value GwonmacStartDictation(napi_env env, napi_callback_info info); +napi_value GwonmacPrepareDictation(napi_env env, napi_callback_info info); +napi_value GwonmacFinishDictation(napi_env env, napi_callback_info info); +napi_value GwonmacCancelDictation(napi_env env, napi_callback_info info); +void GwonmacCleanupDictation(void *data); diff --git a/src/native/host/dictation-host.mm b/src/native/host/dictation-host.mm new file mode 100644 index 00000000..72739892 --- /dev/null +++ b/src/native/host/dictation-host.mm @@ -0,0 +1,245 @@ +/** Node-API ownership for the single app-wide Apple Speech session. */ + +#include "dictation-host.hpp" + +#import + +#include +#include +#include + +extern "C" { +using GwonmacSpeechCallback = void (*)(void *, int32_t, const char *, double, + int32_t); +bool GwonmacModernSpeechAvailable(); +void GwonmacModernSpeechStart(GwonmacSpeechCallback callback, void *context); +void GwonmacModernSpeechPrepare(GwonmacSpeechCallback callback, void *context); +void GwonmacModernSpeechFinish(); +void GwonmacModernSpeechCancel(); +} + +namespace { + +struct DictationSession { + napi_env env = nullptr; + napi_ref callback = nullptr; + napi_async_context async_context = nullptr; + uintptr_t token = 0; + bool finishing = false; +}; + +DictationSession *gSession = nullptr; +uintptr_t gNextToken = 0; + +void ClearPendingException(napi_env env) { + bool pending = false; + if (napi_is_exception_pending(env, &pending) == napi_ok && pending) { + napi_value ignored; + napi_get_and_clear_last_exception(env, &ignored); + } +} + +void SetUtf8(napi_env env, napi_value object, const char *name, + const char *value) { + napi_value string; + if (napi_create_string_utf8(env, value, NAPI_AUTO_LENGTH, &string) == napi_ok) + napi_set_named_property(env, object, name, string); +} + +void SetDouble(napi_env env, napi_value object, const char *name, + double value) { + napi_value number; + if (napi_create_double(env, value, &number) == napi_ok) + napi_set_named_property(env, object, name, number); +} + +void Dispatch(DictationSession *session, const char *type, + NSString *transcript = nil, bool final = false, + const char *reason = nullptr, double progress = -1) { + if (session == nullptr || session != gSession) + return; + napi_handle_scope scope = nullptr; + napi_value callback; + napi_value receiver; + napi_value event; + napi_value ignored; + if (napi_open_handle_scope(session->env, &scope) != napi_ok || + napi_get_reference_value(session->env, session->callback, &callback) != + napi_ok || + napi_get_global(session->env, &receiver) != napi_ok || + napi_create_object(session->env, &event) != napi_ok) { + ClearPendingException(session->env); + if (scope != nullptr) + napi_close_handle_scope(session->env, scope); + return; + } + SetUtf8(session->env, event, "type", type); + if (transcript != nil) { + if (std::strcmp(type, "ready") == 0) { + SetUtf8(session->env, event, "locale", transcript.UTF8String ?: ""); + } else { + SetUtf8(session->env, event, "transcript", transcript.UTF8String ?: ""); + napi_value finalValue; + if (napi_get_boolean(session->env, final, &finalValue) == napi_ok) + napi_set_named_property(session->env, event, "final", finalValue); + } + } + if (reason != nullptr) + SetUtf8(session->env, event, "reason", reason); + if (progress >= 0) + SetDouble(session->env, event, "progress", progress); + napi_make_callback(session->env, session->async_context, receiver, callback, + 1, &event, &ignored); + ClearPendingException(session->env); + napi_close_handle_scope(session->env, scope); +} + +void Cleanup(bool cancelTask) { + DictationSession *session = gSession; + if (session == nullptr) + return; + gSession = nullptr; + if (cancelTask) + GwonmacModernSpeechCancel(); + if (session->callback != nullptr) + napi_delete_reference(session->env, session->callback); + if (session->async_context != nullptr) + napi_async_destroy(session->env, session->async_context); + delete session; +} + +void Receive(void *context, int32_t event, const char *text, double progress, + int32_t final) { + const auto token = reinterpret_cast(context); + DictationSession *session = gSession; + if (session == nullptr || session->token != token) + return; + switch (event) { + case 0: + Dispatch(session, "preparing"); + return; + case 1: + Dispatch(session, "preparing", nil, false, nullptr, progress); + return; + case 2: + Dispatch(session, "ready", text == nullptr + ? @"" : [NSString stringWithUTF8String:text]); + Cleanup(false); + return; + case 3: + Dispatch(session, "listening"); + return; + case 4: { + NSString *transcript = text == nullptr + ? @"" + : [NSString stringWithUTF8String:text]; + Dispatch(session, "result", transcript ?: @"", final != 0); + if (final != 0) + Cleanup(false); + return; + } + case 5: + Dispatch(session, "error", nil, false, + text == nullptr ? "recognition-failed" : text); + Cleanup(false); + return; + default: + Dispatch(session, "error", nil, false, "recognition-failed"); + Cleanup(false); + } +} + +napi_value Begin(napi_env env, napi_callback_info info, bool setup) { + size_t argc = 1; + napi_value argv[1]; + napi_valuetype type = napi_undefined; + if (napi_get_cb_info(env, info, &argc, argv, nullptr, nullptr) != napi_ok || + argc != 1 || napi_typeof(env, argv[0], &type) != napi_ok || + type != napi_function || ![NSThread isMainThread]) { + napi_throw_type_error(env, nullptr, "dictation callback is required"); + return nullptr; + } + Cleanup(true); + auto *session = new (std::nothrow) DictationSession(); + if (session == nullptr) { + napi_throw_error(env, nullptr, "dictation is unavailable"); + return nullptr; + } + session->env = env; + session->token = ++gNextToken; + if (session->token == 0) + session->token = ++gNextToken; + napi_value resource; + napi_value resourceName; + if (napi_create_reference(env, argv[0], 1, &session->callback) != napi_ok || + napi_create_object(env, &resource) != napi_ok || + napi_create_string_utf8(env, "gwonmac.dictation", NAPI_AUTO_LENGTH, + &resourceName) != napi_ok || + napi_async_init(env, resource, resourceName, &session->async_context) != + napi_ok) { + if (session->callback != nullptr) + napi_delete_reference(env, session->callback); + if (session->async_context != nullptr) + napi_async_destroy(env, session->async_context); + delete session; + napi_throw_error(env, nullptr, "dictation is unavailable"); + return nullptr; + } + gSession = session; + if (!GwonmacModernSpeechAvailable()) { + Dispatch(session, "error", nil, false, "unavailable"); + Cleanup(false); + } else if (setup) { + GwonmacModernSpeechPrepare(Receive, + reinterpret_cast(session->token)); + } else { + GwonmacModernSpeechStart(Receive, + reinterpret_cast(session->token)); + } + napi_value undefined; + napi_get_undefined(env, &undefined); + return undefined; +} + +bool HasNoArguments(napi_env env, napi_callback_info info) { + size_t argc = 0; + return napi_get_cb_info(env, info, &argc, nullptr, nullptr, nullptr) == napi_ok + && argc == 0; +} + +} // namespace + +napi_value GwonmacStartDictation(napi_env env, napi_callback_info info) { + return Begin(env, info, false); +} + +napi_value GwonmacPrepareDictation(napi_env env, napi_callback_info info) { + return Begin(env, info, true); +} + +napi_value GwonmacFinishDictation(napi_env env, napi_callback_info info) { + if (!HasNoArguments(env, info)) { + napi_throw_type_error(env, nullptr, "invalid dictation arguments"); + return nullptr; + } + if (gSession != nullptr && !gSession->finishing) { + gSession->finishing = true; + GwonmacModernSpeechFinish(); + } + napi_value undefined; + napi_get_undefined(env, &undefined); + return undefined; +} + +napi_value GwonmacCancelDictation(napi_env env, napi_callback_info info) { + if (!HasNoArguments(env, info)) { + napi_throw_type_error(env, nullptr, "invalid dictation arguments"); + return nullptr; + } + Cleanup(true); + napi_value undefined; + napi_get_undefined(env, &undefined); + return undefined; +} + +void GwonmacCleanupDictation(void *) { Cleanup(true); } diff --git a/src/native/host/dictation-modern.swift b/src/native/host/dictation-modern.swift new file mode 100644 index 00000000..cadb8668 --- /dev/null +++ b/src/native/host/dictation-modern.swift @@ -0,0 +1,339 @@ +import AVFoundation +import Foundation +import Speech + +typealias GwonmacSpeechCallback = @convention(c) ( + UnsafeMutableRawPointer?, + Int32, + UnsafePointer?, + Double, + Int32 +) -> Void + +private enum SpeechEvent: Int32 { + case preparing = 0 + case downloading = 1 + case ready = 2 + case listening = 3 + case result = 4 + case error = 5 +} + +private enum SpeechOperation { + case setup + case capture +} + +@available(macOS 26.0, *) +@MainActor +private final class ModernDictation { + private let callback: GwonmacSpeechCallback + private let context: UnsafeMutableRawPointer? + private let operation: SpeechOperation + private var analyzer: SpeechAnalyzer? + private var engine: AVAudioEngine? + private var inputContinuation: AsyncStream.Continuation? + private var resultsTask: Task? + private var workTask: Task? + private var finalizedText = "" + private var volatileText = "" + private var finishing = false + + init( + operation: SpeechOperation, + callback: @escaping GwonmacSpeechCallback, + context: UnsafeMutableRawPointer? + ) { + self.operation = operation + self.callback = callback + self.context = context + } + + func start() { + emit(.preparing) + workTask = Task { [weak self] in await self?.run() } + } + + func finish() { + guard !finishing else { return } + finishing = true + workTask?.cancel() + workTask = Task { [weak self] in + guard let self else { return } + self.stopCapture() + if let analyzer = self.analyzer { + try? await analyzer.finalizeAndFinishThroughEndOfInput() + } + await self.resultsTask?.value + self.emit(.result, text: self.currentText, final: true) + self.clear() + } + } + + func cancel() { + workTask?.cancel() + resultsTask?.cancel() + stopCapture() + if let analyzer { + Task { await analyzer.cancelAndFinishNow() } + } + clear() + } + + private func run() async { + guard let locale = await preferredLocale() else { + fail("model-unavailable") + return + } + + let transcriber = DictationTranscriber( + locale: locale, + preset: .progressiveShortDictation + ) + do { + if operation == .setup { + if let request = try await AssetInventory.assetInstallationRequest( + supporting: [transcriber] + ) { + let progressTask = Task { [weak self] in + while !Task.isCancelled { + self?.emit(.downloading, progress: request.progress.fractionCompleted) + try? await Task.sleep(for: .milliseconds(200)) + } + } + defer { progressTask.cancel() } + try await request.downloadAndInstall() + } + try Task.checkCancellation() + emit(.ready, text: locale.localizedString(forIdentifier: locale.identifier) + ?? locale.identifier) + clear() + return + } + + guard await AssetInventory.status(forModules: [transcriber]) == .installed else { + fail("setup-required") + return + } + + try await listen(with: transcriber) + } catch is CancellationError { + return + } catch { + fail(operation == .setup ? "model-download-failed" : "recognition-failed") + } + } + + private func listen(with transcriber: DictationTranscriber) async throws { + let analyzer = SpeechAnalyzer(modules: [transcriber]) + let engine = AVAudioEngine() + let input = engine.inputNode + let naturalFormat = input.outputFormat(forBus: 0) + guard naturalFormat.sampleRate > 0, + let analyzerFormat = await SpeechAnalyzer.bestAvailableAudioFormat( + compatibleWith: [transcriber], + considering: naturalFormat + ) else { + fail("audio-unavailable") + return + } + let converter = naturalFormat == analyzerFormat + ? nil + : AVAudioConverter(from: naturalFormat, to: analyzerFormat) + guard naturalFormat == analyzerFormat || converter != nil else { + fail("audio-unavailable") + return + } + let (inputs, continuation) = AsyncStream.makeStream(of: AnalyzerInput.self) + + self.analyzer = analyzer + self.engine = engine + self.inputContinuation = continuation + observe(transcriber) + try await analyzer.start(inputSequence: inputs) + input.installTap(onBus: 0, bufferSize: 1024, format: naturalFormat) { + buffer, _ in + if converter == nil { + continuation.yield(AnalyzerInput(buffer: buffer)) + return + } + let ratio = analyzerFormat.sampleRate / naturalFormat.sampleRate + let capacity = AVAudioFrameCount(ceil(Double(buffer.frameLength) * ratio)) + 1 + guard let output = AVAudioPCMBuffer( + pcmFormat: analyzerFormat, + frameCapacity: capacity + ) else { return } + var supplied = false + var conversionError: NSError? + let status = converter!.convert(to: output, error: &conversionError) { + _, inputStatus in + guard !supplied else { + inputStatus.pointee = .noDataNow + return nil + } + supplied = true + inputStatus.pointee = .haveData + return buffer + } + if status == .haveData && conversionError == nil { + continuation.yield(AnalyzerInput(buffer: output)) + } + } + engine.prepare() + try engine.start() + try Task.checkCancellation() + emit(.listening) + } + + private func preferredLocale() async -> Locale? { + var identifiers = Locale.preferredLanguages + identifiers.append(Locale.current.identifier) + identifiers.append("en-US") + var seen = Set() + for identifier in identifiers where seen.insert(identifier).inserted { + if let locale = await DictationTranscriber.supportedLocale( + equivalentTo: Locale(identifier: identifier) + ) { + return locale + } + } + return nil + } + + private func observe(_ transcriber: DictationTranscriber) { + resultsTask = Task { [weak self] in + do { + for try await result in transcriber.results { + guard let self, !Task.isCancelled else { return } + let text = String(result.text.characters).trimmingCharacters(in: .whitespacesAndNewlines) + if result.isFinal { + self.appendFinal(text) + self.volatileText = "" + } else { + self.volatileText = text + } + self.emit(.result, text: self.currentText) + } + } catch is CancellationError { + return + } catch { + self?.fail("recognition-failed") + } + } + } + + private func appendFinal(_ text: String) { + guard !text.isEmpty else { return } + finalizedText += finalizedText.isEmpty ? text : " \(text)" + } + + private var currentText: String { + guard !volatileText.isEmpty else { return finalizedText } + return finalizedText.isEmpty ? volatileText : "\(finalizedText) \(volatileText)" + } + + private func fail(_ reason: String) { + guard !finishing else { return } + emit(.error, text: reason) + cancel() + } + + private func stopCapture() { + if let engine { + engine.inputNode.removeTap(onBus: 0) + engine.stop() + } + inputContinuation?.finish() + inputContinuation = nil + } + + private func clear() { + workTask = nil + resultsTask = nil + analyzer = nil + engine = nil + inputContinuation = nil + if modernDictation === self { + modernDictation = nil + } + } + + private func emit( + _ event: SpeechEvent, + text: String? = nil, + progress: Double = -1, + final: Bool = false + ) { + if let text { + text.withCString { + callback(context, event.rawValue, $0, progress, final ? 1 : 0) + } + } else { + callback(context, event.rawValue, nil, progress, final ? 1 : 0) + } + } +} + +@available(macOS 26.0, *) +@MainActor +private var modernDictation: ModernDictation? + +@_cdecl("GwonmacModernSpeechAvailable") +func GwonmacModernSpeechAvailable() -> Bool { + if #available(macOS 26.0, *) { return true } + return false +} + +@_cdecl("GwonmacModernSpeechStart") +func GwonmacModernSpeechStart( + _ callback: GwonmacSpeechCallback?, + _ context: UnsafeMutableRawPointer? +) { + guard let callback else { return } + if #available(macOS 26.0, *) { + Task { @MainActor in + modernDictation?.cancel() + let dictation = ModernDictation( + operation: .capture, + callback: callback, + context: context + ) + modernDictation = dictation + dictation.start() + } + } +} + +@_cdecl("GwonmacModernSpeechPrepare") +func GwonmacModernSpeechPrepare( + _ callback: GwonmacSpeechCallback?, + _ context: UnsafeMutableRawPointer? +) { + guard let callback else { return } + if #available(macOS 26.0, *) { + Task { @MainActor in + modernDictation?.cancel() + let dictation = ModernDictation( + operation: .setup, + callback: callback, + context: context + ) + modernDictation = dictation + dictation.start() + } + } +} + +@_cdecl("GwonmacModernSpeechFinish") +func GwonmacModernSpeechFinish() { + if #available(macOS 26.0, *) { + Task { @MainActor in modernDictation?.finish() } + } +} + +@_cdecl("GwonmacModernSpeechCancel") +func GwonmacModernSpeechCancel() { + if #available(macOS 26.0, *) { + Task { @MainActor in modernDictation?.cancel() } + } +} diff --git a/src/native/host/host.mm b/src/native/host/host.mm index 58c951c8..b9b2438f 100644 --- a/src/native/host/host.mm +++ b/src/native/host/host.mm @@ -6,6 +6,8 @@ #include +#include "dictation-host.hpp" + #import #import #import @@ -583,7 +585,16 @@ napi_value Init(napi_env env, napi_value exports) { nullptr}, {"monitorCommandKeyUps", nullptr, MonitorCommandKeyUpsCallback, nullptr, nullptr, nullptr, napi_default, nullptr}, + {"startDictation", nullptr, GwonmacStartDictation, nullptr, nullptr, + nullptr, napi_default, nullptr}, + {"prepareDictation", nullptr, GwonmacPrepareDictation, nullptr, nullptr, + nullptr, napi_default, nullptr}, + {"finishDictation", nullptr, GwonmacFinishDictation, nullptr, nullptr, + nullptr, napi_default, nullptr}, + {"cancelDictation", nullptr, GwonmacCancelDictation, nullptr, nullptr, + nullptr, napi_default, nullptr}, }; + napi_add_env_cleanup_hook(env, GwonmacCleanupDictation, nullptr); if (napi_define_properties(env, exports, sizeof(properties) / sizeof(properties[0]), properties) != napi_ok) { diff --git a/src/preload/preload.body.cjs b/src/preload/preload.body.cjs index bb0413ba..94bfbfff 100644 --- a/src/preload/preload.body.cjs +++ b/src/preload/preload.body.cjs @@ -188,6 +188,13 @@ const api = { restartForTools: () => ipcRenderer.invoke(IPC.settingsRestartForTools), onChange: (callback) => listen(IPC.settingsEvent, callback), }, + dictation: { + prepare: () => ipcRenderer.invoke(IPC.dictationPrepare), + start: () => ipcRenderer.invoke(IPC.dictationStart), + finish: () => ipcRenderer.invoke(IPC.dictationFinish), + cancel: () => ipcRenderer.invoke(IPC.dictationCancel), + onEvent: (callback) => listen(IPC.dictationEvent, callback), + }, shortcuts: { capture: () => ipcRenderer.invoke(IPC.shortcutCapture), cancelCapture: () => ipcRenderer.invoke(IPC.shortcutCaptureCancel), diff --git a/src/renderer/certified-companion-core-installation.ts b/src/renderer/certified-companion-core-installation.ts index 504f2026..bf9876cc 100644 --- a/src/renderer/certified-companion-core-installation.ts +++ b/src/renderer/certified-companion-core-installation.ts @@ -25,6 +25,8 @@ import { } from "./cursor-refresh.js"; import { decodeEnhancementManifest } from "./enhancement-manifest.js"; import { createPlayRegionObservationInstallation } from "./play-region-state-installation.js"; +import { createSkillSlotGeometryInstallation } from "./skill-slot-geometry-installation.js"; +import type { CompanionSkillSlotState } from "./companion-interface-geometry-snapshot.js"; import type { CompanionExtensionSession, PrepareCompanionExtension, @@ -59,6 +61,9 @@ export async function installCoreCertifiedCompanion( (capabilities.nativeCursor ? COMPANION_FEATURE_BITS.nativeCursor : 0) | (capabilities.playRegionObservation ? COMPANION_FEATURE_BITS.playRegionObservation + : 0) + | (capabilities.skillSlotGeometry + ? COMPANION_FEATURE_BITS.skillSlotGeometry : 0); const manifest = decodeEnhancementManifest(module, capabilities); const exports = instance.exports; @@ -119,6 +124,9 @@ export async function installCoreCertifiedCompanion( const playRegions = createPlayRegionObservationInstallation( capabilities.playRegionObservation, ); + const skillGeometry = createSkillSlotGeometryInstallation( + capabilities.skillSlotGeometry, + ); let coreMemory: ReturnType | null = null; let stopObserver = () => {}; let disposeCursor = () => {}; @@ -162,7 +170,8 @@ export async function installCoreCertifiedCompanion( attempt("extension presentation disposal", () => { extensionSession?.disposePresentation(); }); - attempt("play-region feed disposal", playRegions.dispose); + attempt("play-region feed disposal", playRegions.dispose); + attempt("interface geometry feed disposal", skillGeometry.dispose); } const callbackWithdrawn = attempt("callback withdrawal", () => { if (table.get(manifest.tableSlot) === installedCallback) { @@ -176,6 +185,7 @@ export async function installCoreCertifiedCompanion( extensionSession?.releaseObserverMemory(free); }); attempt("play-region allocation release", () => playRegions.release(free)); + attempt("interface geometry allocation release", () => skillGeometry.release(free)); } attempt("extension callback resource release", () => { if (extensionSession) extensionSession.releaseCallbackResources(free); @@ -234,12 +244,16 @@ export async function installCoreCertifiedCompanion( }); const core = coreMemory; playRegions.allocate(malloc); + skillGeometry.allocate(malloc); extension?.allocate(malloc); - if (!playRegions.allocated) throw new Error("Companion allocation failed"); + if (!playRegions.allocated || !skillGeometry.allocated) { + throw new Error("Companion allocation failed"); + } validateCompanionOwnedRegions([ ...core.regions, ...(extension?.ownedRegions() ?? []), ...(playRegions.region === null ? [] : [playRegions.region]), + ...(skillGeometry.region === null ? [] : [skillGeometry.region]), ], memory.buffer.byteLength); core.initialize(); extension?.initialize(memory); @@ -254,7 +268,7 @@ export async function installCoreCertifiedCompanion( cursor: core.cursor, toolbox: core.toolbox, party: core.party, - skillSlots: extension?.kernelRegions.skillSlots ?? { pointer: 0, bytes: 0 }, + skillSlots: { pointer: skillGeometry.pointer, bytes: skillGeometry.bytes }, skillCooldowns: extension?.kernelRegions.skillCooldowns ?? { pointer: 0, bytes: 0 }, playRegion: { pointer: playRegions.pointer, bytes: playRegions.bytes }, }, @@ -302,6 +316,7 @@ export async function installCoreCertifiedCompanion( core, kernel, playRegions, + skillGeometry, capabilities, program, isCleaned: () => cleaned, @@ -311,6 +326,7 @@ export async function installCoreCertifiedCompanion( }, }) ?? null; playRegions.setActive(true); + skillGeometry.setActive(true); extensionSession?.beforeHook(); table.set(manifest.tableSlot, kernel.dispatch); installedCallback = kernel.dispatch; @@ -319,7 +335,7 @@ export async function installCoreCertifiedCompanion( snapshotPointer: extensionSession?.observer.pointers.snapshot ?? 0, toolboxPointer: extensionSession?.observer.pointers.toolbox ?? 0, partyPointer: extensionSession?.observer.pointers.party ?? 0, - skillSlotPointer: extensionSession?.observer.pointers.skillSlots ?? 0, + skillSlotPointer: skillGeometry.pointer, skillCooldownPointer: extensionSession?.observer.pointers.skillCooldowns ?? 0, playRegionPointer: playRegions.pointer, hertz: 0, @@ -345,7 +361,16 @@ export async function installCoreCertifiedCompanion( extensionSession?.observer.toolbox ?? null, extensionSession?.observer.observeState ?? false, extensionSession?.observer.publishState ?? false, - extensionSession?.observer.skillSlots ?? null, + capabilities.skillSlotGeometry ? { + enabled: () => skillGeometry.active, + inactive: () => skillGeometry.setActive(false), + update: (state: CompanionSkillSlotState) => { + skillGeometry.sink?.update(state); + window.dispatchEvent(new CustomEvent("gwonmac:chat-geometry", { + detail: skillGeometry.state, + })); + }, + } : null, extensionSession?.observer.skillCooldowns ?? null, playRegions.sink, extensionSession?.observer.readers ?? null, diff --git a/src/renderer/certified-companion-extension.ts b/src/renderer/certified-companion-extension.ts index e3fb1ac3..54cb69d1 100644 --- a/src/renderer/certified-companion-extension.ts +++ b/src/renderer/certified-companion-extension.ts @@ -20,10 +20,12 @@ import type { ToolboxConsumer, } from "./companion-observer.js"; import type { createPlayRegionObservationInstallation } from "./play-region-state-installation.js"; +import type { createSkillSlotGeometryInstallation } from "./skill-slot-geometry-installation.js"; type CoreMemory = ReturnType; type Kernel = Awaited>; type PlayRegions = ReturnType; +type SkillGeometry = ReturnType; export type KernelRegion = Readonly<{ pointer: number; bytes: number }>; export type CompanionObserverExtension = Readonly<{ @@ -63,6 +65,7 @@ export type CompanionExtensionActivation = Readonly<{ core: CoreMemory; kernel: Kernel; playRegions: PlayRegions; + skillGeometry: SkillGeometry; capabilities: EnhancementCapabilities; program: EnhancementProgram; isCleaned(): boolean; diff --git a/src/renderer/certified-companion-tools-installation.ts b/src/renderer/certified-companion-tools-installation.ts index e6950a8a..e73d8557 100644 --- a/src/renderer/certified-companion-tools-installation.ts +++ b/src/renderer/certified-companion-tools-installation.ts @@ -6,7 +6,7 @@ import { COMPANION_DISPATCH_KINDS, COMPANION_FEATURE_BITS } from "../shared/comp import type { EnhancementCapabilities, EnhancementProgram } from "../shared/enhancement-contracts.js"; import type { ToolboxObservation } from "../shared/builds/live-party.js"; import type { CompanionSnapshot } from "./companion-snapshot.js"; -import type { CompanionSkillSlotState } from "./companion-skill-snapshot.js"; +import { createSkillCooldownObservationInstallation } from "./skill-cooldown-state-installation.js"; import type { EnhancementCommandEnqueue } from "./enhancement-team-commands.js"; import type * as TeamCommandsModule from "./enhancement-team-commands.js"; import type { ProfessionCommandTraceReader } from "./profession-command-trace.js"; @@ -19,7 +19,6 @@ import type { } from "./certified-companion-extension.js"; import { setSkillCooldownReadiness, - setSkillGeometryReadiness, } from "./observer-readiness.js"; import * as tools from "./certified-companion-tools.js"; import { @@ -44,25 +43,6 @@ function runCleanupSteps( if (failures.length > 0) throw new AggregateError(failures, message); } -function reportSkillGeometry(state: CompanionSkillSlotState): void { - const readiness = state.status === "ready" - ? Object.freeze({ status: "ready" as const }) - : Object.freeze({ status: "waiting" as const, reason: state.reason }); - if (!setSkillGeometryReadiness(readiness)) return; - const fields = state.status === "ready" - ? { state: "ready" as const, reason: null, candidates: null } - : { - state: "waiting" as const, - reason: state.reason, - candidates: "candidateCount" in state ? state.candidateCount : null, - }; - void window.gwNative.diagnostics.recordRendererMilestone( - "enhancement.skillGeometryState", - performance.now() * 1_000, - fields, - ).catch(() => {}); -} - export async function prepareToolsCompanionExtension( exports: WebAssembly.Exports, capabilities: EnhancementCapabilities, @@ -70,9 +50,9 @@ export async function prepareToolsCompanionExtension( ): Promise { const foundation = capabilities.partyObservation; const observeState = capabilities.targetObservation || capabilities.xunlaiAction; - const skills = tools.createSkillOverlaysInstallation(capabilities); - const slots = skills.geometry; - const cooldowns = skills.cooldowns; + const cooldowns = createSkillCooldownObservationInstallation( + capabilities.skillCooldownObservation, + ); const enqueue = capabilities.teamApply && typeof exports.enhancement_command === "function" ? exports.enhancement_command as EnhancementCommandEnqueue : null; const traceReader = capabilities.teamApply @@ -109,7 +89,9 @@ export async function prepareToolsCompanionExtension( (observeState ? COMPANION_FEATURE_BITS.gameSnapshot : 0) | (foundation ? COMPANION_FEATURE_BITS.toolboxFoundation : 0) | (capabilities.targetObservation ? COMPANION_FEATURE_BITS.targetObservation : 0) - | skills.certifiedFeatureFlags, + | (capabilities.skillCooldownObservation + ? COMPANION_FEATURE_BITS.skillCooldownObservation + : 0), memoryNeeds: { snapshot: observeState, toolbox: foundation, @@ -120,12 +102,10 @@ export async function prepareToolsCompanionExtension( }, allocate(malloc) { allocated = true; - slots?.allocate(malloc); cooldowns?.allocate(malloc); storage?.allocate(malloc); travel?.allocate(malloc); - if ((slots !== null && !slots.allocated) - || (cooldowns !== null && !cooldowns.allocated) + if ((cooldowns !== null && !cooldowns.allocated) || (storage !== null && !storage.region().pointer) || (travel !== null && !travel.region().pointer)) { throw new Error("Companion Tools allocation failed"); @@ -133,15 +113,12 @@ export async function prepareToolsCompanionExtension( }, initialize(memory) { storage?.initialize(memory); travel?.initialize(); }, ownedRegions: () => [ - ...(slots?.region == null ? [] : [slots.region]), ...(cooldowns?.region == null ? [] : [cooldowns.region]), ...(storage === null ? [] : [storage.region()]), ...(travel === null ? [] : [travel.region()]), ], kernelRegions: { - get skillSlots() { - return slots === null ? EMPTY_REGION : { pointer: slots.pointer, bytes: slots.bytes }; - }, + skillSlots: EMPTY_REGION, get skillCooldowns() { return cooldowns === null ? EMPTY_REGION : { pointer: cooldowns.pointer, bytes: cooldowns.bytes }; @@ -149,7 +126,7 @@ export async function prepareToolsCompanionExtension( }, activate(context) { const session = activateTools({ context, capabilities, program, foundation, observeState, - skills, slots, cooldowns, enqueue, traceReader, teamCommands, storage, + cooldowns, enqueue, traceReader, teamCommands, storage, travel, configureTrade, takeTrade }); activated = true; return session; @@ -157,7 +134,6 @@ export async function prepareToolsCompanionExtension( rollback(free) { if (!allocated || activated) return; runCleanupSteps("Companion Tools allocation rollback failed", [ - () => slots?.release(free), () => cooldowns?.release(free), () => storage?.dispose(free), () => travel?.dispose(free), @@ -172,8 +148,6 @@ type ToolsInput = Readonly<{ program: EnhancementProgram; foundation: boolean; observeState: boolean; - skills: ReturnType; - slots: ReturnType["geometry"]; cooldowns: ReturnType["cooldowns"]; enqueue: EnhancementCommandEnqueue | null; traceReader: ProfessionCommandTraceReader | null; @@ -185,10 +159,15 @@ type ToolsInput = Readonly<{ }>; function activateTools(input: ToolsInput): CompanionExtensionSession { - const { context, capabilities, program, foundation, observeState, skills, - slots, cooldowns, enqueue, traceReader, teamCommands, storage, travel, + const { context, capabilities, program, foundation, observeState, + cooldowns, enqueue, traceReader, teamCommands, storage, travel, configureTrade, takeTrade } = input; - const { memory, core, kernel, playRegions } = context; + const { memory, core, kernel, playRegions, skillGeometry } = context; + const skills = tools.createSkillOverlaysInstallation( + capabilities, + skillGeometry, + cooldowns, + ); const source = tools.createCompanionPolicySource({ program, readSettings: window.gwToolsSettings, @@ -216,7 +195,6 @@ function activateTools(input: ToolsInput): CompanionExtensionSession { () => { readout?.dispose(); readout = null; }, () => toolbox?.dispose(), () => skills.disposePresentation(), - () => slots?.dispose(), () => cooldowns?.dispose(), () => { configureTrade?.(0); }, () => professionTrace?.dispose(), @@ -300,7 +278,10 @@ function activateTools(input: ToolsInput): CompanionExtensionSession { | (capabilities.playRegionObservation ? COMPANION_FEATURE_BITS.playRegionObservation : 0) | (policy().targetReadout || cartographyActive() ? COMPANION_FEATURE_BITS.targetObservation : 0) - | skills.activeFeatureFlags, + | skills.activeFeatureFlags + | (snapshot().settings.dictationEnabled && capabilities.skillSlotGeometry + ? COMPANION_FEATURE_BITS.skillSlotGeometry + : 0), 0, 0, 0, 0, ); const syncStorage = () => storage?.update({ @@ -328,6 +309,7 @@ function activateTools(input: ToolsInput): CompanionExtensionSession { const syncConsumers = () => { syncTarget(); skills.sync(snapshot().settings, policy()); + if (snapshot().settings.dictationEnabled) skillGeometry.setActive(true); syncObservers(); syncStorage(); syncTravel(); @@ -380,11 +362,7 @@ function activateTools(input: ToolsInput): CompanionExtensionSession { publishState: program === "target-observer", // Geometry is shared by both HUDs. Cooldowns must keep the slot feed // alive even when custom key labels are disabled. - skillSlots: slots?.sink == null ? null : { enabled: () => slots.active, - inactive: () => reportSkillGeometry(Object.freeze({ - status: "waiting", reason: "inactive", - })), - update: (state) => { slots.sink?.update(state); reportSkillGeometry(state); } }, + skillSlots: null, skillCooldowns: cooldowns?.sink == null ? null : { enabled: () => policy().skillCooldowns, inactive: () => { setSkillCooldownReadiness("waiting"); }, update: (state) => { @@ -393,7 +371,7 @@ function activateTools(input: ToolsInput): CompanionExtensionSession { } }, readers: tools.observerReaders, pointers: { snapshot: core.snapshot.pointer, toolbox: core.toolbox.pointer, - party: core.party.pointer, skillSlots: slots?.pointer ?? 0, + party: core.party.pointer, skillSlots: skillGeometry.pointer, skillCooldowns: cooldowns?.pointer ?? 0 }, }, beforeHook() { @@ -429,7 +407,6 @@ function activateTools(input: ToolsInput): CompanionExtensionSession { }, releaseObserverMemory(free) { runCleanupSteps("Companion Tools observer memory cleanup failed", [ - () => slots?.release(free), () => cooldowns?.release(free), ]); }, diff --git a/src/renderer/companion-interface-geometry-snapshot.ts b/src/renderer/companion-interface-geometry-snapshot.ts new file mode 100644 index 00000000..787c03bc --- /dev/null +++ b/src/renderer/companion-interface-geometry-snapshot.ts @@ -0,0 +1,164 @@ +/** + * Decodes the Core-owned interface-frame region. The kernel publishes skill + * slots and the chat input together because both come from the same certified + * Guild Wars frame table; consumers may use either projection independently. + */ +import { + COMPANION_ABI, + SKILL_GEOMETRY_NATIVE_REASONS, +} from "../shared/companion-abi.js"; + +export const COMPANION_SKILL_SLOT_ABI = COMPANION_ABI.skillSlots.abi; +export const COMPANION_SKILL_SLOT_BYTES = COMPANION_ABI.skillSlots.bytes; + +const SKILL_SLOT_MAGIC = 0x534b5747; +const SKILL_SLOT_READY = 1; +const CHAT_INPUT_READY = 1 << 1; + +/** Decode the bounded frame projection; presentation remains app-owned. */ +export function readCompanionSkillSlots(buffer: ArrayBuffer, pointer: number) { + if ( + !(buffer instanceof ArrayBuffer) + || !Number.isInteger(pointer) + || pointer < 0 + || pointer + COMPANION_SKILL_SLOT_BYTES > buffer.byteLength + ) return Object.freeze({ status: "waiting", reason: "memory" } as const); + const view = new DataView(buffer, pointer, COMPANION_SKILL_SLOT_BYTES); + const firstSequence = view.getUint32(8, true); + if ((firstSequence & 1) !== 0) { + return Object.freeze({ status: "waiting", reason: "writing" } as const); + } + const magic = view.getUint32(0, true); + const abi = view.getUint16(4, true); + const bytes = view.getUint16(6, true); + const flags = view.getUint32(12, true); + const frameId = view.getUint32(16, true); + const outcome = view.getUint32(20, true); + const candidateCount = view.getUint32(24, true); + const viewportWidth = view.getFloat32(28, true); + const viewportHeight = view.getFloat32(32, true); + const slots = Object.freeze(Array.from({ length: 8 }, (_, index) => { + const at = 36 + index * 16; + return Object.freeze({ + left: view.getFloat32(at, true), + bottom: view.getFloat32(at + 4, true), + right: view.getFloat32(at + 8, true), + top: view.getFloat32(at + 12, true), + }); + })); + const chatFrameId = view.getUint32(164, true); + const chatOutcome = view.getUint32(168, true); + const chatInput = Object.freeze({ + left: view.getFloat32(172, true), + bottom: view.getFloat32(176, true), + right: view.getFloat32(180, true), + top: view.getFloat32(184, true), + }); + const secondSequence = view.getUint32(8, true); + if ( + magic !== SKILL_SLOT_MAGIC + || abi !== COMPANION_SKILL_SLOT_ABI + || bytes !== COMPANION_SKILL_SLOT_BYTES + || firstSequence !== secondSequence + || (secondSequence & 1) !== 0 + || (flags & ~(SKILL_SLOT_READY | CHAT_INPUT_READY)) !== 0 + ) return Object.freeze({ status: "waiting", reason: "snapshot" } as const); + if ((flags & (SKILL_SLOT_READY | CHAT_INPUT_READY)) === 0) { + const reason = SKILL_GEOMETRY_NATIVE_REASONS[ + outcome as keyof typeof SKILL_GEOMETRY_NATIVE_REASONS + ]; + if ( + reason === undefined + || (reason === "slot-ambiguous" + ? candidateCount < 2 || candidateCount > 16_384 + : candidateCount !== 0) + ) { + return Object.freeze({ status: "waiting", reason: "snapshot" } as const); + } + return reason === "slot-ambiguous" + ? Object.freeze({ status: "waiting" as const, reason, candidateCount }) + : Object.freeze({ status: "waiting" as const, reason }); + } + const finite = (value: number) => Number.isFinite(value) && Math.abs(value) <= 32_768; + const validRect = ( + { left, bottom, right, top }: typeof chatInput, + width: number, + height: number, + ) => + [left, bottom, right, top].every(finite) + && right > left && top > bottom + && right > 0 && top > 0 + && left < width && bottom < height; + const skillReady = (flags & SKILL_SLOT_READY) !== 0; + const chatReady = (flags & CHAT_INPUT_READY) !== 0; + const validRefusal = (value: number, candidates: number) => { + const reason = SKILL_GEOMETRY_NATIVE_REASONS[ + value as keyof typeof SKILL_GEOMETRY_NATIVE_REASONS + ]; + return reason !== undefined && (reason === "slot-ambiguous" + ? candidates >= 2 && candidates <= 16_384 + : candidates === 0); + }; + if ( + !skillReady + || outcome !== 0 + || frameId === 0 + || candidateCount !== 0 + || !finite(viewportWidth) + || !finite(viewportHeight) + || viewportWidth <= 0 + || viewportHeight <= 0 + || slots.some((slot) => !validRect(slot, viewportWidth, viewportHeight)) + || (chatReady + ? chatOutcome !== 0 || chatFrameId === 0 + || !validRect(chatInput, viewportWidth, viewportHeight) + : !validRefusal(chatOutcome, 0) + || chatFrameId !== 0 + || Object.values(chatInput).some((value) => value !== 0)) + ) return Object.freeze({ status: "waiting", reason: "corrupt" } as const); + return Object.freeze({ + status: "ready" as const, + sequence: secondSequence, + frameId, + chatFrameId, + viewportWidth, + viewportHeight, + slots, + skillSlotsReady: skillReady, + chatInput: chatReady ? chatInput : null, + }); +} + +export type CompanionSkillSlotState = + | ReturnType + | Readonly<{ status: "waiting"; reason: "stale" }>; + +/** Ignore heartbeat-only sequence changes while preserving moved-frame updates. */ +export function sameCompanionSkillSlotGeometry( + previous: CompanionSkillSlotState, + next: CompanionSkillSlotState, +): boolean { + if (previous.status !== "ready" || next.status !== "ready") return false; + type Rect = Readonly<{ left: number; bottom: number; right: number; top: number }>; + const sameRect = (left: Rect | null, right: Rect | null) => left === null + ? right === null + : right !== null + && left.left === right.left + && left.bottom === right.bottom + && left.right === right.right + && left.top === right.top; + const sameSlots = previous.slots.every((slot, index) => { + const candidate = next.slots[index]; + return candidate !== undefined + && slot.left === candidate.left + && slot.bottom === candidate.bottom + && slot.right === candidate.right + && slot.top === candidate.top; + }); + return previous.frameId === next.frameId + && previous.viewportWidth === next.viewportWidth + && previous.viewportHeight === next.viewportHeight + && previous.chatFrameId === next.chatFrameId + && sameRect(previous.chatInput, next.chatInput) + && sameSlots; +} diff --git a/src/renderer/companion-observer.ts b/src/renderer/companion-observer.ts index 57f334f2..bdc583ea 100644 --- a/src/renderer/companion-observer.ts +++ b/src/renderer/companion-observer.ts @@ -18,9 +18,10 @@ import { type PublishedCompanionState, } from "./companion-snapshot.js"; import { - type CompanionSkillCooldownState, + readCompanionSkillSlots, type CompanionSkillSlotState, -} from "./companion-skill-snapshot.js"; +} from "./companion-interface-geometry-snapshot.js"; +import type { CompanionSkillCooldownState } from "./companion-skill-snapshot.js"; import { type CompanionPlayRegionState, readCompanionPlayRegion, @@ -200,8 +201,7 @@ export function observeCompanion( if (skillSlots && skillSlots.enabled?.() === false) { skillSlots.inactive?.(); } else if (skillSlots) { - if (!readers) throw new Error("Tools skill readers are unavailable"); - const state = readers.readCompanionSkillSlots( + const state = readCompanionSkillSlots( runtime.memory.buffer, runtime.skillSlotPointer ?? 0, ); diff --git a/src/renderer/companion-skill-snapshot.ts b/src/renderer/companion-skill-snapshot.ts index 1eb18e74..34f9445b 100644 --- a/src/renderer/companion-skill-snapshot.ts +++ b/src/renderer/companion-skill-snapshot.ts @@ -1,124 +1,10 @@ /** - * Decodes the bounded skill-slot records published by the companion kernel. - * Skill geometry is independent from key labels so every skill-bar - * presentation can consume the same certified, fail-closed projection. + * Decodes the optional Tools cooldown region published by the kernel. + * Core interface geometry deliberately lives in its own dependency graph. */ -import { - COMPANION_ABI, - SKILL_GEOMETRY_NATIVE_REASONS, -} from "../shared/companion-abi.js"; +import { COMPANION_ABI } from "../shared/companion-abi.js"; import { MAX_SKILL_COOLDOWN_MS } from "../shared/skill-cooldowns.js"; -export const COMPANION_SKILL_SLOT_ABI = COMPANION_ABI.skillSlots.abi; -export const COMPANION_SKILL_SLOT_BYTES = COMPANION_ABI.skillSlots.bytes; - -const SKILL_SLOT_MAGIC = 0x534b5747; -const SKILL_SLOT_READY = 1; - -/** Decode the bounded frame projection; presentation remains app-owned. */ -export function readCompanionSkillSlots(buffer: ArrayBuffer, pointer: number) { - if ( - !(buffer instanceof ArrayBuffer) - || !Number.isInteger(pointer) - || pointer < 0 - || pointer + COMPANION_SKILL_SLOT_BYTES > buffer.byteLength - ) return Object.freeze({ status: "waiting", reason: "memory" } as const); - const view = new DataView(buffer, pointer, COMPANION_SKILL_SLOT_BYTES); - const firstSequence = view.getUint32(8, true); - if ((firstSequence & 1) !== 0) { - return Object.freeze({ status: "waiting", reason: "writing" } as const); - } - const magic = view.getUint32(0, true); - const abi = view.getUint16(4, true); - const bytes = view.getUint16(6, true); - const flags = view.getUint32(12, true); - const frameId = view.getUint32(16, true); - const outcome = view.getUint32(20, true); - const candidateCount = view.getUint32(24, true); - const viewportWidth = view.getFloat32(28, true); - const viewportHeight = view.getFloat32(32, true); - const slots = Object.freeze(Array.from({ length: 8 }, (_, index) => { - const at = 36 + index * 16; - return Object.freeze({ - left: view.getFloat32(at, true), - bottom: view.getFloat32(at + 4, true), - right: view.getFloat32(at + 8, true), - top: view.getFloat32(at + 12, true), - }); - })); - const secondSequence = view.getUint32(8, true); - if ( - magic !== SKILL_SLOT_MAGIC - || abi !== COMPANION_SKILL_SLOT_ABI - || bytes !== COMPANION_SKILL_SLOT_BYTES - || firstSequence !== secondSequence - || (secondSequence & 1) !== 0 - || (flags & ~SKILL_SLOT_READY) !== 0 - ) return Object.freeze({ status: "waiting", reason: "snapshot" } as const); - if ((flags & SKILL_SLOT_READY) === 0) { - const reason = SKILL_GEOMETRY_NATIVE_REASONS[ - outcome as keyof typeof SKILL_GEOMETRY_NATIVE_REASONS - ]; - if ( - reason === undefined - || (reason === "slot-ambiguous" - ? candidateCount < 2 || candidateCount > 16_384 - : candidateCount !== 0) - ) { - return Object.freeze({ status: "waiting", reason: "snapshot" } as const); - } - return reason === "slot-ambiguous" - ? Object.freeze({ status: "waiting" as const, reason, candidateCount }) - : Object.freeze({ status: "waiting" as const, reason }); - } - const finite = (value: number) => Number.isFinite(value) && Math.abs(value) <= 32_768; - if ( - outcome !== 0 - || candidateCount !== 0 - || frameId === 0 - || !finite(viewportWidth) - || !finite(viewportHeight) - || viewportWidth <= 0 - || viewportHeight <= 0 - || slots.some(({ left, bottom, right, top }) => - ![left, bottom, right, top].every(finite) - || right <= left || top <= bottom - || right <= 0 || top <= 0 - || left >= viewportWidth || bottom >= viewportHeight) - ) return Object.freeze({ status: "waiting", reason: "corrupt" } as const); - return Object.freeze({ - status: "ready" as const, - sequence: secondSequence, - frameId, - viewportWidth, - viewportHeight, - slots, - }); -} - -export type CompanionSkillSlotState = - | ReturnType - | Readonly<{ status: "waiting"; reason: "stale" }>; - -/** Ignore heartbeat-only sequence changes while preserving moved-bar updates. */ -export function sameCompanionSkillSlotGeometry( - previous: CompanionSkillSlotState, - next: CompanionSkillSlotState, -): boolean { - if (previous.status !== "ready" || next.status !== "ready") return false; - return previous.frameId === next.frameId - && previous.viewportWidth === next.viewportWidth - && previous.viewportHeight === next.viewportHeight - && previous.slots.every((slot, index) => { - const candidate = next.slots[index]; - return candidate !== undefined - && slot.left === candidate.left - && slot.bottom === candidate.bottom - && slot.right === candidate.right - && slot.top === candidate.top; - }); -} - export const COMPANION_SKILL_COOLDOWN_ABI = COMPANION_ABI.skillCooldowns.abi; export const COMPANION_SKILL_COOLDOWN_BYTES = COMPANION_ABI.skillCooldowns.bytes; diff --git a/src/renderer/companion-tools-observer-readers.ts b/src/renderer/companion-tools-observer-readers.ts index 5d5a2b17..077668b7 100644 --- a/src/renderer/companion-tools-observer-readers.ts +++ b/src/renderer/companion-tools-observer-readers.ts @@ -10,5 +10,4 @@ export { } from "./companion-snapshot.js"; export { readCompanionSkillCooldowns, - readCompanionSkillSlots, } from "./companion-skill-snapshot.js"; diff --git a/src/renderer/dictation.ts b/src/renderer/dictation.ts new file mode 100644 index 00000000..c28478c5 --- /dev/null +++ b/src/renderer/dictation.ts @@ -0,0 +1,220 @@ +/** + * The player-facing dictation control beside Guild Wars' hidden text proxies. + * Apple owns recognition; this module owns eligibility, visible state, and the + * final explicit insertion into the still-active draft. It never sends chat. + */ + +import type { DictationEvent } from '../shared/contracts.js'; +import type { CompanionSkillSlotState } from './companion-interface-geometry-snapshot.js'; +import { projectGameRect } from './skill-slot-projection.js'; + +export interface DictationControl { + setEnabled(enabled: boolean): void; + dispose(): void; +} + +export function dictationPlacement( + chat: Readonly<{ x: number; y: number; width: number; height: number }>, + viewport: Readonly<{ width: number; height: number }>, +) { + const size = 36; + const gap = 8; + return Object.freeze({ + left: Math.min(viewport.width - size - gap, chat.x + chat.width + gap), + top: Math.max(gap, Math.min( + viewport.height - size - gap, + chat.y + (chat.height - size) / 2, + )), + }); +} + +export function installDictation(document: Document): DictationControl { + const control = document.getElementById('dictation-control'); + const button = document.getElementById('dictation-button'); + const status = document.getElementById('dictation-status'); + if (!(control instanceof HTMLElement) + || !(button instanceof HTMLButtonElement) + || !(status instanceof HTMLElement)) { + throw new Error('dictation controls are missing'); + } + + let enabled = false; + let state: 'idle' | 'requesting' | 'preparing' | 'listening' | 'finishing' | 'error' = 'idle'; + let eligibleInput: HTMLInputElement | HTMLTextAreaElement | null = null; + let geometryReady = false; + let geometryState: CompanionSkillSlotState | null = null; + let errorTimer: number | null = null; + + const activeEligibleInput = () => { + const active = document.activeElement; + if (window.Module?.oskActiveInput !== active) return null; + if (active instanceof HTMLTextAreaElement && active.id === 'osk-input-multiline') { + return active; + } + if (active instanceof HTMLInputElement + && active.id === 'osk-input-text' + && active.type === 'text') { + return active; + } + return null; + }; + + const render = (message = 'Dictate') => { + control.hidden = !enabled || eligibleInput === null || !geometryReady; + control.dataset.state = state; + status.textContent = message; + button.disabled = state === 'requesting' || state === 'preparing' || state === 'finishing'; + button.setAttribute( + 'aria-label', + state === 'listening' ? 'Finish dictation' : 'Start dictation', + ); + }; + + const reset = () => { + state = 'idle'; + if (errorTimer !== null) window.clearTimeout(errorTimer); + errorTimer = null; + render(); + }; + + const cancel = () => { + if (state === 'idle') return; + void window.gwNative.dictation.cancel(); + reset(); + }; + + const syncFocus = () => { + const next = activeEligibleInput(); + if (next !== eligibleInput) cancel(); + eligibleInput = next; + render(); + }; + + const onEvent = (event: DictationEvent) => { + const input = eligibleInput; + if (!enabled || input === null || input !== activeEligibleInput()) { + cancel(); + return; + } + if (event.state === 'requesting') { + state = 'requesting'; + render('Allow microphone access…'); + return; + } + if (event.state === 'preparing') { + state = 'preparing'; + const percentage = event.progress === undefined + ? null + : Math.round(Math.max(0, Math.min(1, event.progress)) * 100); + render(percentage === null + ? 'Preparing on-device dictation…' + : `Downloading on-device dictation… ${percentage}%`); + return; + } + if (event.state === 'listening') { + state = 'listening'; + render(event.transcript || 'Listening…'); + return; + } + if (event.state === 'final') { + reset(); + return; + } + state = 'error'; + const errorMessage = event.reason === 'permission-denied' + ? 'Allow Microphone and Speech Recognition in System Settings' + : event.reason === 'audio-unavailable' + ? 'No microphone is available' + : event.reason === 'model-unavailable' + ? 'No on-device model is available for this language' + : event.reason === 'model-download-failed' + ? 'Could not download the on-device dictation model' + : event.reason === 'setup-required' + ? 'Finish dictation setup in Settings' + : event.reason === 'unavailable' + ? 'On-device dictation is unavailable' + : event.reason === 'insertion-failed' + ? 'Could not add dictation to chat' + : 'Dictation could not understand that'; + render(errorMessage); + errorTimer = window.setTimeout(reset, 6_000); + }; + + const updateGeometry = (state: CompanionSkillSlotState) => { + const canvas = document.getElementById('canvas'); + const projected = state.status === 'ready' + && canvas instanceof HTMLCanvasElement + && state.chatInput != null + ? projectGameRect( + state.chatInput, + { width: state.viewportWidth, height: state.viewportHeight }, + canvas, + ) + : null; + geometryReady = projected !== null; + if (projected !== null) { + const placement = dictationPlacement(projected, { + width: window.innerWidth, + height: window.innerHeight, + }); + control.style.left = `${placement.left}px`; + control.style.top = `${placement.top}px`; + } + render(); + }; + + const geometryListener = (event: WindowEventMap['gwonmac:chat-geometry']) => { + geometryState = event.detail; + updateGeometry(geometryState); + }; + const resizeListener = () => { + if (geometryState !== null) updateGeometry(geometryState); + }; + + const removeEventListener = window.gwNative.dictation.onEvent(onEvent); + const focusListener = () => window.setTimeout(syncFocus, 0); + document.addEventListener('focusin', focusListener); + document.addEventListener('focusout', focusListener); + window.addEventListener('blur', cancel); + window.addEventListener('gwonmac:chat-geometry', geometryListener); + window.addEventListener('resize', resizeListener); + + // Keep the hidden Guild Wars proxy focused when the pointer presses the mic. + // Losing focus makes the official client close the text editor. + button.addEventListener('pointerdown', (event) => event.preventDefault()); + button.addEventListener('click', () => { + if (state === 'listening') { + state = 'finishing'; + render('Finishing…'); + void window.gwNative.dictation.finish(); + return; + } + if (state !== 'idle' && state !== 'error') return; + if (errorTimer !== null) window.clearTimeout(errorTimer); + errorTimer = null; + state = 'requesting'; + render('Requesting access…'); + void window.gwNative.dictation.start().catch(() => { + onEvent({ state: 'error', reason: 'unavailable' }); + }); + }); + + return { + setEnabled(next) { + if (enabled === next) return; + enabled = next; + if (!enabled) cancel(); + syncFocus(); + }, + dispose() { + cancel(); + removeEventListener(); + document.removeEventListener('focusin', focusListener); + document.removeEventListener('focusout', focusListener); + window.removeEventListener('blur', cancel); + window.removeEventListener('gwonmac:chat-geometry', geometryListener); + window.removeEventListener('resize', resizeListener); + if (errorTimer !== null) window.clearTimeout(errorTimer); + }, + }; +} diff --git a/src/renderer/gw-native.d.ts b/src/renderer/gw-native.d.ts index f7ea9b70..040df980 100644 --- a/src/renderer/gw-native.d.ts +++ b/src/renderer/gw-native.d.ts @@ -33,6 +33,7 @@ import type { PathingSpikeController, WorldMapAnchorSpikeController, } from "../shared/cartography-spike.js"; +import type { CompanionSkillSlotState } from "./companion-interface-geometry-snapshot.js"; import type { InputTrace as SharedInputTrace, InputTraceEntry as SharedInputTraceEntry, @@ -56,6 +57,9 @@ export interface AppSettingsNegativeTypeTest extends AppSettings { } declare global { + interface WindowEventMap { + "gwonmac:chat-geometry": CustomEvent; + } interface GameInputDiagnostics { event(name: string, value?: unknown): void; } @@ -240,11 +244,13 @@ declare global { }; Module?: { canvas?: HTMLCanvasElement & { offscreen?: OffscreenCanvas }; + oskActiveInput?: EventTarget | null; }; gwApplySettings?(settings: AppSettings): void; gwCartographyGridStats?(): CartographyGridStats; gwSurfaces: GwonmacSurfaceController; gwToolsSettings(): Readonly<{ + dictationEnabled?: boolean; gwonmacTools: boolean; buildLibrary: boolean; tradeChat: boolean; diff --git a/src/renderer/harness.css b/src/renderer/harness.css index 16de9068..b598ae57 100644 --- a/src/renderer/harness.css +++ b/src/renderer/harness.css @@ -226,3 +226,86 @@ html,body { width:100%; height:100%; margin:0; overflow:hidden; .osk-input { width:80%; font-size:1.5rem; color:#fff; background:#404040; border:2px solid #666; border-radius:4px; outline:none; resize:none; } .osk-input:focus { outline:none; border-width:4px; } + +.dictation-control { + position: fixed; + left: 0; + top: 0; + z-index: 12; + width: 36px; + height: 36px; + font: var(--ui-font-size-sm)/1.35 var(--ui-font-readable); +} +.dictation-control[hidden] { display: none; } +#dictation-button { + display: grid; + width: 36px; + height: 36px; + place-items: center; + padding: 0; + border: 1px solid rgba(255, 255, 255, .34); + border-radius: 50%; + color: #fff; + background: #0a84ff; + box-shadow: inset 0 1px 0 rgba(255, 255, 255, .28), + 0 5px 16px rgba(0, 32, 74, .44); + cursor: pointer; + transition: filter 140ms ease-out, transform 140ms var(--spring), + box-shadow 180ms ease-out; +} +#dictation-button svg { + width: 18px; + height: 18px; + fill: none; + stroke: currentColor; + stroke-width: 1.8; + stroke-linecap: round; + stroke-linejoin: round; +} +#dictation-button:hover { filter: brightness(1.1); } +#dictation-button:active { transform: scale(.94); } +#dictation-button:focus-visible { outline: 2px solid var(--ui-focus); outline-offset: 3px; } +#dictation-button:disabled { cursor: wait; opacity: .72; } +#dictation-status { + position: absolute; + right: 0; + bottom: 44px; + width: max-content; + max-width: min(42ch, calc(100vw - 24px)); + box-sizing: border-box; + padding: 7px 11px; + overflow: hidden; + border: 1px solid rgba(132, 197, 255, .34); + border-radius: 10px; + color: #fff; + background: #07559f; + box-shadow: 0 5px 18px rgba(0, 24, 54, .42); + text-overflow: ellipsis; + white-space: nowrap; + transform: translateY(0); + opacity: 1; + transition: opacity 140ms ease-out, transform 180ms var(--spring); + pointer-events: none; +} +.dictation-control[data-state="idle"] #dictation-status { + transform: translateY(4px); + opacity: 0; +} +.dictation-control[data-state="listening"] #dictation-button { + box-shadow: inset 0 1px 0 rgba(255, 255, 255, .28), + 0 5px 16px rgba(0, 32, 74, .44), + 0 0 0 3px rgba(10, 132, 255, .28); + animation: dictation-pulse 1.6s ease-in-out infinite; +} +@keyframes dictation-pulse { + 50% { box-shadow: inset 0 1px 0 rgba(255, 255, 255, .28), + 0 5px 16px rgba(0, 32, 74, .44), + 0 0 0 7px rgba(10, 132, 255, .16); } +} +@media (prefers-reduced-motion: reduce) { + #dictation-button, #dictation-status { transition: none; } + .dictation-control[data-state="listening"] #dictation-button { animation: none; } +} +@media (prefers-contrast: more) { + #dictation-button, #dictation-status { border-color: #fff; } +} diff --git a/src/renderer/harness.ts b/src/renderer/harness.ts index 4579b20e..650c0ffa 100644 --- a/src/renderer/harness.ts +++ b/src/renderer/harness.ts @@ -495,6 +495,7 @@ const STARTUP_LABELS = { const SNAPSHOT_URL = 'Gw.snapshot'; let appSettings: import('../shared/contracts.js').AppSettings | null = null; +let dictationControl: import('./dictation.js').DictationControl | null = null; let clientResizeFrame = 0; function currentRenderScale(): import('../shared/contracts.js').AppSettings['renderScale'] { @@ -528,6 +529,7 @@ const emptySkillKeyBindings: import('../shared/contracts.js').AppSettings['skillKeyBindings'] = [null, null, null, null, null, null, null, null]; window.gwToolsSettings = () => Object.freeze({ + dictationEnabled: appSettings?.dictationEnabled ?? false, gwonmacTools: appSettings?.gwonmacTools ?? false, buildLibrary: appSettings?.buildLibrary ?? true, tradeChat: appSettings?.tradeChat ?? true, @@ -551,6 +553,7 @@ window.gwApplySettings = (next) => { } window.gwDiagnostics?.setVisible(updated.showDiagnostics); applyAppearance(updated); + dictationControl?.setEnabled(updated.dictationEnabled); window.dispatchEvent(new CustomEvent('gw:tools-settings', { detail: window.gwToolsSettings(), })); @@ -1377,6 +1380,9 @@ function loadGlue(isProxyRouteLabel: (route: string) => boolean) { native().client.session(), ]); appSettings = settings; + const { installDictation } = await import('./dictation.js'); + dictationControl = installDictation(document); + dictationControl.setEnabled(settings.dictationEnabled); if (settings.controllerPromptStyle === 'playstation') { try { controllerPrompts = await host.preparePlayStationControllerPrompts({ diff --git a/src/renderer/index.html b/src/renderer/index.html index ba7eaa3d..163cde80 100644 --- a/src/renderer/index.html +++ b/src/renderer/index.html @@ -147,6 +147,16 @@

autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" writingsuggestions="false"> +