diff --git a/.agents/skills/test-t3-app/SKILL.md b/.agents/skills/test-t3-app/SKILL.md index 9b577fe7c6bc4..81ddbb6a39adf 100644 --- a/.agents/skills/test-t3-app/SKILL.md +++ b/.agents/skills/test-t3-app/SKILL.md @@ -5,12 +5,11 @@ description: Test T3 Code's web and desktop UI through its built-in Browser pane # Test T3 web and desktop -Use T3's built-in Browser panel for verification. If its tools are absent or -the panel reports unavailable, explain the blocker and stop verification. -Do not install or switch to another automation system. The one exception is -screenshots the panel cannot capture, covered in -[references/screenshots.md](references/screenshots.md). For native mobile -testing, use [test-t3-mobile](../test-t3-mobile/SKILL.md). +Use T3's built-in Browser panel for verification. Follow the recovery steps +below before declaring it unavailable. Do not install or switch to another +automation system. The one exception is screenshots the panel cannot capture, covered in +[references/screenshots.md](references/screenshots.md). For native mobile testing, use +[test-t3-mobile](../test-t3-mobile/SKILL.md). For desktop-only Electron behaviour, use [test-t3-desktop](../test-t3-desktop/SKILL.md). @@ -40,9 +39,15 @@ model, such as Haiku for Claude or Luna for Codex. ## Use the Browser panel -Call `preview_status`, then `preview_open` if the Browser panel is -closed. Open localhost dev servers with `profileId: "incognito"`: the default -profile's cookies from other dev servers cause HTTP 431. Navigate to the complete startup pairing URL once with +Call `preview_status`. If no tab can be automated, call `preview_open`, including +when status reports `available: false` or `tabId: null`. Status can report +unavailable simply because no tab exists yet; opening a tab needs no user action. +Stop verification and explain the blocker only if the tools are absent or +`preview_open` reports unavailable or fails. + +Open localhost dev servers with `profileId: "incognito"`: the default +profile's cookies from other dev servers cause HTTP 431. +Navigate to the complete startup pairing URL once with `preview_navigate`, then use `preview_snapshot` and T3's interaction tools. If the token was consumed or expired, run `node apps/server/src/bin.ts pair` for a fresh one. Keep using the same tab. diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 68321456683eb..729690b88c989 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -22,7 +22,7 @@ "@t3tools/tailscale": "workspace:*", "dbus-next": "0.10.2", "effect": "catalog:", - "electron": "44.10.0", + "electron": "44.10.1", "electron-store": "^8.2.0", "electron-updater": "^6.8.9", "electron-webauthn": "catalog:", diff --git a/apps/desktop/src/app/DesktopStartupRecovery.test.ts b/apps/desktop/src/app/DesktopStartupRecovery.test.ts index 12d5289af67c7..029bcaac2b487 100644 --- a/apps/desktop/src/app/DesktopStartupRecovery.test.ts +++ b/apps/desktop/src/app/DesktopStartupRecovery.test.ts @@ -74,6 +74,19 @@ describe("resolveStartupRecoveryOffer", () => { ).toEqual({ _tag: "Reinstall", version: "0.1.2" }); }); + it("blames the managed policy unless a newer version can read it", () => { + expect( + resolveStartupRecoveryOffer( + updateState({ status: "available", availableVersion: "0.1.4" }), + "0.1.2", + true, + ), + ).toEqual({ _tag: "Update", version: "0.1.4" }); + expect( + resolveStartupRecoveryOffer(updateState({ status: "up-to-date" }), "0.1.2", true), + ).toEqual({ _tag: "ManagedPolicy" }); + }); + it("falls back to the releases page when this version never started", () => { expect(resolveStartupRecoveryOffer(updateState({ status: "disabled" }), "0.1.3")).toEqual({ _tag: "Releases", @@ -168,6 +181,7 @@ it.effect.each([ yield* SubscriptionRef.set(state.primaryBackendStartup, { _tag: "Failing", reason: "Server failed before readiness", + managedPolicy: false, output: "Failure details", }); yield* Deferred.await(checked); diff --git a/apps/desktop/src/app/DesktopStartupRecovery.ts b/apps/desktop/src/app/DesktopStartupRecovery.ts index ae94fd5b4c135..c8228183aad91 100644 --- a/apps/desktop/src/app/DesktopStartupRecovery.ts +++ b/apps/desktop/src/app/DesktopStartupRecovery.ts @@ -32,7 +32,8 @@ export type StartupRecoveryOffer = | { readonly _tag: "Update"; readonly version: string } | { readonly _tag: "Downloading"; readonly version: string; readonly percent: number | null } | { readonly _tag: "Reinstall"; readonly version: string } - | { readonly _tag: "Releases" }; + | { readonly _tag: "Releases" } + | { readonly _tag: "ManagedPolicy" }; const STARTUP_RECOVERY_ACTIONS = ["update", "reinstall", "releases", "copy", "quit"] as const; type StartupRecoveryAction = (typeof STARTUP_RECOVERY_ACTIONS)[number]; @@ -41,6 +42,7 @@ type StartupRecoveryAction = (typeof STARTUP_RECOVERY_ACTIONS)[number]; export function resolveStartupRecoveryOffer( update: DesktopUpdateState, lastStartedVersion: string | null, + managedPolicy = false, ): StartupRecoveryOffer { if (update.status === "downloading" && update.availableVersion !== null) { return { @@ -55,6 +57,8 @@ export function resolveStartupRecoveryOffer( if (updateVersion !== null && updateVersion !== update.notAppliedVersion) { return { _tag: "Update", version: updateVersion }; } + // A newer release can know a key this one refuses; an older one or the releases page cannot help. + if (managedPolicy) return { _tag: "ManagedPolicy" }; if (update.enabled && (update.status === "idle" || update.status === "checking")) { return { _tag: "Checking" }; } @@ -98,6 +102,11 @@ function describeOffer(offer: StartupRecoveryOffer): { text: string; action: str text: "Get a different version from the releases page.", action: actionLink("releases", "Open releases", true), }; + case "ManagedPolicy": + return { + text: "Your organization's managed policy stopped T3 Code from starting. Send the details below to your IT administrator.", + action: "", + }; } } @@ -171,8 +180,14 @@ export const listen: Effect.Effect< const { latest, changes: updateChanges } = yield* updates.subscribe; const updateStateRef = yield* Ref.make(latest); - const offer = Ref.get(updateStateRef).pipe( - Effect.map((state) => resolveStartupRecoveryOffer(state, lastStartedVersion)), + const offer = Effect.all([Ref.get(updateStateRef), Ref.get(failureRef)]).pipe( + Effect.map(([state, failure]) => + resolveStartupRecoveryOffer( + state, + lastStartedVersion, + Option.exists(failure, ({ managedPolicy }) => managedPolicy), + ), + ), ); const render = Effect.gen(function* () { @@ -332,7 +347,8 @@ export const listen: Effect.Effect< } if (startup._tag !== "Failing") return; if (Option.isSome(yield* Ref.get(failureRef))) return; - yield* Ref.set(failureRef, Option.some({ reason: startup.reason, output: startup.output })); + const { _tag, ...failure } = startup; + yield* Ref.set(failureRef, Option.some(failure)); // The backend manager already logs each failure's reason. yield* logWarning("primary backend failing to start"); yield* show(); diff --git a/apps/desktop/src/backend/DesktopBackendManager.test.ts b/apps/desktop/src/backend/DesktopBackendManager.test.ts index a5c12043d9aaa..e4714673777d9 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.test.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.test.ts @@ -1,4 +1,5 @@ import { + SERVER_EXIT_CODE_MANAGED_POLICY, SERVER_EXIT_CODE_STATE_DIR_OWNED, DesktopBackendBootstrap, type DesktopBackendBootstrap as DesktopBackendBootstrapValue, @@ -623,56 +624,63 @@ describe("DesktopBackendManager", () => { ), ); - it.effect("reports a crash loop with the server output on the third restart", () => - Effect.scoped( - Effect.gen(function* () { - const message = "Error: Cannot find module 'zod/v4' in /Users/René"; - let exitCount = 0; - const firstExit = yield* Deferred.make(); - const reports = yield* Queue.unbounded<{ - readonly exitCount: number; - readonly failure: DesktopBackendManager.BackendStartupFailure; - }>(); - const instance = yield* makeTestInstance({ - spawnerLayer: Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.succeed( - makeProcess({ - // "é" split across chunks must not decode as U+FFFD, and the 4,000-char cut - // lands inside the emoji's surrogate pair. - stderr: Stream.make( - new TextEncoder().encode( - `😀${"x".repeat(3_999 - message.length)}${message.slice(0, -1)}`, + // A policy refusal is reported on the first exit, since retrying cannot fix it. + it.effect.each([ + { code: 1, exits: 3, managedPolicy: false }, + { code: SERVER_EXIT_CODE_MANAGED_POLICY, exits: 1, managedPolicy: true }, + ])( + "reports exit $code with the server output after $exits exits", + ({ code, exits, managedPolicy }) => + Effect.scoped( + Effect.gen(function* () { + const message = "Error: Cannot find module 'zod/v4' in /Users/René"; + let exitCount = 0; + const firstExit = yield* Deferred.make(); + const reports = yield* Queue.unbounded<{ + readonly exitCount: number; + readonly failure: DesktopBackendManager.BackendStartupFailure; + }>(); + const instance = yield* makeTestInstance({ + spawnerLayer: Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.succeed( + makeProcess({ + // "é" split across chunks must not decode as U+FFFD, and the 4,000-char cut + // lands inside the emoji's surrogate pair. + stderr: Stream.make( + new TextEncoder().encode( + `😀${"x".repeat(3_999 - message.length)}${message.slice(0, -1)}`, + ), + new Uint8Array([0xc3]), + new Uint8Array([0xa9]), ), - new Uint8Array([0xc3]), - new Uint8Array([0xa9]), - ), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(1)), - }), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(code)), + }), + ), ), ), - ), - httpClientLayer: layerHttpClient(() => Effect.never), - backendOutputLog: { - persistFailure: () => { - exitCount += 1; - return Deferred.succeed(firstExit, undefined).pipe(Effect.asVoid); + httpClientLayer: layerHttpClient(() => Effect.never), + backendOutputLog: { + persistFailure: () => { + exitCount += 1; + return Deferred.succeed(firstExit, undefined).pipe(Effect.asVoid); + }, }, - }, - onStartupFailure: (failure) => - Queue.offer(reports, { exitCount, failure }).pipe(Effect.asVoid), - }); - yield* instance.start; - yield* Deferred.await(firstExit); - yield* TestClock.adjust(Duration.minutes(1)); - const { exitCount: exitsWhenReported, failure } = yield* Queue.take(reports); - assert.equal(exitsWhenReported, 3); - assert.equal(failure.reason, "The server could not start (code=1)."); - assert.isTrue(failure.output.endsWith(message)); - assert.isTrue(failure.output.isWellFormed()); - }).pipe(Effect.provide(TestClock.layer())), - ), + onStartupFailure: (failure) => + Queue.offer(reports, { exitCount, failure }).pipe(Effect.asVoid), + }); + yield* instance.start; + yield* Deferred.await(firstExit); + yield* TestClock.adjust(Duration.minutes(1)); + const { exitCount: exitsWhenReported, failure } = yield* Queue.take(reports); + assert.equal(exitsWhenReported, exits); + assert.equal(failure.reason, `The server could not start (code=${code}).`); + assert.equal(failure.managedPolicy, managedPolicy); + assert.isTrue(failure.output.endsWith(message)); + assert.isTrue(failure.output.isWellFormed()); + }).pipe(Effect.provide(TestClock.layer())), + ), ); it.effect("waits out a slow startup and reports one that never becomes ready", () => diff --git a/apps/desktop/src/backend/DesktopBackendManager.ts b/apps/desktop/src/backend/DesktopBackendManager.ts index aa3b4d59ef1bf..2f5b57c4e3c95 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.ts @@ -17,6 +17,7 @@ import { HttpClient } from "effect/http"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { + SERVER_EXIT_CODE_MANAGED_POLICY, SERVER_EXIT_CODE_STATE_DIR_OWNED, DesktopBackendBootstrap, type DesktopBackendBootstrap as DesktopBackendBootstrapValue, @@ -223,6 +224,8 @@ export const PRIMARY_INSTANCE_ID: BackendInstanceId = BackendInstanceId( export interface BackendStartupFailure { readonly reason: string; + /** The server refused to start under this machine's managed policy. */ + readonly managedPolicy: boolean; /** Tail of the last run's stdout and stderr. */ readonly output: string; } @@ -635,9 +638,11 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( // One streaming decoder per stream keeps UTF-8 characters split across chunks intact. const makeOutputDecoders = () => ({ stdout: new TextDecoder(), stderr: new TextDecoder() }); let outputDecoders = makeOutputDecoders(); - const reportStartupFailure = (reason: string) => + const reportStartupFailure = (reason: string, managedPolicy = false) => Ref.get(outputTail).pipe( - Effect.flatMap((output) => spec.onStartupFailure?.({ reason, output }) ?? Effect.void), + Effect.flatMap( + (output) => spec.onStartupFailure?.({ reason, managedPolicy, output }) ?? Effect.void, + ), ); const { logWarning: logInstanceWarning, logError: logInstanceError } = @@ -880,7 +885,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( yield* (spec.onStateDirOwned?.() ?? Effect.void).pipe(Effect.ignore); return; } - yield* scheduleRestart(reason); + yield* scheduleRestart(reason, exitCode); } }), ); @@ -980,6 +985,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( const scheduleRestart = Effect.fn("desktop.backendInstance.scheduleRestart")(function* ( reason: string, + exitCode?: number, ) { const scheduled = yield* SubscriptionRef.modify(state, (latest) => { if (!latest.desiredRunning || Option.isSome(latest.restartFiber)) { @@ -1003,8 +1009,10 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( reason, delayMs: Duration.toMillis(delay), }); - if (attempt >= STARTUP_FAILURE_RESTART_ATTEMPTS) { - yield* reportStartupFailure(`The server could not start (${reason}).`); + // Retrying cannot fix a policy refusal, so say so at once; retries still pick up a fix. + const managedPolicy = exitCode === SERVER_EXIT_CODE_MANAGED_POLICY; + if (managedPolicy || attempt >= STARTUP_FAILURE_RESTART_ATTEMPTS) { + yield* reportStartupFailure(`The server could not start (${reason}).`, managedPolicy); } const restartFiber = yield* Effect.forkIn( Effect.sleep(delay).pipe( diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index d917af72d389b..32b9ad23f9c93 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -615,6 +615,33 @@ describe("PreviewManager", () => { ), ); + effectIt.effect.each([ + ["darwin", true, false, true], + ["darwin", false, true, false], + ["linux", false, true, true], + ["win32", true, false, false], + ] as const)( + "reloads the preview only on the platform refresh chord: %s meta=%s ctrl=%s", + ([platform, meta, control, expected]) => + withManager( + (manager) => + Effect.gen(function* () { + const preview = makeFaviconWebContents(); + fromId.mockReturnValue(preview.webContents); + yield* manager.createTab("tab_refresh"); + yield* manager.registerWebview("tab_refresh", 42); + const preventDefault = vi.fn(); + preview.listeners.get("before-input-event")!( + { preventDefault } as never, + { type: "keyDown", key: "r", meta, control, shift: false, alt: false } as never, + ); + expect(preventDefault).toHaveBeenCalledTimes(expected ? 1 : 0); + expect(preview.reload).toHaveBeenCalledTimes(expected ? 1 : 0); + }), + platform, + ), + ); + effectIt.effect("keeps editing and forwarded shortcuts in whichever tab holds focus", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index c83eae8d8358c..18eb4b2b75968 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -464,10 +464,10 @@ export const previewWindowOpenAction = (details: { : "navigate"; }; -const isPreviewRefreshShortcut = (input: Electron.Input): boolean => +const isPreviewRefreshShortcut = (input: Electron.Input, platform: NodeJS.Platform): boolean => input.type === "keyDown" && input.key.toLowerCase() === "r" && - (input.meta || input.control) && + (platform === "darwin" ? input.meta && !input.control : input.control && !input.meta) && !input.shift && !input.alt; @@ -1488,7 +1488,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function } return; } - if (isPreviewRefreshShortcut(input)) { + if (isPreviewRefreshShortcut(input, hostPlatform)) { event.preventDefault(); runFork( attempt({ operation: "shortcut.refresh", tabId, webContentsId: wc.id }, () => diff --git a/apps/desktop/src/settings/DesktopAppSettings.test.ts b/apps/desktop/src/settings/DesktopAppSettings.test.ts index ab3e2e4cd0755..7e4b530083097 100644 --- a/apps/desktop/src/settings/DesktopAppSettings.test.ts +++ b/apps/desktop/src/settings/DesktopAppSettings.test.ts @@ -264,6 +264,29 @@ describe("DesktopSettings", () => { ), ); + it.effect("removes its temporary file when replacing the settings file fails", () => + withSettings( + Effect.gen(function* () { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const fileSystem = yield* FileSystem.FileSystem; + const settings = yield* DesktopAppSettings.DesktopAppSettings; + yield* fileSystem.makeDirectory(`${environment.desktopSettingsPath}/occupied`, { + recursive: true, + }); + + const error = yield* settings.setServerExposureMode("network-accessible").pipe(Effect.flip); + + assert.equal(error.operation, "replace-settings-file"); + assert.deepEqual( + (yield* fileSystem.readDirectory(environment.stateDir)).filter((name) => + name.endsWith(".tmp"), + ), + [], + ); + }), + ), + ); + it.effect("migrates legacy implicit update channels to the runtime default", () => withSettings( Effect.gen(function* () { diff --git a/apps/desktop/src/settings/DesktopAppSettings.ts b/apps/desktop/src/settings/DesktopAppSettings.ts index aa16502a16585..6cd34322d98c2 100644 --- a/apps/desktop/src/settings/DesktopAppSettings.ts +++ b/apps/desktop/src/settings/DesktopAppSettings.ts @@ -442,26 +442,28 @@ const writeSettings = Effect.fn("desktop.settings.writeSettings")(function* (inp }), ), ); - yield* input.fileSystem.writeFileString(tempPath, `${encoded}\n`).pipe( - Effect.mapError( - (cause) => - new DesktopSettingsWriteError({ - operation: "write-temporary-file", - path: tempPath, - cause, - }), - ), - ); - yield* input.fileSystem.rename(tempPath, targetPath).pipe( - Effect.mapError( - (cause) => - new DesktopSettingsWriteError({ - operation: "replace-settings-file", - path: input.settingsPath, - cause, - }), - ), - ); + yield* Effect.gen(function* () { + yield* input.fileSystem.writeFileString(tempPath, `${encoded}\n`).pipe( + Effect.mapError( + (cause) => + new DesktopSettingsWriteError({ + operation: "write-temporary-file", + path: tempPath, + cause, + }), + ), + ); + yield* input.fileSystem.rename(tempPath, targetPath).pipe( + Effect.mapError( + (cause) => + new DesktopSettingsWriteError({ + operation: "replace-settings-file", + path: input.settingsPath, + cause, + }), + ), + ); + }).pipe(Effect.ensuring(input.fileSystem.remove(tempPath, { force: true }).pipe(Effect.ignore))); }); /** @public */ diff --git a/apps/desktop/src/settings/DesktopClientSettings.test.ts b/apps/desktop/src/settings/DesktopClientSettings.test.ts index 2d89754ac1150..0dab1c8d83cc6 100644 --- a/apps/desktop/src/settings/DesktopClientSettings.test.ts +++ b/apps/desktop/src/settings/DesktopClientSettings.test.ts @@ -144,6 +144,29 @@ describe("DesktopClientSettings", () => { ), ); + it.effect("removes its temporary file when replacing the settings file fails", () => + withClientSettings( + Effect.gen(function* () { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const fileSystem = yield* FileSystem.FileSystem; + const settings = yield* DesktopClientSettings.DesktopClientSettings; + yield* fileSystem.makeDirectory(`${environment.clientSettingsPath}/occupied`, { + recursive: true, + }); + + const error = yield* settings.set(clientSettings).pipe(Effect.flip); + + assert.equal(error.operation, "replace-settings-file"); + assert.deepEqual( + (yield* fileSystem.readDirectory(environment.stateDir)).filter((name) => + name.endsWith(".tmp"), + ), + [], + ); + }), + ), + ); + it.effect.each([ { label: "permission", reason: "PermissionDenied" }, { label: "I/O", reason: "Unknown" }, diff --git a/apps/desktop/src/settings/DesktopClientSettings.ts b/apps/desktop/src/settings/DesktopClientSettings.ts index 52b619494626f..b840bb7a058e3 100644 --- a/apps/desktop/src/settings/DesktopClientSettings.ts +++ b/apps/desktop/src/settings/DesktopClientSettings.ts @@ -162,26 +162,28 @@ const writeClientSettings = Effect.fnUntraced(function* (input: { }), ), ); - yield* input.fileSystem.writeFileString(tempPath, `${encoded}\n`).pipe( - Effect.mapError( - (cause) => - new DesktopClientSettingsWriteError({ - operation: "write-temporary-file", - path: tempPath, - cause, - }), - ), - ); - yield* input.fileSystem.rename(tempPath, targetPath).pipe( - Effect.mapError( - (cause) => - new DesktopClientSettingsWriteError({ - operation: "replace-settings-file", - path: input.settingsPath, - cause, - }), - ), - ); + yield* Effect.gen(function* () { + yield* input.fileSystem.writeFileString(tempPath, `${encoded}\n`).pipe( + Effect.mapError( + (cause) => + new DesktopClientSettingsWriteError({ + operation: "write-temporary-file", + path: tempPath, + cause, + }), + ), + ); + yield* input.fileSystem.rename(tempPath, targetPath).pipe( + Effect.mapError( + (cause) => + new DesktopClientSettingsWriteError({ + operation: "replace-settings-file", + path: input.settingsPath, + cause, + }), + ), + ); + }).pipe(Effect.ensuring(input.fileSystem.remove(tempPath, { force: true }).pipe(Effect.ignore))); }); export const make = Effect.gen(function* () { diff --git a/apps/desktop/src/shell/DesktopShellEnvironment.test.ts b/apps/desktop/src/shell/DesktopShellEnvironment.test.ts index 79344567fcd6c..124dbbeae89e0 100644 --- a/apps/desktop/src/shell/DesktopShellEnvironment.test.ts +++ b/apps/desktop/src/shell/DesktopShellEnvironment.test.ts @@ -3,6 +3,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, it } from "@effect/vitest"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Deferred from "effect/Deferred"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -14,6 +15,7 @@ import * as Schema from "effect/Schema"; import * as Option from "effect/Option"; import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import * as TestClock from "effect/testing/TestClock"; import * as NodeOS from "node:os"; import { vi } from "vite-plus/test"; @@ -60,6 +62,22 @@ function makeProcess(output: string, exitCode = 0): ChildProcessSpawner.ChildPro }); } +function makeHungProcess(): ChildProcessSpawner.ChildProcessHandle { + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(124), + stdout: Stream.never, + stderr: Stream.never, + all: Stream.never, + exitCode: Effect.never, + isRunning: Effect.succeed(true), + kill: () => Effect.never, + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); +} + function withProcessEnv( env: NodeJS.ProcessEnv, effect: Effect.Effect, @@ -673,6 +691,36 @@ describe("DesktopShellEnvironment", () => { }), ); + it.effect("gives up on a Windows shell that survives the kill", () => + Effect.gen(function* () { + const commands: Array = []; + // Both PowerShell probes spawn pwsh, time out, then spawn powershell. Each probe's timeout + // is armed before its spawn runs, so waiting for the spawns keeps the clock in step. + const spawned = yield* Effect.forEach([0, 1, 2, 3], () => Deferred.make()); + const fiber = yield* runShellEnvironment({ + env: { PATH: "C:\\Windows", APPDATA: "C:\\Roaming" }, + platform: "win32", + handler: () => "", + processSpawner: ChildProcessSpawner.make((command) => { + const spawn = spawned[commands.length]; + commands.push(command._tag === "StandardCommand" ? command.command : ""); + return (spawn ? Deferred.succeed(spawn, undefined) : Effect.void).pipe( + Effect.andThen( + Effect.acquireRelease(Effect.succeed(makeHungProcess()), () => Effect.never), + ), + ); + }), + }).pipe(Effect.forkChild({ startImmediately: true })); + + yield* Effect.forEach(spawned.slice(0, 2), Deferred.await); + yield* TestClock.adjust("5 seconds"); + yield* Effect.forEach(spawned.slice(2), Deferred.await); + yield* TestClock.adjust("5 seconds"); + yield* Fiber.join(fiber); + assert.deepEqual([...new Set(commands)], ["pwsh.exe", "powershell.exe"]); + }), + ); + it.effect("replaces a Path-spelled Windows PATH instead of adding a second key", () => Effect.gen(function* () { const env: NodeJS.ProcessEnv = { Path: "C:\\Windows\\System32" }; diff --git a/apps/desktop/src/shell/DesktopShellEnvironment.ts b/apps/desktop/src/shell/DesktopShellEnvironment.ts index 556f7090b422e..37826f8ee5aba 100644 --- a/apps/desktop/src/shell/DesktopShellEnvironment.ts +++ b/apps/desktop/src/shell/DesktopShellEnvironment.ts @@ -8,6 +8,7 @@ import { import * as Context from "effect/Context"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Deferred from "effect/Deferred"; import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; @@ -259,7 +260,7 @@ const runCommandOutput = Effect.fn("desktop.shellEnvironment.runCommandOutput")( readonly shell?: boolean; }): Effect.fn.Return { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const output = yield* Effect.scoped( + const fiber = yield* Effect.scoped( Effect.gen(function* () { const child = yield* spawner.spawn( ChildProcess.make(input.command, input.args, { @@ -302,7 +303,12 @@ const runCommandOutput = Effect.fn("desktop.shellEnvironment.runCommandOutput")( DesktopShellEnvironmentCommandError: (error) => logShellEnvironmentCommandError(error).pipe(Effect.as("")), }), + Effect.forkDetach, + ); + const output = yield* Fiber.join(fiber).pipe( Effect.timeoutOption(input.timeout), + // Interrupting awaits the child's exit, which a shell stuck in the kernel never reaches. + Effect.ensuring(Fiber.interrupt(fiber).pipe(Effect.forkDetach)), ); if (Option.isSome(output)) { return output.value; diff --git a/apps/mobile/app.config.test.ts b/apps/mobile/app.config.test.ts index b8111f45587cf..9be2204d5132c 100644 --- a/apps/mobile/app.config.test.ts +++ b/apps/mobile/app.config.test.ts @@ -1,5 +1,9 @@ +import { DEFAULT_HOSTED_APP_URL } from "@t3tools/shared/connectAuth"; import { afterEach, expect, it, vi } from "vite-plus/test"; +import appleAppSiteAssociation from "../web/public/.well-known/apple-app-site-association.json"; +import assetLinks from "../web/public/.well-known/assetlinks.json"; + // Keep config parsing independent of local env files and build credentials. vi.mock("../../scripts/lib/public-config.ts", () => ({ loadRepoEnv: () => ({ APP_VARIANT: process.env.APP_VARIANT }), @@ -30,4 +34,14 @@ it.each([ const config = await loadConfig(value); expect(config.ios?.bundleIdentifier).toBe(identifier); expect(config.android?.package).toBe(identifier); + // Copied thread links open this app only while app.t3.codes lists it. + const host = new URL(DEFAULT_HOSTED_APP_URL).host; + expect(config.ios?.associatedDomains).toContain(`applinks:${host}`); + expect(appleAppSiteAssociation.applinks.details[0]?.appIDs).toContain( + `${config.ios?.appleTeamId}.${identifier}`, + ); + expect(config.android?.intentFilters?.[0]?.data).toEqual([ + { scheme: "https", host, pathPrefix: "/threads/" }, + ]); + expect(assetLinks.map((link) => link.target.package_name)).toContain(identifier); }); diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index e81afaf62134d..b3416b726ce4f 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -116,6 +116,8 @@ function resolveAppVariant(value: string | undefined): AppVariant { } const variant = VARIANT_CONFIG[APP_VARIANT]; +// Copied thread links open the app. app.t3.codes/.well-known lists every variant's identifiers. +const HOSTED_APP_HOST = "app.t3.codes"; const iosBundleIdentifier = isIosPersonalTeamBuild ? personalTeamBundleIdentifier! : variant.iosBundleIdentifier; @@ -254,6 +256,7 @@ const config: ExpoConfig = { associatedDomains: [ `applinks:${variant.relyingParty}`, `webcredentials:${variant.relyingParty}`, + `applinks:${HOSTED_APP_HOST}`, ], } : {}), @@ -306,6 +309,14 @@ const config: ExpoConfig = { predictiveBackGestureEnabled: true, // expo-sensors declares this for its pedometer, which the app does not use. blockedPermissions: ["android.permission.ACTIVITY_RECOGNITION"], + intentFilters: [ + { + action: "VIEW", + autoVerify: true, + data: [{ scheme: "https", host: HOSTED_APP_HOST, pathPrefix: "/threads/" }], + category: ["BROWSABLE", "DEFAULT"], + }, + ], }, web: { favicon: variant.assets.appIcon, @@ -373,7 +384,7 @@ const config: ExpoConfig = { "expo-audio", { microphonePermission: "Allow T3 Code to use your microphone for voice input.", - recordAudioAndroid: false, + recordAudioAndroid: true, enableBackgroundPlayback: false, enableBackgroundRecording: false, }, @@ -387,7 +398,14 @@ const config: ExpoConfig = { recordAudioAndroid: false, }, ], - ["expo-image-picker", { photosPermission: false, microphonePermission: false }], + [ + "expo-image-picker", + { + photosPermission: false, + // `false` makes this plugin add RECORD_AUDIO to blockedPermissions, breaking Android dictation. + microphonePermission: "Allow T3 Code to use your microphone for voice input.", + }, + ], [ "expo-splash-screen", { diff --git a/apps/mobile/modules/t3-composer-editor/android/src/main/java/expo/modules/t3composereditor/T3ComposerEditorModule.kt b/apps/mobile/modules/t3-composer-editor/android/src/main/java/expo/modules/t3composereditor/T3ComposerEditorModule.kt index 1703a76702024..fd48492a11c9e 100644 --- a/apps/mobile/modules/t3-composer-editor/android/src/main/java/expo/modules/t3composereditor/T3ComposerEditorModule.kt +++ b/apps/mobile/modules/t3-composer-editor/android/src/main/java/expo/modules/t3composereditor/T3ComposerEditorModule.kt @@ -9,6 +9,10 @@ import org.json.JSONObject import org.json.JSONArray internal object T3ComposerClipboard { + // Pasted thread links become chips in JS; packages/shared/src/threadLinks.ts parses them. + fun hasThreadLink(text: String): Boolean = + text.contains("t3-thread://v2/") || text.contains("https://app.t3.codes/threads/") + fun write(context: Context, text: String, fragment: String) { val clipboard = context.getSystemService(Context.CLIPBOARD_SERVICE) as ClipboardManager val payload = try { diff --git a/apps/mobile/modules/t3-composer-editor/android/src/main/java/expo/modules/t3composereditor/T3ComposerEditorView.kt b/apps/mobile/modules/t3-composer-editor/android/src/main/java/expo/modules/t3composereditor/T3ComposerEditorView.kt index efa54e2ec374a..6b0534fdaccbd 100644 --- a/apps/mobile/modules/t3-composer-editor/android/src/main/java/expo/modules/t3composereditor/T3ComposerEditorView.kt +++ b/apps/mobile/modules/t3-composer-editor/android/src/main/java/expo/modules/t3composereditor/T3ComposerEditorView.kt @@ -735,11 +735,24 @@ internal class SelectionAwareEditText( private fun pasteContextOrImages(): Boolean { val payload = T3ComposerClipboard.read(context) - if (payload["html"]?.contains("data-t3-context-fragment=") == true) { - pasteContextListener?.invoke(payload) - return true + return when { + payload["html"]?.contains("data-t3-context-fragment=") == true -> pasteContext(payload) + pasteImagesOrInterceptedText() -> true + isSingleItemClip() && T3ComposerClipboard.hasThreadLink(payload["text"].orEmpty()) -> + pasteContext(payload) + else -> false } - return pasteImagesOrInterceptedText() + } + + // read() sees only the first item; native paste keeps every item of a longer clip. + private fun isSingleItemClip(): Boolean { + val clipboard = context.getSystemService(Context.CLIPBOARD_SERVICE) as? ClipboardManager + return clipboard?.primaryClip?.itemCount == 1 + } + + private fun pasteContext(payload: Map): Boolean { + pasteContextListener?.invoke(payload) + return true } private fun pasteImagesOrInterceptedText(): Boolean { diff --git a/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorModule.swift b/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorModule.swift index 99504e6de8936..e16582ce2b514 100644 --- a/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorModule.swift +++ b/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorModule.swift @@ -4,6 +4,11 @@ import UIKit enum T3ComposerClipboard { static let fragmentType = "app.t3.context-fragment" + // Pasted thread links become chips in JS; packages/shared/src/threadLinks.ts parses them. + static func hasThreadLink(_ text: String) -> Bool { + text.contains("t3-thread://v2/") || text.contains("https://app.t3.codes/threads/") + } + static func write(text: String, fragment: String) { var items: [String: Any] = ["public.utf8-plain-text": text] if let data = fragment.data(using: .utf8), diff --git a/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift b/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift index f54f75caf5f50..188ab4a97b316 100644 --- a/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift +++ b/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift @@ -226,6 +226,12 @@ private final class ComposerTextView: UITextView { onPasteText?(text, selectedRange) return } + // read() sees only the first item; native paste keeps every item of a longer pasteboard. + if !bypassTextPasteInterception, pasteboard.numberOfItems == 1, + T3ComposerClipboard.hasThreadLink(context["text", default: ""]) { + onPasteContext?(context) + return + } super.paste(sender) } diff --git a/apps/mobile/modules/t3-voice-transcription/android/build.gradle b/apps/mobile/modules/t3-voice-transcription/android/build.gradle new file mode 100644 index 0000000000000..05995a7cc4cce --- /dev/null +++ b/apps/mobile/modules/t3-voice-transcription/android/build.gradle @@ -0,0 +1,31 @@ +apply plugin: 'com.android.library' +apply plugin: 'org.jetbrains.kotlin.android' + +group = 'com.t3tools.voicetranscription' +version = '0.0.0' + +android { + namespace 'expo.modules.t3voicetranscription' + compileSdk rootProject.ext.compileSdkVersion + + defaultConfig { + minSdkVersion rootProject.ext.minSdkVersion + targetSdkVersion rootProject.ext.targetSdkVersion + } +} + +dependencies { + implementation project(':expo-modules-core') + implementation 'androidx.media3:media3-common:1.9.1' + implementation 'com.google.mlkit:genai-speech-recognition:1.0.0-alpha2' + testImplementation 'junit:junit:4.13.2' + testImplementation 'org.robolectric:robolectric:4.16.1' + testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2' +} + +tasks.withType(Test).configureEach { + systemProperty 'robolectric.dependency.repo.url', 'https://repo.maven.apache.org/maven2' + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(21) + } +} diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/main/AndroidManifest.xml b/apps/mobile/modules/t3-voice-transcription/android/src/main/AndroidManifest.xml new file mode 100644 index 0000000000000..05bc93c9a7d5c --- /dev/null +++ b/apps/mobile/modules/t3-voice-transcription/android/src/main/AndroidManifest.xml @@ -0,0 +1,5 @@ + + + + diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/GoogleVoiceTranscriber.kt b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/GoogleVoiceTranscriber.kt new file mode 100644 index 0000000000000..95423844c1d2c --- /dev/null +++ b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/GoogleVoiceTranscriber.kt @@ -0,0 +1,162 @@ +package expo.modules.t3voicetranscription + +import android.content.Context +import com.google.mlkit.genai.common.DownloadStatus +import com.google.mlkit.genai.common.FeatureStatus +import com.google.mlkit.genai.common.GenAiException +import com.google.mlkit.genai.speechrecognition.SpeechRecognition +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerOptions +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse +import com.google.mlkit.genai.speechrecognition.speechRecognizerOptions +import com.google.mlkit.genai.speechrecognition.speechRecognizerRequest +import java.io.Closeable +import java.util.Locale +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.withContext + +internal class VoiceTranscriptionException( + val code: String, + message: String, + cause: Throwable? = null +) : Exception(message, cause) + +private val recognitionModes = listOf( + SpeechRecognizerOptions.Mode.MODE_ADVANCED, + SpeechRecognizerOptions.Mode.MODE_BASIC +) + +internal suspend fun isGoogleVoiceTranscriptionSupported( + createClient: (Int) -> GoogleSpeechClient +): Boolean { + for (mode in recognitionModes) { + createClient(mode).use { client -> + when (checkModelSupport(client)) { + FeatureStatus.AVAILABLE, + FeatureStatus.DOWNLOADABLE, + FeatureStatus.DOWNLOADING -> return true + } + } + } + return false +} + +internal suspend fun prepareGoogleVoiceTranscription( + createClient: (Int) -> GoogleSpeechClient +): Int { + for (mode in recognitionModes) { + createClient(mode).use { client -> + if (prepareModel(client)) return mode + } + } + throw VoiceTranscriptionException( + "unavailable", + "On-device voice transcription is not available for this device language." + ) +} + +private suspend fun prepareModel(client: GoogleSpeechClient): Boolean = try { + val status = client.checkStatus() + if (status == FeatureStatus.UNAVAILABLE) { + false + } else { + if (status == FeatureStatus.DOWNLOADABLE || + status == FeatureStatus.DOWNLOADING + ) { + client.download() + } + when (client.checkStatus()) { + FeatureStatus.UNAVAILABLE -> false + FeatureStatus.AVAILABLE -> true + else -> throw VoiceTranscriptionException( + "preparation-failed", + "Voice transcription could not prepare this language." + ) + } + } +} catch (error: GenAiException) { + if (!isModelUnavailable(error)) throw error + false +} + +// Any probe failure for one mode means "try the next mode"; prepare surfaces the real error. +private suspend fun checkModelSupport(client: GoogleSpeechClient): Int = try { + client.checkStatus() +} catch (_: GenAiException) { + FeatureStatus.UNAVAILABLE +} + +private fun isModelUnavailable(error: GenAiException): Boolean = when (error.errorCode) { + GenAiException.ErrorCode.NOT_AVAILABLE, + GenAiException.ErrorCode.NOT_SUPPORTED, + GenAiException.ErrorCode.AICORE_INCOMPATIBLE, + GenAiException.ErrorCode.NEEDS_SYSTEM_UPDATE -> true + else -> false +} + +internal suspend fun collectGoogleVoiceTranscript( + responses: Flow +): String { + val text = StringBuilder() + var partial = "" + var completed = false + responses.collect { response -> + when (response) { + // Separators between final utterances are supplied by the SDK. + is SpeechRecognizerResponse.FinalTextResponse -> { + text.append(response.text) + partial = "" + } + is SpeechRecognizerResponse.PartialTextResponse -> partial = response.text + is SpeechRecognizerResponse.ErrorResponse -> throw response.e + is SpeechRecognizerResponse.CompletedResponse -> completed = true + else -> Unit + } + } + requireCompleteAudio(completed) { "Speech recognition did not complete the recording." } + // Some sessions complete with a trailing partial instead of another final event. + return text.append(partial).toString().trim() +} + +internal interface GoogleSpeechClient : Closeable { + suspend fun checkStatus(): Int + suspend fun download() +} + +internal class MlKitSpeechClient(context: Context, locale: Locale, mode: Int) : GoogleSpeechClient { + private val context = context.applicationContext + private val recognizer = SpeechRecognition.getClient( + speechRecognizerOptions { + this.locale = locale + preferredMode = mode + } + ) + + override suspend fun checkStatus() = recognizer.checkStatus() + + override suspend fun download() { + val result = recognizer.download().first { + it is DownloadStatus.DownloadCompleted || it is DownloadStatus.DownloadFailed + } + if (result is DownloadStatus.DownloadFailed) throw result.e + } + + suspend fun transcribe(uri: String): String = withRecordedPcm(context, uri) { pcm -> + PcmPipe.open().use { pipe -> + val request = speechRecognizerRequest { audioSource = pipe.source } + try { + collectGoogleVoiceTranscript( + pacedRecordedResponses(pcm, pipe) { + recognizer.startRecognition(request) + } + ) + } finally { + // Stop inference before closing the SDK-owned descriptor, including cancellation. + withContext(NonCancellable) { runCatching { recognizer.stopRecognition() } } + } + } + } + + override fun close() = recognizer.close() +} diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/PcmOutput.kt b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/PcmOutput.kt new file mode 100644 index 0000000000000..8673ce0d790e8 --- /dev/null +++ b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/PcmOutput.kt @@ -0,0 +1,85 @@ +package expo.modules.t3voicetranscription + +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import java.io.File +import java.util.concurrent.atomic.AtomicBoolean + +internal interface PcmOutput : AutoCloseable { + suspend fun write(buffer: ByteArray, count: Int) + fun finish() + fun verifyConsumed() +} + +private suspend fun replayPcm(file: File, output: PcmOutput) { + val expected = file.length() + var sent = 0L + file.inputStream().use { input -> + val buffer = ByteArray(3200) + while (true) { + val count = input.read(buffer) + if (count == -1) break + requireCompleteAudio(count > 0 && count % 2 == 0) { + "Recorded PCM contains an incomplete audio frame." + } + // STREAMING requires real-time delivery, including the final short buffer. + delay((count * 1000L + 31999) / 32000) + output.write(buffer, count) + sent += count + } + } + requireCompleteAudio(sent == expected && file.length() == expected) { + "The recording changed while speech recognition was reading it. No transcript was added." + } +} + +internal fun pacedRecordedResponses( + file: File, + output: PcmOutput, + responses: () -> Flow +): Flow = flow { + coroutineScope { + val exhausted = AtomicBoolean() + val sender = async { + replayPcm(file, output) + exhausted.set(true) + output.finish() + } + try { + withTimeout(file.length() / 32 + 30000) { + responses().collect { emit(it) } + requireCompleteAudio(exhausted.get()) { + "Speech recognition ended before receiving the complete recording. " + + "No transcript was added. Retry dictation." + } + sender.await() + output.verifyConsumed() + } + } catch (e: TimeoutCancellationException) { + throw VoiceTranscriptionException( + "incomplete-audio", + "Speech recognition did not finish the complete recording. No transcript was added. Retry dictation.", + e + ) + } finally { + sender.cancel() + try { + output.finish() + } finally { + withContext(NonCancellable) { sender.join() } + } + } + } +} + +internal inline fun requireCompleteAudio(condition: Boolean, message: () -> String) { + if (!condition) throw VoiceTranscriptionException("incomplete-audio", message()) +} diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/PcmPipe.kt b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/PcmPipe.kt new file mode 100644 index 0000000000000..b654ce01a8116 --- /dev/null +++ b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/PcmPipe.kt @@ -0,0 +1,105 @@ +package expo.modules.t3voicetranscription + +import android.os.ParcelFileDescriptor +import android.system.ErrnoException +import android.system.Os +import android.system.OsConstants +import com.google.mlkit.genai.common.audio.AudioSource +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.delay +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.withTimeout +import kotlinx.coroutines.TimeoutCancellationException +import java.util.concurrent.atomic.AtomicBoolean + +internal class PcmPipe private constructor( + val source: AudioSource, + private val writer: ParcelFileDescriptor, + verificationReader: ParcelFileDescriptor +) : PcmOutput { + private val closed = AtomicBoolean() + private val reader = ParcelFileDescriptor.AutoCloseInputStream(verificationReader) + override suspend fun write(buffer: ByteArray, count: Int) { + try { + withTimeout(5000) { + var offset = 0 + while (offset < count) { + currentCoroutineContext().ensureActive() + try { + val written = Os.write(writer.fileDescriptor, buffer, offset, count - offset) + requireCompleteAudio(written > 0) { "On-device speech stopped reading the recording." } + offset += written + } catch (e: ErrnoException) { + if (e.errno != OsConstants.EAGAIN) { + throw VoiceTranscriptionException( + "incomplete-audio", + "On-device speech stopped reading the complete recording. No transcript was added. Retry dictation.", + e + ) + } + delay(10) + } + } + } + } catch (e: TimeoutCancellationException) { + throw VoiceTranscriptionException( + "incomplete-audio", + "On-device speech could not read the complete recording. No transcript was added. Retry dictation.", + e + ) + } + } + override fun finish() = writer.close() + override fun verifyConsumed() { + requireCompleteAudio(reader.available() == 0) { + "Speech recognition ended with unread recorded audio. No transcript was added. Retry dictation." + } + } + override fun close() { + if (!closed.compareAndSet(false, true)) return + try { + writer.close() + } finally { + try { + reader.close() + } finally { + source.close() + } + } + } + companion object { + fun open(): PcmPipe { + val ends = ParcelFileDescriptor.createPipe() + var sdkReader: ParcelFileDescriptor? = null + var retained = false + try { + Os.fcntlInt(ends[1].fileDescriptor, OsConstants.F_SETFL, OsConstants.O_NONBLOCK) + // The SDK closes its descriptor; verification must retain a separate read end. + sdkReader = ends[0].dup() + val pipe = PcmPipe( + AudioSource.fromPfd(sdkReader, AudioSource.Mode.STREAMING), + ends[1], + ends[0] + ) + retained = true + return pipe + } finally { + if (!retained) closeFailedPipe(sdkReader, ends) + } + } + private fun closeFailedPipe( + sdkReader: ParcelFileDescriptor?, + ends: Array + ) { + try { + sdkReader?.close() + } finally { + try { + ends[0].close() + } finally { + ends[1].close() + } + } + } + } +} diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/RecordedPcm.kt b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/RecordedPcm.kt new file mode 100644 index 0000000000000..b719981df19bd --- /dev/null +++ b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/RecordedPcm.kt @@ -0,0 +1,172 @@ +package expo.modules.t3voicetranscription + +import android.content.Context +import android.media.AudioFormat +import android.media.MediaCodec +import android.media.MediaExtractor +import android.media.MediaFormat +import android.net.Uri +import java.io.Closeable +import java.io.File +import java.io.OutputStream +import java.nio.ByteOrder +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.withTimeout + +internal suspend fun withRecordedPcm( + context: Context, + uri: String, + transcribe: suspend (File) -> T +): T { + val pcm = File.createTempFile("dictation-", ".pcm", context.cacheDir) + try { + try { + withTimeout(30000) { decodeRecording(context, Uri.parse(uri), pcm) } + } catch (_: TimeoutCancellationException) { + throw VoiceTranscriptionException( + "incomplete-audio", + "Audio decoding did not finish the recording. Retry dictation." + ) + } + requireCompleteAudio(pcm.length() > 0 && pcm.length() % 2 == 0L) { + "The recording did not contain complete audio samples." + } + return transcribe(pcm) + } finally { + pcm.delete() + } +} + +private suspend fun decodeRecording(context: Context, uri: Uri, output: File) { + val extractor = MediaExtractor() + Closeable { extractor.release() }.use { + extractor.setDataSource(context, uri, null) + val track = (0 until extractor.trackCount).firstOrNull { + extractor.getTrackFormat(it).getString(MediaFormat.KEY_MIME)?.startsWith("audio/") == true + } ?: throw VoiceTranscriptionException("incomplete-audio", "The recording has no audio track.") + extractor.selectTrack(track) + decodeTrack(extractor, extractor.getTrackFormat(track), output) + } +} + +private suspend fun decodeTrack(extractor: MediaExtractor, format: MediaFormat, output: File) { + format.setInteger(MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_16BIT) + val decoder = MediaCodec.createDecoderByType( + requireNotNull(format.getString(MediaFormat.KEY_MIME)) + ) + Closeable { decoder.release() }.use { + decoder.configure(format, null, null, 0) + decoder.start() + Closeable { decoder.stop() }.use { + output.outputStream().buffered().use { stream -> decodeBuffers(decoder, extractor, stream) } + } + } +} + +private suspend fun decodeBuffers( + decoder: MediaCodec, + extractor: MediaExtractor, + stream: OutputStream +) { + var converter: RecordingPcmConverter? = null + var decodedFormat: Triple? = null + try { + var inputEnded = false + var outputEnded = false + val info = MediaCodec.BufferInfo() + while (!outputEnded) { + currentCoroutineContext().ensureActive() + if (!inputEnded) inputEnded = queueDecoderInput(decoder, extractor) + val index = decoder.dequeueOutputBuffer(info, 10000) + when { + index == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { + val next = pcmFormat(decoder.outputFormat, decodedFormat) + decodedFormat = next + converter = converter ?: createConverter(next, stream) + } + index >= 0 -> { + consumeDecodedBuffer(decoder, index, info, requireNotNull(converter)) + outputEnded = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 + } + } + } + requireCompleteAudio(inputEnded) { + "Audio decoding ended before reading the complete recording." + } + converter?.finish() + } finally { + converter?.close() + } +} + +private fun pcmFormat( + format: MediaFormat, + previous: Triple? +): Triple { + val encoding = if (format.containsKey(MediaFormat.KEY_PCM_ENCODING)) { + format.getInteger(MediaFormat.KEY_PCM_ENCODING) + } else { + AudioFormat.ENCODING_PCM_16BIT + } + requireCompleteAudio(encoding == AudioFormat.ENCODING_PCM_16BIT) { + "The decoder did not provide 16-bit audio." + } + val next = Triple( + format.getInteger(MediaFormat.KEY_SAMPLE_RATE), + format.getInteger(MediaFormat.KEY_CHANNEL_COUNT), + encoding + ) + requireCompleteAudio(previous == null || previous == next) { + "The recording changed audio format while decoding." + } + return next +} + +private fun createConverter( + format: Triple, + stream: OutputStream +): RecordingPcmConverter { + val bytes = ByteArray(8192) + return RecordingPcmConverter(format.first, format.second) { buffer -> + while (buffer.hasRemaining()) { + val count = minOf(buffer.remaining(), bytes.size) + buffer.get(bytes, 0, count) + stream.write(bytes, 0, count) + } + } +} + +private fun queueDecoderInput(decoder: MediaCodec, extractor: MediaExtractor): Boolean { + val index = decoder.dequeueInputBuffer(10000) + if (index < 0) return false + val buffer = requireNotNull(decoder.getInputBuffer(index)) + val count = extractor.readSampleData(buffer, 0) + return if (count < 0) { + decoder.queueInputBuffer(index, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM) + true + } else { + decoder.queueInputBuffer(index, 0, count, extractor.sampleTime, 0) + extractor.advance() + false + } +} + +private fun consumeDecodedBuffer( + decoder: MediaCodec, + index: Int, + info: MediaCodec.BufferInfo, + converter: RecordingPcmConverter +) { + try { + if (info.size <= 0 || info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) return + val buffer = requireNotNull(decoder.getOutputBuffer(index)) + buffer.position(info.offset) + buffer.limit(info.offset + info.size) + buffer.order(ByteOrder.nativeOrder()) + converter.write(buffer) + } finally { + decoder.releaseOutputBuffer(index, false) + } +} diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/RecordingPcmConverter.kt b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/RecordingPcmConverter.kt new file mode 100644 index 0000000000000..3272b2bd7d5f5 --- /dev/null +++ b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/RecordingPcmConverter.kt @@ -0,0 +1,68 @@ +package expo.modules.t3voicetranscription + +import androidx.annotation.OptIn +import androidx.media3.common.C +import androidx.media3.common.audio.AudioProcessingPipeline +import androidx.media3.common.audio.AudioProcessor +import androidx.media3.common.audio.ChannelMixingAudioProcessor +import androidx.media3.common.audio.ChannelMixingMatrix +import androidx.media3.common.audio.SonicAudioProcessor +import androidx.media3.common.util.UnstableApi +import com.google.common.collect.ImmutableList +import java.nio.ByteBuffer + +@OptIn(UnstableApi::class) +internal class RecordingPcmConverter( + sampleRate: Int, + channels: Int, + private val consume: (ByteBuffer) -> Unit +) : AutoCloseable { + private val frameSize = channels * 2 + private val pipeline = AudioProcessingPipeline( + ImmutableList.of( + ChannelMixingAudioProcessor().apply { + putChannelMixingMatrix(ChannelMixingMatrix.createForConstantGain(channels, 1)) + }, + SonicAudioProcessor().apply { setOutputSampleRateHz(16000) } + ) + ) + + init { + pipeline.configure(AudioProcessor.AudioFormat(sampleRate, channels, C.ENCODING_PCM_16BIT)) + pipeline.flush() + } + + fun write(input: ByteBuffer) { + requireCompleteAudio(input.remaining() % frameSize == 0) { + "Decoded audio contains an incomplete sample." + } + if (!pipeline.isOperational) { + consume(input) + return + } + while (input.hasRemaining()) { + val before = input.remaining() + pipeline.queueInput(input) + val output = pipeline.output + val produced = output.remaining() + consume(output) + requireCompleteAudio(input.remaining() < before || produced > 0) { + "Audio conversion stopped before the recording ended." + } + } + } + + fun finish() { + if (!pipeline.isOperational) return + pipeline.queueEndOfStream() + while (!pipeline.isEnded) { + val output = pipeline.output + requireCompleteAudio(output.hasRemaining() || pipeline.isEnded) { + "Audio conversion did not finish the recording." + } + consume(output) + } + } + + override fun close() = pipeline.reset() +} diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/T3VoiceTranscriptionModule.kt b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/T3VoiceTranscriptionModule.kt new file mode 100644 index 0000000000000..b49791bea2b9b --- /dev/null +++ b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/T3VoiceTranscriptionModule.kt @@ -0,0 +1,112 @@ +package expo.modules.t3voicetranscription + +import android.content.res.Resources +import android.os.Build +import expo.modules.kotlin.Promise +import expo.modules.kotlin.functions.Coroutine +import expo.modules.kotlin.modules.Module +import expo.modules.kotlin.modules.ModuleDefinition +import java.util.Locale +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext + +class T3VoiceTranscriptionModule : Module() { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val lock = Any() + private var operation: Pair? = null + + override fun definition() = ModuleDefinition { + Name("T3VoiceTranscription") + + AsyncFunction("checkAvailability").Coroutine { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) { + false + } else { + withContext(Dispatchers.IO) { + val context = requireNotNull(appContext.reactContext) + val locale = Resources.getSystem().configuration.locales[0] + isGoogleVoiceTranscriptionSupported { MlKitSpeechClient(context, locale, it) } + } + } + } + + AsyncFunction("prepare") { id: Int, promise: Promise -> + runOperation(id, promise, "preparation-failed") { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) { + throw VoiceTranscriptionException( + "unavailable", + "Voice transcription requires Android 12 or later." + ) + } + val context = requireNotNull(appContext.reactContext) + val locale = Resources.getSystem().configuration.locales[0] + val mode = prepareGoogleVoiceTranscription { MlKitSpeechClient(context, locale, it) } + mapOf("locale" to locale.toLanguageTag(), "mode" to mode) + } + } + + AsyncFunction("transcribe") { + id: Int, + uri: String, + language: String, + mode: Int, + promise: Promise + -> + runOperation(id, promise, "transcription-failed") { + val context = requireNotNull(appContext.reactContext) + MlKitSpeechClient(context, Locale.forLanguageTag(language), mode).use { it.transcribe(uri) } + } + } + + AsyncFunction("cancel") { id: Int, promise: Promise -> + scope.launch { + val job = synchronized(lock) { operation?.takeIf { it.first == id }?.second } + job?.cancelAndJoin() + promise.resolve(null) + } + } + + OnDestroy { scope.cancel() } + } + + private fun runOperation( + id: Int, + promise: Promise, + errorCode: String, + block: suspend () -> Any? + ) { + var result: Result? = null + val job = scope.launch(start = CoroutineStart.LAZY) { + result = runCatching { block() } + } + synchronized(lock) { + if (operation != null) { + job.cancel() + promise.reject("busy", "Another voice transcription is already active.", null) + return + } + operation = id to job + } + job.invokeOnCompletion { cause -> + synchronized(lock) { if (operation?.first == id) operation = null } + val outcome = cause?.let { Result.failure(it) } ?: result + ?: Result.failure(CancellationException("Voice transcription was cancelled.")) + outcome.fold( + onSuccess = { promise.resolve(it) }, + onFailure = { error -> + val code = voiceTranscriptionErrorCode(error, errorCode) + promise.reject(code, error.message, error) + } + ) + } + job.start() + } +} diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/VoiceTranscriptionFailure.kt b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/VoiceTranscriptionFailure.kt new file mode 100644 index 0000000000000..bc3dca6f4efbf --- /dev/null +++ b/apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/VoiceTranscriptionFailure.kt @@ -0,0 +1,24 @@ +package expo.modules.t3voicetranscription + +import com.google.mlkit.genai.common.GenAiException +import kotlinx.coroutines.CancellationException + +internal fun voiceTranscriptionErrorCode( + error: Throwable, + fallback: String +): String = when (error) { + is CancellationException -> "cancelled" + is VoiceTranscriptionException -> error.code + is GenAiException -> when { + error.message.orEmpty().contains("NO_SPEECH_DETECTED") -> "no-speech" + error.errorCode == GenAiException.ErrorCode.NOT_ENOUGH_DISK_SPACE -> "storage" + error.errorCode == GenAiException.ErrorCode.PER_APP_BATTERY_USE_QUOTA_EXCEEDED -> "quota" + error.errorCode == GenAiException.ErrorCode.BUSY -> "busy" + error.errorCode == GenAiException.ErrorCode.AUDIO_BUFFER_OVERFLOW -> "audio-overflow" + error.errorCode == GenAiException.ErrorCode.NEEDS_SYSTEM_UPDATE || + error.errorCode == GenAiException.ErrorCode.AICORE_INCOMPATIBLE -> "needs-update" + error.message.orEmpty().contains("DOWNLOAD_ERROR") -> "download-failed" + else -> fallback + } + else -> fallback +} diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/GoogleVoiceTranscriberTest.kt b/apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/GoogleVoiceTranscriberTest.kt new file mode 100644 index 0000000000000..aaa76dfbf07e2 --- /dev/null +++ b/apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/GoogleVoiceTranscriberTest.kt @@ -0,0 +1,211 @@ +package expo.modules.t3voicetranscription + +import com.google.mlkit.genai.common.FeatureStatus +import com.google.mlkit.genai.common.GenAiException +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerOptions +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class GoogleVoiceTranscriberTest { + @Test + fun preservesUtterancesAndTheirSpacingWithoutDuplicatingHypotheses() = runBlocking { + for ((finals, trailing, expected) in listOf( + Triple( + listOf("The beginning.", " The middle.", " Repeat this.", " Repeat this."), + emptyList(), + "The beginning. The middle. Repeat this. Repeat this." + ), + Triple(listOf("最初の文。", "次の文。"), emptyList(), "最初の文。次の文。"), + Triple( + listOf("The beginning."), + listOf(" An unfinished ending", " The final sentence."), + "The beginning. The final sentence." + ) + )) { + val responses = flow { + for (text in finals) { + emit(SpeechRecognizerResponse.PartialTextResponse("Unfinished hypothesis")) + emit(SpeechRecognizerResponse.FinalTextResponse(text, emptyList())) + } + for (text in trailing) emit(SpeechRecognizerResponse.PartialTextResponse(text)) + emit(SpeechRecognizerResponse.CompletedResponse) + } + assertEquals(expected, collectGoogleVoiceTranscript(responses)) + } + val incomplete = runCatching { + collectGoogleVoiceTranscript( + flow { + emit(SpeechRecognizerResponse.FinalTextResponse("Incomplete recording.", emptyList())) + } + ) + } + assertEquals( + "incomplete-audio", + (incomplete.exceptionOrNull() as VoiceTranscriptionException).code + ) + } + + private class Client( + var status: Int, + val onStatus: () -> Unit = {}, + val downloadedStatus: Int = FeatureStatus.AVAILABLE, + val onDownload: suspend () -> Unit = {} + ) : GoogleSpeechClient { + var closed = false + var downloads = 0 + + override suspend fun checkStatus(): Int { + onStatus() + return status + } + + override suspend fun download() { + downloads += 1 + onDownload() + status = downloadedStatus + } + + override fun close() { + closed = true + } + } + + @Test + fun checksSupportWithoutDownloadingAndClosesProbedClients() = runBlocking { + for ((advancedStatus, basicStatus, supported) in listOf( + Triple(FeatureStatus.AVAILABLE, FeatureStatus.UNAVAILABLE, true), + Triple(FeatureStatus.DOWNLOADABLE, FeatureStatus.UNAVAILABLE, true), + Triple(FeatureStatus.DOWNLOADING, FeatureStatus.UNAVAILABLE, true), + Triple(FeatureStatus.UNAVAILABLE, FeatureStatus.AVAILABLE, true), + Triple(FeatureStatus.UNAVAILABLE, FeatureStatus.DOWNLOADABLE, true), + Triple(FeatureStatus.UNAVAILABLE, FeatureStatus.DOWNLOADING, true), + Triple(FeatureStatus.UNAVAILABLE, FeatureStatus.UNAVAILABLE, false) + )) { + val clients = mutableListOf() + assertEquals( + supported, + isGoogleVoiceTranscriptionSupported { mode -> + Client( + if (mode == SpeechRecognizerOptions.Mode.MODE_ADVANCED) advancedStatus else basicStatus + ).also { clients.add(it) } + } + ) + assertTrue(clients.all { it.closed && it.downloads == 0 }) + assertEquals(if (advancedStatus == FeatureStatus.UNAVAILABLE) 2 else 1, clients.size) + } + } + + @Test + fun preparesAdvancedBeforeConsideringBasic() = runBlocking { + for (status in listOf( + FeatureStatus.AVAILABLE, + FeatureStatus.DOWNLOADABLE, + FeatureStatus.DOWNLOADING + )) { + val client = Client(status) + val mode = prepareGoogleVoiceTranscription { + assertEquals(SpeechRecognizerOptions.Mode.MODE_ADVANCED, it) + client + } + assertEquals(SpeechRecognizerOptions.Mode.MODE_ADVANCED, mode) + assertEquals(if (status == FeatureStatus.AVAILABLE) 0 else 1, client.downloads) + assertTrue(client.closed) + } + } + + @Test + fun fallsBackToBasicOnlyForUnavailableAdvancedModels() = runBlocking { + for ((duringDownload, errorCode) in listOf( + false to null, + false to GenAiException.ErrorCode.NOT_AVAILABLE, + false to GenAiException.ErrorCode.NOT_SUPPORTED, + true to GenAiException.ErrorCode.NOT_AVAILABLE, + true to GenAiException.ErrorCode.NOT_SUPPORTED, + true to null + )) { + val advanced = Client( + if (duringDownload) FeatureStatus.DOWNLOADABLE else FeatureStatus.UNAVAILABLE, + downloadedStatus = if (duringDownload && + errorCode == null + ) { + FeatureStatus.UNAVAILABLE + } else { + FeatureStatus.AVAILABLE + }, + onStatus = { + if (!duringDownload && errorCode != null) { + throw GenAiException(Exception("Unavailable"), errorCode) + } + }, + onDownload = { + if (duringDownload && errorCode != null) { + throw GenAiException(Exception("Unavailable"), errorCode) + } + } + ) + val basic = Client(FeatureStatus.DOWNLOADABLE) + val mode = prepareGoogleVoiceTranscription { + if (it == SpeechRecognizerOptions.Mode.MODE_ADVANCED) advanced else basic + } + assertEquals(SpeechRecognizerOptions.Mode.MODE_BASIC, mode) + assertTrue(advanced.closed && basic.closed) + assertEquals(1, basic.downloads) + } + val clients = mutableListOf() + val result = runCatching { + prepareGoogleVoiceTranscription { Client(FeatureStatus.UNAVAILABLE).also { clients.add(it) } } + } + assertEquals("unavailable", (result.exceptionOrNull() as VoiceTranscriptionException).code) + assertTrue(clients.all { it.closed }) + } + + @Test + fun preservesDownloadAndQuotaFailuresInsteadOfSilentlyFallingBack() = runBlocking { + for (errorCode in listOf( + GenAiException.ErrorCode.NOT_ENOUGH_DISK_SPACE, + GenAiException.ErrorCode.PER_APP_BATTERY_USE_QUOTA_EXCEEDED + )) { + val error = GenAiException(Exception("Preparation failed"), errorCode) + val client = Client(FeatureStatus.DOWNLOADABLE, onDownload = { throw error }) + val result = runCatching { + prepareGoogleVoiceTranscription { + assertEquals(SpeechRecognizerOptions.Mode.MODE_ADVANCED, it) + client + } + } + assertEquals(error, result.exceptionOrNull()) + assertEquals( + if (errorCode == GenAiException.ErrorCode.NOT_ENOUGH_DISK_SPACE) "storage" else "quota", + voiceTranscriptionErrorCode(error, "preparation-failed") + ) + assertTrue(client.closed) + } + } + + @Test + fun cancellationClosesTheDownloadingClientWithoutStartingBasic() = runBlocking { + val enteredDownload = CompletableDeferred() + val download = CompletableDeferred() + val client = Client(FeatureStatus.DOWNLOADABLE, onDownload = { + enteredDownload.complete(Unit) + download.await() + }) + val job = launch { + prepareGoogleVoiceTranscription { + assertEquals(SpeechRecognizerOptions.Mode.MODE_ADVANCED, it) + client + } + } + enteredDownload.await() + job.cancelAndJoin() + assertTrue(client.closed) + assertEquals(FeatureStatus.DOWNLOADABLE, client.status) + } +} diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/PcmReplayTest.kt b/apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/PcmReplayTest.kt new file mode 100644 index 0000000000000..f226cbebca7cb --- /dev/null +++ b/apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/PcmReplayTest.kt @@ -0,0 +1,126 @@ +package expo.modules.t3voicetranscription + +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse +import java.io.ByteArrayOutputStream +import java.io.File +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.launch +import kotlinx.coroutines.test.currentTime +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +@OptIn(kotlinx.coroutines.ExperimentalCoroutinesApi::class) +class PcmReplayTest { + private class Output : PcmOutput { + val bytes = ByteArrayOutputStream() + val ended = CompletableDeferred() + var unread = false + var closed = false + + override suspend fun write(buffer: ByteArray, count: Int) { + bytes.write(buffer, 0, count) + } + override fun finish() { + ended.complete(Unit) + } + override fun verifyConsumed() { + requireCompleteAudio(!unread) { "Unread audio." } + } + override fun close() { + closed = true + } + } + + @Test fun replaysTheEntireLongFileIncludingTheLastShortBuffer() = runTest { + val pcm = ByteArray(1994140) { (it % 251).toByte() } + val file = File.createTempFile("dictation-replay-", ".pcm").apply { writeBytes(pcm) } + try { + Output().use { output -> + val responses = pacedRecordedResponses(file, output) { + flow { + output.ended.await() + emit(SpeechRecognizerResponse.FinalTextResponse("Opening. ", emptyList())) + emit(SpeechRecognizerResponse.FinalTextResponse("Repeat. Repeat. 中間。", emptyList())) + emit(SpeechRecognizerResponse.PartialTextResponse(" A longer unfinished ending")) + emit(SpeechRecognizerResponse.PartialTextResponse(" End.")) + emit(SpeechRecognizerResponse.CompletedResponse) + } + } + assertEquals("Opening. Repeat. Repeat. 中間。 End.", collectGoogleVoiceTranscript(responses)) + assertArrayEquals(pcm, output.bytes.toByteArray()) + assertEquals(62317L, currentTime) + } + } finally { + file.delete() + } + } + + @Test fun refusesSuccessfulSdkCompletionBeforeAllAudioIsConsumed() = runTest { + val file = File.createTempFile("dictation-replay-", ".pcm").apply { + writeBytes(ByteArray(6400)) + } + try { + for (early in listOf(true, false)) { + Output().use { output -> + output.unread = !early + val result = runCatching { + collectGoogleVoiceTranscript( + pacedRecordedResponses(file, output) { + flow { + if (!early) output.ended.await() + emit( + SpeechRecognizerResponse.FinalTextResponse( + "Plausible but incomplete.", + emptyList() + ) + ) + emit(SpeechRecognizerResponse.CompletedResponse) + } + } + ) + } + assertEquals( + "incomplete-audio", + (result.exceptionOrNull() as VoiceTranscriptionException).code + ) + assertTrue(output.ended.isCompleted) + } + } + } finally { + file.delete() + } + } + + @Test fun cancellationSettlesTheSenderBeforeReleasingItsOutput() = runTest { + val file = File.createTempFile("dictation-replay-", ".pcm").apply { + writeBytes(ByteArray(6400)) + } + val entered = CompletableDeferred() + val output = Output() + try { + val job = launch { + output.use { + collectGoogleVoiceTranscript( + pacedRecordedResponses(file, it) { + flow { + entered.complete(Unit) + CompletableDeferred().await() + } + } + ) + } + } + entered.await() + job.cancelAndJoin() + assertTrue(output.ended.isCompleted) + assertTrue(output.closed) + } finally { + file.delete() + } + } +} diff --git a/apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/RecordingPcmConverterTest.kt b/apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/RecordingPcmConverterTest.kt new file mode 100644 index 0000000000000..694523610e880 --- /dev/null +++ b/apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/RecordingPcmConverterTest.kt @@ -0,0 +1,51 @@ +package expo.modules.t3voicetranscription + +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import java.nio.ByteOrder +import kotlin.math.abs +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(manifest = Config.NONE, sdk = [31]) +class RecordingPcmConverterTest { + @Test fun preservesAlreadyCompatiblePcmAndResamplesStereoThroughItsTail() { + for ((rate, channels) in listOf(16000 to 1, 44100 to 2, 48000 to 2)) { + val frames = rate * 2 + 37 + val input = ByteBuffer.allocateDirect(frames * channels * 2).order(ByteOrder.nativeOrder()) + repeat(frames) { frame -> + val level = if (frame < rate) 6000 else 18000 + repeat(channels) { input.putShort(level.toShort()) } + } + input.flip() + val original = ByteArray(input.remaining()).also { input.duplicate().get(it) } + val result = ByteArrayOutputStream() + RecordingPcmConverter(rate, channels) { buffer -> + val bytes = ByteArray(buffer.remaining()) + buffer.get(bytes) + result.write(bytes) + }.use { converter -> + while (input.hasRemaining()) { + val end = minOf(input.limit(), input.position() + channels * 2 * 317) + val block = input.slice().order(ByteOrder.nativeOrder()) + block.limit(end - input.position()) + converter.write(block) + input.position(end) + } + converter.finish() + } + val bytes = result.toByteArray() + if (rate == 16000) assertArrayEquals(original, bytes) + val samples = ByteBuffer.wrap(bytes).order(ByteOrder.nativeOrder()).asShortBuffer() + assertTrue(abs(samples.remaining() * rate.toLong() - frames * 16000L) <= rate) + assertEquals(6000, samples[8000].toInt()) + assertEquals(18000, samples[samples.limit() - 10].toInt()) + } + } +} diff --git a/apps/mobile/modules/t3-voice-transcription/expo-module.config.json b/apps/mobile/modules/t3-voice-transcription/expo-module.config.json new file mode 100644 index 0000000000000..5867f41ffe75e --- /dev/null +++ b/apps/mobile/modules/t3-voice-transcription/expo-module.config.json @@ -0,0 +1,6 @@ +{ + "platforms": ["android"], + "android": { + "modules": ["expo.modules.t3voicetranscription.T3VoiceTranscriptionModule"] + } +} diff --git a/apps/mobile/src/App.tsx b/apps/mobile/src/App.tsx index 906b6d5635ce3..f79e0f6dcc686 100644 --- a/apps/mobile/src/App.tsx +++ b/apps/mobile/src/App.tsx @@ -19,6 +19,7 @@ import { type NavigationState, } from "@react-navigation/native"; import { pipe } from "effect/Function"; +import { DEFAULT_HOSTED_APP_URL } from "@t3tools/shared/connectAuth"; import { RegistryContext } from "@effect/atom-react"; import { ProjectChipIconRasterizer } from "./native/ProjectChipIconImages"; @@ -125,7 +126,13 @@ function getAppStateFromPath(path: string, options: Parameters shouldHandleAppLink(url, hasCloudPublicConfig()), getInitialURL: async () => (await Linking.getInitialURL()) ?? (await launchWorkspaceURL()), diff --git a/apps/mobile/src/components/ui/AndroidAnchoredMenu.tsx b/apps/mobile/src/components/ui/AndroidAnchoredMenu.tsx index a1f6a02cc32eb..369fcf19ef55a 100644 --- a/apps/mobile/src/components/ui/AndroidAnchoredMenu.tsx +++ b/apps/mobile/src/components/ui/AndroidAnchoredMenu.tsx @@ -4,7 +4,6 @@ import { use, useRef, useState } from "react"; import type { StyleProp, ViewInstance, ViewStyle } from "react-native"; import { Pressable, ScrollView, View } from "react-native"; import { useKeyboardState } from "react-native-keyboard-controller"; -import Animated, { FadeIn } from "react-native-reanimated"; import { useHardwareBack } from "../../lib/backHandler"; import { deriveMenuHorizontalPlacement } from "../../lib/menu-placement"; @@ -181,8 +180,8 @@ export function AndroidAnchoredMenu(props: AndroidAnchoredMenuProps) { onClose={close} /> ) : ( - - + )} diff --git a/apps/mobile/src/features/pull-requests/PullRequestRouteScreen.tsx b/apps/mobile/src/features/pull-requests/PullRequestRouteScreen.tsx index 4e55ed4ef651b..6129f80d439ec 100644 --- a/apps/mobile/src/features/pull-requests/PullRequestRouteScreen.tsx +++ b/apps/mobile/src/features/pull-requests/PullRequestRouteScreen.tsx @@ -423,6 +423,13 @@ export function PullRequestRouteScreen({ route }: StaticScreenProps + stateAction === "close" ? action.confirmClose() : void action.perform("reopen"); const confirmDeleteBranch = () => pr && action.confirm( @@ -557,6 +564,17 @@ export function PullRequestRouteScreen({ route }: StaticScreenPropsShare… ) : null} + {stateAction ? ( + + {pullRequestActionLabel(stateAction, terminology)} + + ) : null} {showsDeleteBranch ? ( { const selected = actions.find((id) => id === nativeEvent.event); if (!selected) return; - if (selected === "close") - action.confirm( - selected, - pullRequestActionLabel("close", terminology), - `Close this ${terminology.singular} without merging its changes?`, - ); + if (selected === "close") action.confirmClose(); else if (selected === "revert") action.confirm( selected, diff --git a/apps/mobile/src/features/pull-requests/usePullRequestAction.ts b/apps/mobile/src/features/pull-requests/usePullRequestAction.ts index f53a8523d6b24..4658330dd6cc3 100644 --- a/apps/mobile/src/features/pull-requests/usePullRequestAction.ts +++ b/apps/mobile/src/features/pull-requests/usePullRequestAction.ts @@ -11,7 +11,7 @@ import type { import { useRef, useState } from "react"; import { composerPullRequests } from "../../state/pull-requests"; import { useAtomCommand } from "../../state/use-atom-command"; -import { canPerformPullRequestAction } from "./pullRequestActions"; +import { canPerformPullRequestAction, pullRequestActionLabel } from "./pullRequestActions"; import { pullRequestTarget, type PullRequestParams } from "./usePullRequestDiff"; import { usePullRequestPermissions } from "./usePullRequestPermissions"; @@ -55,5 +55,13 @@ export function usePullRequestAction(params: PullRequestParams, pr: PullRequestD onPress: () => void perform(action), }, ]); - return { available, perform, confirm, busy }; + const confirmClose = () => { + const terminology = sourceControlClients.get(pr?.provider).changeRequest; + confirm( + "close", + pullRequestActionLabel("close", terminology), + `Close this ${terminology.singular} without merging its changes?`, + ); + }; + return { available, perform, confirm, confirmClose, busy }; } diff --git a/apps/mobile/src/features/settings/SettingsEnvironmentDetailRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsEnvironmentDetailRouteScreen.tsx index 6c44aad6cf34f..e18cebdb2d346 100644 --- a/apps/mobile/src/features/settings/SettingsEnvironmentDetailRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsEnvironmentDetailRouteScreen.tsx @@ -337,7 +337,11 @@ function EnvironmentDetail({ environmentId }: { readonly environmentId: Environm - + {provider.displayName ?? provider.driver} diff --git a/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx index 480ae40232e39..be236d80ccefc 100644 --- a/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx @@ -3,7 +3,7 @@ import { useAtomSet, useAtomValue } from "@effect/atom-react"; import { AsyncResult } from "effect/reactivity"; import { getRecordingPermissionsAsync, requestRecordingPermissionsAsync } from "expo-audio"; import { useState } from "react"; -import { Linking, useWindowDimensions, View } from "react-native"; +import { Linking, Platform, useWindowDimensions, View } from "react-native"; import Reanimated, { ReduceMotion, useAnimatedStyle, withTiming } from "react-native-reanimated"; import { useSafeAreaInsets } from "react-native-safe-area-context"; @@ -16,15 +16,17 @@ import { type MicrophoneEntry, type MicrophoneKind, microphoneEntryKey, - microphoneOrder, } from "../../lib/microphonePriority"; -import { getLocalVoiceTranscriber } from "../../native/voiceTranscription"; import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences"; import { ListRowLayout } from "../../components/ui/ListRowLayout"; import { ThreadSwipeable } from "../home/thread-swipe-actions"; import { ListActionRow as SettingsActionRow } from "../../components/ui/ListActionRow"; import { SettingsDragHandle } from "./components/SettingsDragHandle"; import { SettingsSection } from "./components/SettingsSection"; +import { + deviceMicrophoneOrder, + localVoiceTranscriberAtom, +} from "../voice-input/VoiceInputProvider"; type MicrophonePermission = "checking" | "granted" | "ask" | "denied"; @@ -72,7 +74,10 @@ const KINDS: Record< bluetooth: { label: "Bluetooth", row: "Other Bluetooth", - rowDescription: "AirPods, headsets, and cars not listed", + rowDescription: + Platform.OS === "android" + ? "Earbuds, headsets, and cars, including Android Auto, not listed" + : "AirPods, headsets, and cars not listed", icon: "headphones", }, builtIn: { @@ -89,8 +94,16 @@ const KINDS: Record< }, }; +const UNAVAILABLE_EXPLANATION = + Platform.OS === "android" + ? "T3 Code's voice input uses Google's on-device speech recognition, which needs Android 12 or later with compatible Google speech services for the device language. Without it, the composer has no microphone button, and allowing microphone access won't add one. Keyboard voice typing is separate and isn't affected." + : "T3 Code's voice input uses Apple's on-device speech transcription, which needs iOS 26 or later on a device that supports it. Without it, the composer has no microphone button, and allowing microphone access won't add one. Apple keyboard dictation is separate and isn't affected."; + export function SettingsMicrophoneRouteScreen() { - return getLocalVoiceTranscriber() === null ? : ; + const transcriber = useAtomValue(localVoiceTranscriberAtom); + // Android's probe is asynchronous; an empty sheet avoids flashing "unavailable" first. + if (transcriber === "checking") return ; + return transcriber === null ? : ; } function VoiceInputUnavailable() { @@ -107,12 +120,7 @@ function VoiceInputUnavailable() { - - T3 Code's voice input uses Apple's on-device speech transcription, which needs iOS 26 or - later on a device that supports it. Without it, the composer has no microphone button, and - allowing microphone access won't add one. Apple keyboard dictation is separate and isn't - affected. - + {UNAVAILABLE_EXPLANATION} ); @@ -123,7 +131,7 @@ function MicrophoneSettings() { const preferencesResult = useAtomValue(mobilePreferencesAtom); const savePreferences = useAtomSet(updateMobilePreferencesAtom); const ready = AsyncResult.isSuccess(preferencesResult) && !preferencesResult.waiting; - const entries = microphoneOrder( + const entries = deviceMicrophoneOrder( AsyncResult.isSuccess(preferencesResult) ? preferencesResult.value.microphones : undefined, ); const { permission, request: requestPermission } = useMicrophonePermission(); @@ -138,7 +146,7 @@ function MicrophoneSettings() { const move = (key: string, steps: number) => savePreferences({ transform: (current) => { - const next = [...microphoneOrder(current.microphones)]; + const next = [...deviceMicrophoneOrder(current.microphones)]; const from = next.findIndex((entry) => microphoneEntryKey(entry) === key); const to = Math.min(next.length - 1, Math.max(0, from + steps)); if (from === -1 || from === to) return {}; @@ -150,7 +158,7 @@ function MicrophoneSettings() { const forget = (key: string) => savePreferences({ transform: (current) => ({ - microphones: microphoneOrder(current.microphones).filter( + microphones: deviceMicrophoneOrder(current.microphones).filter( (entry) => microphoneEntryKey(entry) !== key, ), }), @@ -227,9 +235,12 @@ function MicrophoneSettings() { })} - Voice input records from the first connected microphone in this list. Wired and Bluetooth + Voice input prefers the first connected microphone in this list. Wired and Bluetooth devices are added by name after you use voice input with them connected; until then, the Other row ranks them. Swipe a device left to forget it. + {Platform.OS === "android" + ? " Android may use another input when the preferred microphone cannot be routed." + : null} diff --git a/apps/mobile/src/features/settings/SettingsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsRouteScreen.tsx index 85ceb15d50a2a..341841876d973 100644 --- a/apps/mobile/src/features/settings/SettingsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsRouteScreen.tsx @@ -244,12 +244,8 @@ function useSettingsIndexSections(): SettingsListSection[] { ...(navigationCustomizationAvailable() ? [page("SettingsNavigation", "line.3.horizontal", "Navigation tabs")] : []), - ...(Platform.OS === "ios" - ? [ - page("SettingsKeyboard", "keyboard", "Keyboard"), - page("SettingsMicrophone", "mic", "Microphone"), - ] - : []), + ...(Platform.OS === "ios" ? [page("SettingsKeyboard", "keyboard", "Keyboard")] : []), + page("SettingsMicrophone", "mic", "Microphone"), ], }, { diff --git a/apps/mobile/src/features/settings/components/ScheduledTaskPromptField.tsx b/apps/mobile/src/features/settings/components/ScheduledTaskPromptField.tsx index 171ea51db1247..10e9ad3528cae 100644 --- a/apps/mobile/src/features/settings/components/ScheduledTaskPromptField.tsx +++ b/apps/mobile/src/features/settings/components/ScheduledTaskPromptField.tsx @@ -4,6 +4,7 @@ import { AppText as Text } from "../../../components/ui/AppText"; import type { ComposerEditorSelection } from "../../../components/ComposerEditor"; import { ComposerDictationCancelAction, + ComposerDictationError, ComposerDictationPrimaryAction, ComposerDictationStartAction, ComposerDictationStatus, @@ -24,7 +25,7 @@ export function ScheduledTaskPromptField(props: { }) { const voice = props.voiceInput; const presentation = resolveVoiceComposerPresentation(voice.state, voice.elapsedSeconds); - const showsDictation = presentation.statusLabel !== null; + const showsDictation = presentation.statusKind === "active"; const showsIdleTrigger = voice.isAvailable && !showsDictation; return ( @@ -57,6 +58,10 @@ export function ScheduledTaskPromptField(props: { ) : null} + {showsDictation ? ( @@ -66,7 +71,6 @@ export function ScheduledTaskPromptField(props: { elapsedSeconds={voice.elapsedSeconds} phase={voice.state.phase} presentation={presentation} - onDismissError={voice.cancel} /> + {isExpanded ? editingAttachments : null} { @@ -1280,7 +1285,6 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps elapsedSeconds={voiceInput.elapsedSeconds} phase={voiceInput.state.phase} presentation={voicePresentation} - onDismissError={voiceInput.cancel} /> ) : ( diff --git a/apps/mobile/src/features/threads/ThreadFeed.tsx b/apps/mobile/src/features/threads/ThreadFeed.tsx index fb208d3c0525a..86ed9e58c9d00 100644 --- a/apps/mobile/src/features/threads/ThreadFeed.tsx +++ b/apps/mobile/src/features/threads/ThreadFeed.tsx @@ -65,7 +65,7 @@ import { } from "@t3tools/contracts"; import { renderAssistantCitationsAsText } from "@t3tools/shared/assistantCitations"; import { encodeComposerContextFragment } from "@t3tools/shared/composerContextClipboard"; -import { parseThreadLinkHref } from "@t3tools/shared/threadLinks"; +import { hostedThreadLink, parseThreadLinkHref } from "@t3tools/shared/threadLinks"; import { PROMPT_LINK_MAX_PROMPT_LENGTH } from "@t3tools/shared/promptLinks"; import { parseComposerContextHref, @@ -3128,9 +3128,10 @@ export const ThreadFeed = memo(function ThreadFeed({ else if (actionId === "copy-skill") copyTextWithHaptic(`$${skill.name}`); return; } - if (resolveLinkedThread(href)) { + const thread = resolveLinkedThread(href); + if (thread) { if (actionId === "open-thread") onMarkdownLinkPress(href); - else if (actionId === "copy-thread-link") copyTextWithHaptic(href); + else if (actionId === "copy-thread-link") copyTextWithHaptic(hostedThreadLink(thread)); return; } const target = resolveFileChipTarget(href, props.workspaceRoot); diff --git a/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx b/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx index f38c4f05825a5..615914589e80e 100644 --- a/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx +++ b/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx @@ -319,12 +319,39 @@ function VoiceActionButton(props: { ); } +export function ComposerDictationError(props: { + readonly error: string | null; + readonly onDismiss: () => void; +}) { + if (!props.error) return null; + return ( + + + {props.error} + + + + + + ); +} + export function ComposerDictationStatus(props: { readonly audioLevels: SharedValue>; readonly elapsedSeconds: number; readonly phase: VoiceInputPhase; readonly presentation: VoiceComposerPresentation; - readonly onDismissError: () => void; }) { const recording = props.phase === "recording"; const waveformStyle = useAnimatedStyle(() => ({ @@ -334,54 +361,28 @@ export function ComposerDictationStatus(props: { opacity: withTiming(recording ? 0 : 1, DICTATION_TIMING), })); - if (!props.presentation.statusLabel) return null; - const isError = props.presentation.statusKind === "error"; + if (!props.presentation.statusLabel || props.presentation.statusKind === "error") return null; return ( - {isError ? ( - - + + + + + + + {props.presentation.statusLabel} - - - - - ) : ( - - - - - - - - {props.presentation.statusLabel} - - - - )} + + ); } diff --git a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx index 809ee3a9d21b5..eab40efd4905b 100644 --- a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx +++ b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx @@ -1,18 +1,14 @@ import { useAtomValue } from "@effect/atom-react"; -import { - AudioModule, - RecordingPresets, - requestRecordingPermissionsAsync, - type RecordingStatus, -} from "expo-audio"; +import { AudioModule, RecordingPresets, type RecordingStatus } from "expo-audio"; import { File } from "expo-file-system"; import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake"; import { useSyncExternalStore } from "react"; -import { AppState, Platform } from "react-native"; +import { Platform } from "react-native"; import { Atom, AsyncResult } from "effect/reactivity"; import { makeMutable } from "react-native-reanimated"; import { + type MicrophoneEntry, microphoneOrder, preferredMicrophoneInput, rememberMicrophones, @@ -20,10 +16,15 @@ import { import { getLocalVoiceTranscriber } from "../../native/voiceTranscription"; import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences"; import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene"; -import { voiceInputBlocksSubmission } from "@t3tools/client-runtime/voice-input"; +import { + voiceInputBlocksSubmission, + type VoiceTranscriber, +} from "@t3tools/client-runtime/voice-input"; import { createLazyVoiceRecorder } from "./lazyVoiceRecorder"; import { VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering"; import { VoiceInputSession } from "./voiceInputSession"; +import { createVoiceRecordingPermission } from "./voiceRecordingPermission"; +import { makePermissionStatusAtom } from "../../lib/permissionStatusAtom"; import { acquireAudioSession, activateAudioSession, @@ -49,6 +50,24 @@ function keepScreenAwake(tag: string): () => void { return () => void activation.then(() => deactivateKeepAwake(tag)).catch(() => {}); } +export const deviceMicrophoneOrder = (saved?: ReadonlyArray) => + microphoneOrder(saved, Platform.OS === "android" ? "android" : "ios"); + +/** "checking" until the first availability probe settles; re-probed on foreground. */ +export const localVoiceTranscriberAtom = makePermissionStatusAtom< + VoiceTranscriber | null | "checking" +>({ + label: "mobile:voice-input:transcriber", + initial: "checking", + read: () => getLocalVoiceTranscriber().catch(() => null), +}).pipe(Atom.keepAlive); + +// Each foreground re-probe yields a new transcriber; composers re-render only when availability flips. +const localVoiceInputAvailableAtom = Atom.make((get) => { + const transcriber = get(localVoiceTranscriberAtom); + return transcriber !== null && transcriber !== "checking"; +}); + /** * The app has one microphone and one dictation at a time, so the session * lives for the registry's lifetime rather than any screen's. @@ -58,11 +77,15 @@ const voiceInputAtom = Atom.make((get) => { // Mounted so the recorder can read and remember microphones synchronously. get.mount(mobilePreferencesAtom); get.mount(updateMobilePreferencesAtom); + get.mount(localVoiceTranscriberAtom); const audioLevels = makeMutable>( Array(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0), ); let keepAwakeSequence = 0; let audioToken: ReturnType | null = null; + const permission = createVoiceRecordingPermission(() => { + void session.controller.appMovedToBackground(); + }); const recorder = createLazyVoiceRecorder({ create: () => new AudioModule.AudioRecorder(VOICE_RECORDING_OPTIONS), onStatus: (status: RecordingStatus) => { @@ -77,12 +100,12 @@ const voiceInputAtom = Atom.make((get) => { selectInput: (inputs) => { const preferences = registry.get(mobilePreferencesAtom); if (!AsyncResult.isSuccess(preferences)) return null; - const known = microphoneOrder(preferences.value.microphones); + const known = deviceMicrophoneOrder(preferences.value.microphones); const order = rememberMicrophones(known, inputs); if (order !== known) { registry.set(updateMobilePreferencesAtom, { transform: (current) => ({ - microphones: rememberMicrophones(microphoneOrder(current.microphones), inputs), + microphones: rememberMicrophones(deviceMicrophoneOrder(current.microphones), inputs), }), }); } @@ -91,11 +114,11 @@ const voiceInputAtom = Atom.make((get) => { }); const session: VoiceInputSession = new VoiceInputSession({ recorder, - getTranscriber: getLocalVoiceTranscriber, - requestPermission: async () => { - const permission = await requestRecordingPermissionsAsync(); - return { granted: permission.granted, canAskAgain: permission.canAskAgain }; + getTranscriber: () => { + const transcriber = registry.get(localVoiceTranscriberAtom); + return transcriber === "checking" ? null : transcriber; }, + requestPermission: permission.requestPermission, configureRecording: async () => { const token = acquireAudioSession("recording"); audioToken = token; @@ -120,14 +143,8 @@ const voiceInputAtom = Atom.make((get) => { publishAudioLevels: (levels) => audioLevels.set(levels), keepAwake: () => keepScreenAwake(`voice-input:${++keepAwakeSequence}`), }); - // iOS reports `inactive` while its permission dialog is open. Only the real - // background state cancels preparation; recorder status handles calls and - // route interruptions during capture. - const appStateSubscription = AppState.addEventListener("change", (nextState) => { - if (nextState === "background") void session.controller.appMovedToBackground(); - }); get.addFinalizer(() => { - appStateSubscription.remove(); + permission.dispose(); session.dispose(); recorder.release(); }); @@ -136,10 +153,11 @@ const voiceInputAtom = Atom.make((get) => { export function useGlobalVoiceInput() { const { session, audioLevels } = useAtomValue(voiceInputAtom); + const isTranscriberAvailable = useAtomValue(localVoiceInputAvailableAtom); const snapshot = useSyncExternalStore(session.subscribe, session.getSnapshot); return { ...snapshot, - isAvailable: getLocalVoiceTranscriber() !== null || getNativeShowcaseScene() !== null, + isAvailable: isTranscriberAvailable || getNativeShowcaseScene() !== null, audioLevels, isBusy: voiceInputBlocksSubmission(snapshot.state), session, diff --git a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts index d3236bc7c992b..fbb6059099a8a 100644 --- a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts +++ b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts @@ -9,14 +9,20 @@ type State = { readonly isRecording: boolean }; function createNativeRecorder(uri: string) { const listeners = new Set<(status: Status) => void>(); + // Like expo-audio on Android, preparing again requires a stop in between. + let isPrepared = false; return { uri: null as string | null, prepared: uri, prepareToRecordAsync: vi.fn(async function (this: { uri: string | null; prepared: string }) { + if (isPrepared) throw new Error("Recorder is already prepared"); + isPrepared = true; this.uri = this.prepared; }), record: vi.fn(), - stop: vi.fn(async (): Promise => undefined), + stop: vi.fn(async (): Promise => { + isPrepared = false; + }), getStatus: vi.fn((): State => ({ isRecording: true })), addListener: vi.fn((_event: "recordingStatusUpdate", listener: (status: Status) => void) => { listeners.add(listener); @@ -26,7 +32,8 @@ function createNativeRecorder(uri: string) { { uid: "car", name: "CarPlay", type: "CarAudio" }, { uid: "built-in", name: "iPhone Microphone", type: "MicrophoneBuiltIn" }, ]), - setInput: vi.fn((_uid: string) => undefined), + setInput: vi.fn((_uid: string): void | Promise => undefined), + cancelInputSelection: vi.fn((): void => undefined), release: vi.fn(), emit: (status: Status) => listeners.forEach((listener) => listener(status)), }; @@ -34,12 +41,14 @@ function createNativeRecorder(uri: string) { function createHarness( selectInput?: (inputs: ReadonlyArray) => VoiceRecordingInput | null, + configure?: (native: ReturnType) => void, ) { const created: Array> = []; const statuses: Status[] = []; const recorder = createLazyVoiceRecorder({ create: () => { const native = createNativeRecorder(`file:///voice-${created.length}.m4a`); + configure?.(native); created.push(native); return native; }, @@ -57,10 +66,21 @@ describe("createLazyVoiceRecorder", () => { const disconnected = createHarness((inputs) => inputs[1] ?? null); await disconnected.recorder.prepareToRecordAsync(); + await disconnected.recorder.stop(); disconnected.created[0]!.setInput.mockImplementationOnce(() => { throw new Error("Preferred input not found"); }); await expect(disconnected.recorder.prepareToRecordAsync()).resolves.toBeUndefined(); + + const timedOut = createHarness( + (inputs) => inputs[1] ?? null, + (native) => { + native.setInput.mockImplementationOnce(() => + Promise.reject(new Error("Routing timed out")), + ); + }, + ); + await expect(timedOut.recorder.prepareToRecordAsync()).resolves.toBeUndefined(); }); it("forwards status events until the recorder is released", async () => { @@ -76,8 +96,12 @@ describe("createLazyVoiceRecorder", () => { }); describe("with the voice input controller", () => { - function createController(granted: boolean) { - const harness = createHarness(); + function createController( + granted: boolean, + selectInput?: (inputs: ReadonlyArray) => VoiceRecordingInput | null, + configure?: (native: ReturnType) => void, + ) { + const harness = createHarness(selectInput, configure); const deleted: string[] = []; const controller = new VoiceInputController({ recorder: harness.recorder, @@ -108,5 +132,29 @@ describe("createLazyVoiceRecorder", () => { expect(controller.currentState.phase).toBe("error"); expect(created).toHaveLength(0); }); + + it("abandons a pending input route when dictation is canceled", async () => { + const routing = Promise.withResolvers(); + const { controller, created } = createController( + true, + (inputs) => inputs[1] ?? null, + (native) => { + // Expo drops the promise of a canceled coroutine function, so it never settles. + native.setInput.mockImplementationOnce(() => { + routing.resolve(); + return new Promise(() => undefined); + }); + }, + ); + const starting = controller.start(); + await routing.promise; + controller.cancel(); + await starting; + expect(created[0]!.cancelInputSelection).toHaveBeenCalled(); + expect(controller.currentState.phase).toBe("idle"); + + await controller.start(); + expect(controller.currentState.phase).toBe("recording"); + }); }); }); diff --git a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts index eceb06bab8c64..6100a869fa441 100644 --- a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts +++ b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts @@ -1,4 +1,7 @@ -import type { VoiceRecorder } from "@t3tools/client-runtime/voice-input"; +import { + throwIfVoiceTranscriptionAborted, + type VoiceRecorder, +} from "@t3tools/client-runtime/voice-input"; type Subscription = { remove(): void }; @@ -19,7 +22,8 @@ export interface NativeVoiceRecorder { listener: (status: TStatus) => void, ): Subscription; getAvailableInputs(): ReadonlyArray; - setInput(uid: string): void; + setInput(uid: string): void | Promise; + cancelInputSelection?(): void; release(): void; } @@ -39,6 +43,7 @@ export function createLazyVoiceRecorder(input: { let retired: NativeVoiceRecorder | null = null; let retiredUri: string | null = null; let pending: Promise = Promise.resolve(); + let abandonInputSelection: (() => void) | null = null; const track = (promise: Promise): Promise => { pending = Promise.allSettled([pending, promise]).then(() => undefined); @@ -50,19 +55,42 @@ export function createLazyVoiceRecorder(input: { if (recorder) return recorder.uri; return retired ? retired.uri : retiredUri; }, - prepareToRecordAsync() { + prepareToRecordAsync(signal) { if (!recorder) { recorder = input.create(); subscription = recorder.addListener("recordingStatusUpdate", input.onStatus); } const prepared = recorder; return track( - prepared.prepareToRecordAsync().then(() => { + prepared.prepareToRecordAsync().then(async () => { + if (signal) throwIfVoiceTranscriptionAborted(signal); if (!input.selectInput) return; + let selected: VoiceRecordingInput | null; try { - const selected = input.selectInput(prepared.getAvailableInputs()); - if (selected) prepared.setInput(selected.uid); - } catch {} + selected = input.selectInput(prepared.getAvailableInputs()); + } catch { + return; + } + if (!selected) return; + const uid = selected.uid; + // Expo never settles a canceled coroutine function's promise, so stop waiting on abort. + const abandoned = Promise.withResolvers(); + const abandon = () => { + prepared.cancelInputSelection?.(); + abandoned.resolve(); + }; + signal?.addEventListener("abort", abandon, { once: true }); + abandonInputSelection = abandon; + try { + await Promise.race([ + new Promise((resolve) => resolve(prepared.setInput(uid))).catch(() => {}), + abandoned.promise, + ]); + if (signal) throwIfVoiceTranscriptionAborted(signal); + } finally { + signal?.removeEventListener("abort", abandon); + if (abandonInputSelection === abandon) abandonInputSelection = null; + } }), ); }, @@ -79,6 +107,7 @@ export function createLazyVoiceRecorder(input: { release() { const released = recorder; if (!released) return; + abandonInputSelection?.(); recorder = null; subscription?.remove(); subscription = null; diff --git a/apps/mobile/src/features/voice-input/voiceRecordingPermission.test.ts b/apps/mobile/src/features/voice-input/voiceRecordingPermission.test.ts new file mode 100644 index 0000000000000..ccb108811f78a --- /dev/null +++ b/apps/mobile/src/features/voice-input/voiceRecordingPermission.test.ts @@ -0,0 +1,52 @@ +import { expect, it, vi } from "vite-plus/test"; + +const mocks = vi.hoisted(() => ({ + state: "active", + listeners: new Set<(state: string) => void>(), + request: vi.fn<() => Promise<{ granted: boolean; canAskAgain: boolean }>>(), +})); + +vi.mock("expo-audio", () => ({ requestRecordingPermissionsAsync: mocks.request })); +vi.mock("react-native", () => ({ + Platform: { OS: "android" }, + AppState: { + get currentState() { + return mocks.state; + }, + addEventListener: (_event: string, listener: (state: string) => void) => { + mocks.listeners.add(listener); + return { remove: () => mocks.listeners.delete(listener) }; + }, + }, +})); + +import { createVoiceRecordingPermission } from "./voiceRecordingPermission"; + +function changeAppState(state: string) { + mocks.state = state; + mocks.listeners.forEach((listener) => listener(state)); +} + +it("keeps dictation alive while Android's permission activity backgrounds the app", async () => { + const permission = Promise.withResolvers<{ granted: boolean; canAskAgain: boolean }>(); + mocks.request.mockReturnValue(permission.promise); + const interrupted = vi.fn(); + const recording = createVoiceRecordingPermission(interrupted); + let settled = false; + const result = recording.requestPermission().then((value) => { + settled = true; + return value; + }); + + changeAppState("background"); + permission.resolve({ granted: true, canAskAgain: true }); + await Promise.resolve(); + expect(settled).toBe(false); + changeAppState("active"); + expect(await result).toMatchObject({ granted: true }); + expect(interrupted).not.toHaveBeenCalled(); + + changeAppState("background"); + expect(interrupted).toHaveBeenCalledOnce(); + recording.dispose(); +}); diff --git a/apps/mobile/src/features/voice-input/voiceRecordingPermission.ts b/apps/mobile/src/features/voice-input/voiceRecordingPermission.ts new file mode 100644 index 0000000000000..21ff1df02dd7e --- /dev/null +++ b/apps/mobile/src/features/voice-input/voiceRecordingPermission.ts @@ -0,0 +1,36 @@ +import { requestRecordingPermissionsAsync } from "expo-audio"; +import { AppState, Platform } from "react-native"; + +export function createVoiceRecordingPermission(onBackground: () => void) { + let requesting = false; + let disposed = false; + let resume: (() => void) | null = null; + const subscription = AppState.addEventListener("change", (state) => { + if (state === "active") resume?.(); + if (state === "background" && !requesting) onBackground(); + }); + return { + requestPermission: async () => { + if (Platform.OS !== "android") return requestRecordingPermissionsAsync(); + // Android's permission activity backgrounds the app; resume before preparing Google's model. + requesting = true; + try { + const permission = await requestRecordingPermissionsAsync(); + if (!disposed && AppState.currentState !== "active") { + await new Promise((resolve) => { + resume = resolve; + }); + } + return permission; + } finally { + requesting = false; + resume = null; + } + }, + dispose: () => { + disposed = true; + subscription.remove(); + resume?.(); + }, + }; +} diff --git a/apps/mobile/src/lib/appLinking.test.ts b/apps/mobile/src/lib/appLinking.test.ts index 977a5a5e41ef5..8b873318d72aa 100644 --- a/apps/mobile/src/lib/appLinking.test.ts +++ b/apps/mobile/src/lib/appLinking.test.ts @@ -21,6 +21,7 @@ describe("shouldHandleAppLink", () => { "t3code://threads/env-1/thread-1", "t3code://pair?pairingUrl=x", "t3code-dev://settings/usage?tab=limits", + "https://app.t3.codes/threads/env/thread?prompt=Fix%20expo-development-client%20and%20t3code://expo-sharing", ])("handles path-bearing URL %s", (url) => { expect(shouldHandleAppLink(url, true)).toBe(true); }); diff --git a/apps/mobile/src/lib/appLinking.ts b/apps/mobile/src/lib/appLinking.ts index 9da768150f51b..570c86b36abf4 100644 --- a/apps/mobile/src/lib/appLinking.ts +++ b/apps/mobile/src/lib/appLinking.ts @@ -7,11 +7,13 @@ import type { NavigationState, PartialState } from "@react-navigation/native"; * to the NotFound wildcard route on every dev launch. */ export function shouldHandleAppLink(url: string, cloudConfigured: boolean): boolean { + // Only the route: a thread link's ?prompt= may mention either lifecycle URL. + const route = url.split(/[?#]/)[0] ?? ""; return ( - !url.includes("expo-development-client") && - !url.includes("://expo-sharing") && + !route.includes("expo-development-client") && + !route.includes("://expo-sharing") && !/^t3code(-dev|-preview)?:\/*$/.test(url) && - (cloudConfigured || !CLOUD_ONLY_LINK_PATH.test(url.split(/[?#]/)[0] ?? "")) + (cloudConfigured || !CLOUD_ONLY_LINK_PATH.test(route)) ); } diff --git a/apps/mobile/src/lib/composerContextClipboard.test.ts b/apps/mobile/src/lib/composerContextClipboard.test.ts index b3fea2e6ded7d..77305cce84439 100644 --- a/apps/mobile/src/lib/composerContextClipboard.test.ts +++ b/apps/mobile/src/lib/composerContextClipboard.test.ts @@ -42,6 +42,9 @@ vi.mock("../state/session", () => ({ environmentSession: { preparedConnectionValueAtom: vi.fn() }, })); vi.mock("../state/assets", () => ({ assetEnvironment: { createUrl: (value: unknown) => value } })); +vi.mock("../state/threads", () => ({ + environmentThreadShells: { threadShellAtom: (value: unknown) => value }, +})); vi.mock("../state/use-composer-drafts", () => ({ waitForComposerDraftsLoaded: async () => {}, findLocalComposerClipboardAttachment: mocks.local, @@ -336,4 +339,34 @@ describe("mobile context clipboard imports", () => { collectComposerContextReferences(result!.text).map((reference) => reference.contextId), ).toEqual(result!.context.records.map((record) => record.contextId)); }); + + it("pastes a thread link as a chip only where the destination's agent reads its machine", async () => { + const text = [ + "https://app.t3.codes/threads/destination/local-thread", + "", + ].join(" "); + const result = await importComposerContextClipboard( + { text, fragment: "", html: "" }, + 0, + new AbortController().signal, + 0, + EnvironmentId.make("destination"), + ); + const [record] = result!.context.records; + expect(record).toMatchObject({ + kind: "thread", + environmentId: "destination", + threadId: "local-thread", + }); + expect(result!.text).toBe( + `[Thread](t3-context://v1/thread/${record!.contextId}) `, + ); + expect( + await importComposerContextClipboard( + { text: "plain text", fragment: "", html: "" }, + 0, + new AbortController().signal, + ), + ).toBeNull(); + }); }); diff --git a/apps/mobile/src/lib/composerContextClipboard.ts b/apps/mobile/src/lib/composerContextClipboard.ts index 971191456cc20..8c5a31cdb7e44 100644 --- a/apps/mobile/src/lib/composerContextClipboard.ts +++ b/apps/mobile/src/lib/composerContextClipboard.ts @@ -13,13 +13,17 @@ import { decodeComposerContextFragment, encodeComposerContextFragment, } from "@t3tools/shared/composerContextClipboard"; -import { threadContextsOutsideEnvironment } from "@t3tools/shared/composerContextReferences"; +import { + threadContextsOutsideEnvironment, + threadLinkContextReferences, +} from "@t3tools/shared/composerContextReferences"; import { executeAtomQuery, squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; import { resolveAssetUrl } from "@t3tools/client-runtime/state/assets"; import * as Option from "effect/Option"; import { appAtomRegistry } from "../state/atom-registry"; import { assetEnvironment } from "../state/assets"; import { environmentSession } from "../state/session"; +import { environmentThreadShells } from "../state/threads"; import { workspaceConnections } from "../state/workspace"; import { downloadAttachmentForPreview } from "./attachmentDownload"; import { @@ -71,11 +75,38 @@ export async function importComposerContextClipboard( /** The machines the destination links to, whose threads its agent can read too. */ linkedEnvironments: ReadonlyMap = new Map(), ) { - const fragment = + const copied = decodeComposerContextFragment(input.fragment) ?? decodeComposerContextClipboardHtml(input.html); - if (!fragment) return null; - const selected = referencedComposerContext(input.text, { version: 1, records: fragment.records }); - const imported = reidentifyComposerContext(input.text, selected?.records ?? [], uuidv4); + // A thread link becomes a chip only where this agent can read its machine; others stay links. + const threads = threadLinkContextReferences( + input.text, + copied?.records ?? [], + (ref) => appAtomRegistry.get(environmentThreadShells.threadShellAtom(ref))?.title, + (record) => + destinationEnvironmentId === undefined || + record.environmentId === destinationEnvironmentId || + linkedEnvironments.has(record.environmentId) + ? record.contextId + : undefined, + ); + const source = + copied?.source ?? + (threads.records[0] ? { environmentId: threads.records[0].environmentId } : null); + if (!source) return null; + const fragment = { + source, + records: [ + ...(copied?.records ?? []).filter((record) => + threads.records.every((thread) => thread.contextId !== record.contextId), + ), + ...threads.records, + ], + }; + const selected = referencedComposerContext(threads.text, { + version: 1, + records: fragment.records, + }); + const imported = reidentifyComposerContext(threads.text, selected?.records ?? [], uuidv4); const skippedThreads = destinationEnvironmentId === undefined ? [] diff --git a/apps/mobile/src/lib/microphonePriority.test.ts b/apps/mobile/src/lib/microphonePriority.test.ts index fff3659057f69..d25ec96e7ba17 100644 --- a/apps/mobile/src/lib/microphonePriority.test.ts +++ b/apps/mobile/src/lib/microphonePriority.test.ts @@ -31,6 +31,12 @@ describe("microphoneOrder", () => { "kind:bluetooth", "kind:carPlay", ]); + expect(keys(microphoneOrder([{ kind: "carPlay" }, ...saved], "android"))).toEqual([ + "kind:wired", + "kind:builtIn", + "airpods", + "kind:bluetooth", + ]); }); }); @@ -75,5 +81,9 @@ describe("preferredMicrophoneInput", () => { expect(preferredMicrophoneInput([airPods, usb], order)).toBe(usb); expect(preferredMicrophoneInput([airPods, carPlay], order)).toBe(airPods); expect(preferredMicrophoneInput([airPods, iPhone], order)).toBe(iPhone); + const headset = { uid: "android:7:headset", name: "Headset", type: "BluetoothSCO" }; + expect(preferredMicrophoneInput([headset, iPhone], microphoneOrder([], "android"))).toBe( + headset, + ); }); }); diff --git a/apps/mobile/src/lib/microphonePriority.ts b/apps/mobile/src/lib/microphonePriority.ts index e505837fef313..fbe928f081ab2 100644 --- a/apps/mobile/src/lib/microphonePriority.ts +++ b/apps/mobile/src/lib/microphonePriority.ts @@ -1,11 +1,9 @@ -/** iOS lists connected inputs only after recording-session setup. */ const MICROPHONE_KINDS = ["wired", "bluetooth", "builtIn", "carPlay"] as const; export type MicrophoneKind = (typeof MICROPHONE_KINDS)[number]; export interface MicrophoneDevice { readonly kind: MicrophoneKind; - /** iOS's port identifier, stable across reconnects of the same device. */ readonly uid: string; readonly name: string; } @@ -18,12 +16,13 @@ interface MicrophoneInput { readonly type: string; } -// A car without CarPlay connects as a Bluetooth headset, so it is a Bluetooth device. +// A car without CarPlay, and Android Auto, routes its microphone over Bluetooth, so it is a Bluetooth device. const KIND_BY_PORT_TYPE: Readonly> = { MicrophoneWired: "wired", USBAudio: "wired", LineIn: "wired", BluetoothHFP: "bluetooth", + BluetoothSCO: "bluetooth", BluetoothLE: "bluetooth", MicrophoneBuiltIn: "builtIn", CarAudio: "carPlay", @@ -47,10 +46,13 @@ export function microphoneEntryKey(entry: MicrophoneEntry): string { export function microphoneOrder( saved: ReadonlyArray = [], + platform: "ios" | "android" = "ios", ): ReadonlyArray { + const kinds = MICROPHONE_KINDS.filter((kind) => platform === "ios" || kind !== "carPlay"); const order: MicrophoneEntry[] = []; const keys = new Set(); for (const entry of saved) { + if (!kinds.includes(entry.kind)) continue; const row = isMicrophoneDevice(entry) && DEVICE_KINDS.has(entry.kind) ? entry : { kind: entry.kind }; const key = microphoneEntryKey(row); @@ -58,7 +60,7 @@ export function microphoneOrder( keys.add(key); order.push(row); } - for (const kind of MICROPHONE_KINDS) { + for (const kind of kinds) { if (keys.has(`kind:${kind}`)) continue; const lastDevice = order.map((entry) => entry.kind).lastIndexOf(kind); const below = order.findIndex( @@ -95,7 +97,7 @@ export function rememberMicrophones( return next ?? order; } -/** The connected input of the first entry that has one, or null to leave the route to iOS. */ +/** The connected input of the first entry that has one, or null for the system default. */ export function preferredMicrophoneInput( inputs: ReadonlyArray, order: ReadonlyArray, diff --git a/apps/mobile/src/lib/permissionStatusAtom.test.ts b/apps/mobile/src/lib/permissionStatusAtom.test.ts index 052b2a37cb0f9..ec3330c8f54aa 100644 --- a/apps/mobile/src/lib/permissionStatusAtom.test.ts +++ b/apps/mobile/src/lib/permissionStatusAtom.test.ts @@ -26,6 +26,10 @@ function deferredReads() { pending.shift()?.(status); await Promise.resolve(); }, + resolveLast: async (status: string) => { + pending.pop()?.(status); + await Promise.resolve(); + }, get count() { return pending.length; }, @@ -68,6 +72,21 @@ describe("makePermissionStatusAtom", () => { expect(native.listeners.size).toBe(0); }); + it("keeps the newest read when an older one settles last", async () => { + const reads = deferredReads(); + const atom = makePermissionStatusAtom({ label: "test", initial: "checking", read: reads.read }); + const { registry, flush } = makeRegistry(); + const unmount = registry.mount(atom); + + for (const listener of native.listeners) listener("active"); + await reads.resolveLast("available"); + await reads.resolveNext("unavailable"); + expect(registry.get(atom)).toBe("available"); + + unmount(); + flush(); + }); + it("stores a request's outcome", () => { const reads = deferredReads(); const atom = makePermissionStatusAtom({ label: "test", initial: "checking", read: reads.read }); diff --git a/apps/mobile/src/lib/permissionStatusAtom.ts b/apps/mobile/src/lib/permissionStatusAtom.ts index ee8ff5dcdfecc..0aaded10957f7 100644 --- a/apps/mobile/src/lib/permissionStatusAtom.ts +++ b/apps/mobile/src/lib/permissionStatusAtom.ts @@ -3,9 +3,9 @@ import { Atom } from "effect/reactivity"; import { subscribeAppActive } from "./appState"; /** - * A system permission as the app last saw it. Reads it when first mounted and - * again each time the app returns to the foreground, where the user may have - * changed it in system Settings. Writing stores the outcome of a request. + * A system permission or device capability as the app last saw it. Reads it + * when first mounted and again each time the app returns to the foreground, + * where the user may have changed it in system Settings. Writing stores the outcome of a request. */ export function makePermissionStatusAtom(options: { readonly label: string; @@ -15,8 +15,13 @@ export function makePermissionStatusAtom(options: { return Atom.writable( (get) => { // A read that settles after unmount is dropped: setSelf on a disposed lifetime is a no-op. + // Only the latest read lands, so a slow earlier read cannot overwrite a newer status. + let latest = 0; const refresh = () => { - void options.read().then((status) => get.setSelf(status)); + const read = ++latest; + void options.read().then((status) => { + if (read === latest) get.setSelf(status); + }); }; refresh(); get.addFinalizer(subscribeAppActive(refresh)); diff --git a/apps/mobile/src/lib/threadActivity.test.ts b/apps/mobile/src/lib/threadActivity.test.ts index b4e875d8058eb..9779c5dc77559 100644 --- a/apps/mobile/src/lib/threadActivity.test.ts +++ b/apps/mobile/src/lib/threadActivity.test.ts @@ -18,6 +18,7 @@ import { type OrchestrationV2TurnItem, } from "@t3tools/contracts"; import { deriveThreadCheckpointSummaries } from "@t3tools/client-runtime/state/thread-checkpoints"; +import { summarizeToolGroup } from "@t3tools/client-runtime/work-log/presentation"; import { getReviewSectionIdForCheckpoint } from "../features/review/reviewModel"; import * as DateTime from "effect/DateTime"; import { describe, expect, it } from "vite-plus/test"; @@ -596,6 +597,24 @@ describe("buildThreadFeed", () => { expect(items[0]).toMatchObject({ output: rawOutput }); }); + it("counts every file of a multi-file patch", () => { + const item: OrchestrationV2TurnItem = { + ...base("multi-file-change", "2026-06-20T00:00:04.000Z", 3), + type: "file_change", + fileName: "src/a.ts", + changes: [ + { operation: "update", path: "src/a.ts" }, + { operation: "add", path: "src/b.ts" }, + { operation: "update", path: "src/renamed.ts", oldPath: "src/original.ts" }, + ], + }; + const activity = buildThreadFeed([projected(item, 0)]).flatMap((entry) => + entry.type === "activity-group" ? entry.activities : [], + )[0]!; + expect(workEntryRowLabel(activity.workEntry)).toBe("src/a.ts +2 more"); + expect(summarizeToolGroup([activity.workEntry]).summary).toBe("Changed 3 files"); + }); + it("anchors feedback before later committed turns", () => { const laterUser = { ...userMessage("2026-08-29T00:00:05.000Z"), diff --git a/apps/mobile/src/lib/threadActivity.ts b/apps/mobile/src/lib/threadActivity.ts index 2f1fabe19280e..527f8e1b5508f 100644 --- a/apps/mobile/src/lib/threadActivity.ts +++ b/apps/mobile/src/lib/threadActivity.ts @@ -780,7 +780,7 @@ function toWorkLogEntry( case "file_change": return { ...common, - changedFiles: [item.fileName], + changedFiles: item.changes?.map((change) => change.path) ?? [item.fileName], toolTitle: title ?? "File change", toolData: item, }; diff --git a/apps/mobile/src/native/voiceTranscription.android.test.ts b/apps/mobile/src/native/voiceTranscription.android.test.ts new file mode 100644 index 0000000000000..7a2e39e4304e8 --- /dev/null +++ b/apps/mobile/src/native/voiceTranscription.android.test.ts @@ -0,0 +1,84 @@ +import { expect, it, vi } from "vite-plus/test"; + +const mocks = vi.hoisted(() => ({ + checkAvailability: vi.fn<() => Promise>(), + prepare: vi.fn<(id: number) => Promise<{ locale: string; mode: number }>>(), + transcribe: vi.fn<(id: number, uri: string, locale: string, mode: number) => Promise>(), + cancel: vi.fn<(id: number) => Promise>(), +})); + +vi.mock("expo", () => ({ + requireOptionalNativeModule: () => ({ isAvailable: () => true, ...mocks }), +})); + +import { + VoiceInputController, + resetVoiceInputGlobalsForTests, +} from "../../../../packages/client-runtime/src/voice-input/controller"; + +import { getLocalVoiceTranscriber } from "./voiceTranscription.android"; + +it.each([false, true])( + "waits for device support before offering a transcriber (%s)", + async (supported) => { + vi.resetAllMocks(); + const probe = Promise.withResolvers(); + mocks.checkAvailability.mockReturnValue(probe.promise); + let settled = false; + const result = Promise.resolve(getLocalVoiceTranscriber()).then((transcriber) => { + settled = true; + return transcriber; + }); + await Promise.resolve(); + expect(settled).toBe(false); + probe.resolve(supported); + expect((await result) !== null).toBe(supported); + }, +); + +it.each([ + ["preparation", "storage", /disk|storage|space/i], + ["preparation", "quota", /quota|limit/i], + ["transcription", "storage", /disk|storage|space/i], + ["transcription", "quota", /quota|limit/i], +] as const)("preserves an actionable %s error: %s", async (phase, code, expected) => { + vi.resetAllMocks(); + resetVoiceInputGlobalsForTests(); + mocks.checkAvailability.mockResolvedValue(true); + mocks.prepare.mockResolvedValue({ locale: "en-US", mode: 0 }); + mocks.transcribe.mockResolvedValue("Complete transcript."); + mocks.cancel.mockResolvedValue(undefined); + const rejection = Object.assign(new Error("Native SDK failure"), { code }); + (phase === "preparation" ? mocks.prepare : mocks.transcribe).mockRejectedValue(rejection); + const transcriber = await getLocalVoiceTranscriber(); + const commit = vi.fn(); + const draft = { + ownerKey: "acceptance:dictation", + text: "Preserved draft.", + selection: { start: 16, end: 16 }, + revision: 1, + }; + const controller = new VoiceInputController({ + recorder: { + uri: "file:///acceptance.m4a", + prepareToRecordAsync: async () => {}, + record: () => {}, + stop: async () => {}, + }, + getTranscriber: () => transcriber, + requestPermission: async () => ({ granted: true, canAskAgain: true }), + configureRecording: async () => {}, + releaseRecording: async () => {}, + deleteRecording: () => {}, + readDraft: () => draft, + commitDraft: commit, + onStateChange: () => {}, + }); + await controller.start(); + if (phase === "transcription") await controller.stop(); + + expect(controller.currentState).toMatchObject({ phase: "error", errorAction: "retry" }); + expect(commit).not.toHaveBeenCalled(); + expect(draft.text).toBe("Preserved draft."); + expect(controller.currentState.error).toMatch(expected); +}); diff --git a/apps/mobile/src/native/voiceTranscription.android.ts b/apps/mobile/src/native/voiceTranscription.android.ts new file mode 100644 index 0000000000000..13e233d0cccfa --- /dev/null +++ b/apps/mobile/src/native/voiceTranscription.android.ts @@ -0,0 +1,88 @@ +import { requireOptionalNativeModule } from "expo"; + +import { + VoiceTranscriptionError, + throwIfVoiceTranscriptionAborted, + type VoiceTranscriber, +} from "@t3tools/client-runtime/voice-input"; + +type PreparedGoogleTranscription = { readonly locale: string; readonly mode: number }; + +type GoogleTranscriptionModule = { + checkAvailability(): Promise; + prepare(id: number): Promise; + transcribe(id: number, uri: string, locale: string, mode: number): Promise; + cancel(id: number): Promise; +}; + +const native = requireOptionalNativeModule("T3VoiceTranscription"); + +const nativeErrorMessages: Readonly> = { + unavailable: "On-device voice transcription is not available for this device language.", + storage: "Not enough storage for on-device dictation. Free device storage and retry.", + quota: "On-device speech quota reached. Let the device rest or charge it, then retry.", + "no-speech": "No speech was detected. Speak again and retry dictation.", + busy: "On-device speech is busy. Wait for it to finish, then retry dictation.", + "needs-update": "Update Android and its on-device speech services, then retry dictation.", + "download-failed": "Speech model download failed. Check your network connection and retry.", + "audio-overflow": + "The speech engine could not process the complete recording. No text was added. Retry dictation.", + "incomplete-audio": + "Speech recognition could not process the complete recording. No text was added. Retry dictation.", +}; + +let nextOperationId = 0; + +export async function getLocalVoiceTranscriber(): Promise { + if (!native || !(await native.checkAvailability())) return null; + const module = native; + return { + prepare: async ({ signal }) => { + const prepared = await runOperation(module, signal, "preparation-failed", (id) => + module.prepare(id), + ); + return { + locale: prepared.locale, + transcribe: (uri, { signal }) => + runOperation(module, signal, "transcription-failed", (id) => + module.transcribe(id, uri, prepared.locale, prepared.mode), + ), + }; + }, + }; +} + +async function runOperation( + module: GoogleTranscriptionModule, + signal: AbortSignal, + fallbackCode: "preparation-failed" | "transcription-failed", + operation: (id: number) => Promise, +): Promise { + throwIfVoiceTranscriptionAborted(signal); + const id = ++nextOperationId; + let cancellation: Promise | undefined; + const abort = () => { + cancellation = module.cancel(id).catch(() => {}); + }; + signal.addEventListener("abort", abort, { once: true }); + try { + const result = await operation(id); + throwIfVoiceTranscriptionAborted(signal); + return result; + } catch (error) { + throwIfVoiceTranscriptionAborted(signal); + const code = + typeof error === "object" && error !== null && "code" in error ? error.code : undefined; + throw new VoiceTranscriptionError( + code === "unavailable" || code === "unsupported-locale" ? code : fallbackCode, + (typeof code === "string" ? nativeErrorMessages[code] : undefined) ?? + (fallbackCode === "preparation-failed" + ? "Voice transcription could not prepare this language. Check Google speech services and try again." + : "Voice transcription failed."), + { cause: error }, + ); + } finally { + signal.removeEventListener("abort", abort); + await cancellation; + } +} diff --git a/apps/mobile/src/native/voiceTranscription.ios.test.ts b/apps/mobile/src/native/voiceTranscription.ios.test.ts index d3ee19c262de6..d83c371bc2a90 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.test.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.test.ts @@ -65,7 +65,7 @@ describe("getLocalVoiceTranscriber", () => { }); const controller = new AbortController(); const options = { signal: controller.signal }; - const prepared = await getLocalVoiceTranscriber()!.prepare(options); + const prepared = await (await getLocalVoiceTranscriber())!.prepare(options); const result = prepared .transcribe("file:///voice.m4a", options) .catch((error: unknown) => error); diff --git a/apps/mobile/src/native/voiceTranscription.ios.ts b/apps/mobile/src/native/voiceTranscription.ios.ts index 216b9e958dd66..9f9556645101d 100644 --- a/apps/mobile/src/native/voiceTranscription.ios.ts +++ b/apps/mobile/src/native/voiceTranscription.ios.ts @@ -33,7 +33,7 @@ function getNativeErrorCode(error: unknown): string | undefined { return typeof error.code === "string" ? error.code : undefined; } -export function getLocalVoiceTranscriber(): VoiceTranscriber | null { +export async function getLocalVoiceTranscriber(): Promise { const locale = getDeviceLocale(); if (!AppleTranscription.isAvailable(locale)) return null; return { prepare: (options) => prepareVoiceTranscription(locale, options) }; diff --git a/apps/mobile/src/native/voiceTranscription.ts b/apps/mobile/src/native/voiceTranscription.ts index e003064ae3f84..496a862836a19 100644 --- a/apps/mobile/src/native/voiceTranscription.ts +++ b/apps/mobile/src/native/voiceTranscription.ts @@ -1,5 +1,5 @@ import type { VoiceTranscriber } from "@t3tools/client-runtime/voice-input"; -export function getLocalVoiceTranscriber(): VoiceTranscriber | null { +export async function getLocalVoiceTranscriber(): Promise { return null; } diff --git a/apps/server/src/attachmentStore.ts b/apps/server/src/attachmentStore.ts index 5d8ab2e90181b..a2b531e3fa5b0 100644 --- a/apps/server/src/attachmentStore.ts +++ b/apps/server/src/attachmentStore.ts @@ -4,7 +4,11 @@ import * as NodeFS from "node:fs"; import * as NodePath from "node:path"; import type { ChatAttachment } from "@t3tools/contracts"; -import { htmlRenderFromToolItem, mcpAppFromToolItem } from "@t3tools/shared/toolOutput"; +import { + htmlRenderFromToolItem, + mcpAppFromToolItem, + previewRecordingIdFromToolItem, +} from "@t3tools/shared/toolOutput"; import { normalizeAttachmentRelativePath, @@ -13,7 +17,7 @@ import { import { inferImageExtension, SAFE_IMAGE_FILE_EXTENSIONS } from "./imageMime.ts"; const ATTACHMENT_FILENAME_EXTENSIONS = [...SAFE_IMAGE_FILE_EXTENSIONS, ".bin"]; -const ATTACHMENT_ID_THREAD_SEGMENT_MAX_CHARS = 80; +export const ATTACHMENT_ID_THREAD_SEGMENT_MAX_CHARS = 80; const ATTACHMENT_ID_THREAD_SEGMENT_PATTERN = "[a-z0-9_]+(?:-[a-z0-9_]+)*"; const ATTACHMENT_ID_UUID_PATTERN = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; const ATTACHMENT_ID_FILE_EXTENSION_PATTERN = "[a-z0-9]{1,10}"; @@ -133,7 +137,9 @@ export function threadHtmlRenderAttachmentIds( if (segment === null) return []; return Array.from(items).flatMap((item) => { const attachmentId = - htmlRenderFromToolItem(item)?.attachmentId ?? mcpAppFromToolItem(item)?.attachmentId; + htmlRenderFromToolItem(item)?.attachmentId ?? + mcpAppFromToolItem(item)?.attachmentId ?? + previewRecordingIdFromToolItem(item); return attachmentId !== undefined && parseThreadSegmentFromAttachmentId(attachmentId) === segment ? [attachmentId] diff --git a/apps/server/src/auth/McpOAuth.ts b/apps/server/src/auth/McpOAuth.ts index 0fb7f10d2ba14..3f4d7ff51f803 100644 --- a/apps/server/src/auth/McpOAuth.ts +++ b/apps/server/src/auth/McpOAuth.ts @@ -17,6 +17,7 @@ import { mcpClientScopes, type RuntimeMode, } from "@t3tools/contracts"; +import { DEFAULT_HOSTED_APP_URL } from "@t3tools/shared/connectAuth"; import { encodeOAuthScope } from "@t3tools/shared/oauthScope"; import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; @@ -30,6 +31,7 @@ import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import { HttpServerRequest } from "effect/http"; +import { hostedAppUrlConfig } from "../cloud/publicConfig.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as McpHttpServer from "../mcp/McpHttpServer.ts"; @@ -577,6 +579,13 @@ export const layerMcpClientAuthenticator = Layer.effect( const environmentAuth = yield* EnvironmentAuth.EnvironmentAuth; const environment = yield* ServerEnvironment.ServerEnvironment; const environmentId = yield* environment.getEnvironmentId; + const hostedAppUrl = yield* hostedAppUrlConfig.pipe( + Effect.catch((cause) => + Effect.logWarning("Outside MCP clients get app.t3.codes thread links", { cause }).pipe( + Effect.as(DEFAULT_HOSTED_APP_URL), + ), + ), + ); return McpHttpServer.McpClientAuthenticator.of({ authenticate: Effect.fn("McpHttpServer.McpClientAuthenticator.authenticate")( function (request) { @@ -592,7 +601,7 @@ export const layerMcpClientAuthenticator = Layer.effect( label: client.label, access: client.access, linked: client.peerLink, - ...(client.peerLink ? {} : { webOrigin: requestUrls(request).issuer }), + ...(client.peerLink ? {} : { hostedAppUrl }), }, capabilities: new Set([ "orchestration", diff --git a/apps/server/src/checkpointing/CheckpointStore.test.ts b/apps/server/src/checkpointing/CheckpointStore.test.ts index 51c461d4a6d59..36e58024adbae 100644 --- a/apps/server/src/checkpointing/CheckpointStore.test.ts +++ b/apps/server/src/checkpointing/CheckpointStore.test.ts @@ -159,6 +159,10 @@ it.layer(layerTest)("CheckpointStore.layer", (it) => { yield* initRepoWithCommit(tmp); yield* git(tmp, ["config", "diff.noprefix", "true"]); yield* git(tmp, ["config", "diff.suppressBlankEmpty", "true"]); + yield* git(tmp, ["config", "diff.submodule", "log"]); + const sub = NodePath.join(tmp, "sub"); + yield* git(tmp, ["init", "sub"]); + yield* initRepoWithCommit(sub); const checkpointStore = yield* CheckpointStore.CheckpointStore; const threadId = ThreadId.make("thread-checkpoint-store-noprefix"); const fromCheckpointRef = checkpointRefForThreadTurn(threadId, 0); @@ -170,6 +174,7 @@ it.layer(layerTest)("CheckpointStore.layer", (it) => { checkpointRef: fromCheckpointRef, }); yield* writeTextFile(NodePath.join(tmp, "README.md"), "# test\n\nafter\n"); + yield* git(sub, ["commit", "--allow-empty", "-m", "moved"]); yield* checkpointStore.captureCheckpoint({ cwd: tmp, checkpointRef: toCheckpointRef, @@ -184,6 +189,8 @@ it.layer(layerTest)("CheckpointStore.layer", (it) => { expect(diff).toContain("diff --git a/README.md b/README.md"); expect(diff).toContain(" # test\n \n-before\n+after\n"); + expect(diff).toContain("diff --git a/sub b/sub"); + expect(diff).toContain("+Subproject commit "); }), ); diff --git a/apps/server/src/cli/automaticPort.test.ts b/apps/server/src/cli/automaticPort.test.ts new file mode 100644 index 0000000000000..1f7805569744f --- /dev/null +++ b/apps/server/src/cli/automaticPort.test.ts @@ -0,0 +1,165 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as NetService from "@t3tools/shared/Net"; +import * as Effect from "effect/Effect"; +import * as CliError from "effect/cli/CliError"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as PlatformError from "effect/PlatformError"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; + +import * as ProcessRunner from "../processRunner.ts"; +import * as AutomaticPort from "./automaticPort.ts"; + +const WSL_KERNEL = "6.6.87.2-microsoft-standard-WSL2\n"; +const NETSTAT_PATH = "/windows/c/Windows/System32/netstat.exe"; + +const output = (stdout: string, code: number | null = null): ProcessRunner.ProcessRunOutput => ({ + stdout, + stderr: "", + code: code === null ? null : ChildProcessSpawner.ExitCode(code), + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + stdoutInvalidUtf8: false, + stderrInvalidUtf8: false, +}); + +const listening = (ports: ReadonlyArray, host = "127.0.0.1", foreign = "0.0.0.0:0") => + ports.map((port) => ` TCP ${host}:${port} ${foreign} ABHÖREN 4242`).join("\r\n"); + +interface Scenario { + readonly kernelRelease?: string; + readonly windowsPorts?: ReadonlyArray; + readonly windowsPortsV6?: ReadonlyArray; + readonly tcpv6ExitCode?: number; + readonly wslpathMissing?: boolean; + readonly reservedPorts?: ReadonlyArray; +} + +const select = (scenario: Scenario) => { + const commands: string[] = []; + const reserved = [...(scenario.reservedPorts ?? [50_001])]; + const net: NetService.NetServiceShape = { + canListenOnHost: () => Effect.succeed(true), + hasListenerOnHost: () => Effect.succeed(false), + isPortAvailableOnLoopback: () => Effect.succeed(true), + reserveLoopbackPort: () => Effect.sync(() => reserved.shift() ?? 60_000), + findAvailablePort: Effect.succeed, + }; + const run: ProcessRunner.ProcessRunner["Service"]["run"] = (input) => { + commands.push(input.command); + if (input.command === "wslpath") { + if (scenario.wslpathMissing) { + return Effect.fail( + new ProcessRunner.ProcessSpawnError({ + command: input.command, + argumentCount: input.args.length, + cause: PlatformError.systemError({ + _tag: "NotFound", + module: "ChildProcess", + method: "spawn", + }), + }), + ); + } + return Effect.succeed(output(`${NETSTAT_PATH}\n`)); + } + if (input.args.includes("tcpv6")) { + return Effect.succeed( + output( + listening(scenario.windowsPortsV6 ?? [], "[::1]", "[::]:0"), + scenario.tcpv6ExitCode ?? null, + ), + ); + } + return Effect.succeed(output(listening(scenario.windowsPorts ?? []))); + }; + const layer = Layer.mergeAll( + Layer.succeed(NetService.NetService, net), + Layer.succeed(ProcessRunner.ProcessRunner, { run }), + FileSystem.layerNoop({ + readFileString: (path) => + path === "/proc/sys/kernel/osrelease" && scenario.kernelRelease !== undefined + ? Effect.succeed(scenario.kernelRelease) + : Effect.fail( + PlatformError.systemError({ + _tag: "NotFound", + module: "FileSystem", + method: "readFileString", + }), + ), + }), + ); + return Effect.gen(function* () { + const selectAutomaticPort = yield* AutomaticPort.make; + const port = yield* selectAutomaticPort(3773); + return { port, commands }; + }).pipe(Effect.provide(layer)); +}; + +describe("automatic port selection", () => { + it.effect("moves off the default port only when Windows already listens on it", () => + Effect.gen(function* () { + const free = yield* select({ kernelRelease: WSL_KERNEL, windowsPorts: [8080] }); + expect(free.port).toBe(3773); + expect(free.commands).toEqual(["wslpath", NETSTAT_PATH, NETSTAT_PATH]); + + const owned = yield* select({ kernelRelease: WSL_KERNEL, windowsPorts: [3773] }); + expect(owned.port).toBe(50_001); + }), + ); + + it.effect("retries a fallback port that Windows already listens on", () => + Effect.gen(function* () { + const result = yield* select({ + kernelRelease: WSL_KERNEL, + windowsPorts: [3773], + windowsPortsV6: [50_001], + reservedPorts: [50_001, 50_002], + }); + expect(result.port).toBe(50_002); + }), + ); + + it.effect("fails with an actionable error when every fallback belongs to Windows", () => + Effect.gen(function* () { + const ports = Array.from({ length: 8 }, (_, index) => 50_001 + index); + const error = yield* select({ + kernelRelease: WSL_KERNEL, + windowsPorts: [3773], + windowsPortsV6: ports, + reservedPorts: ports, + }).pipe(Effect.flip); + expect(error).toBeInstanceOf(CliError.UserError); + expect(error.cause).toContain("--port"); + }), + ); + + it.effect("keeps the local choice when the Windows snapshot is unavailable", () => + Effect.gen(function* () { + const partial = yield* select({ + kernelRelease: WSL_KERNEL, + windowsPorts: [3773], + tcpv6ExitCode: 1, + }); + expect(partial.port).toBe(3773); + + const noInterop = yield* select({ kernelRelease: WSL_KERNEL, wslpathMissing: true }); + expect(noInterop.port).toBe(3773); + }), + ); + + it.effect("never queries Windows outside WSL", () => + Effect.gen(function* () { + for (const scenario of [ + { kernelRelease: "6.8.0-45-generic\n" }, + // macOS and Windows have no kernel release file. + {}, + ] satisfies ReadonlyArray) { + const result = yield* select(scenario); + expect(result.port).toBe(3773); + expect(result.commands).toEqual([]); + } + }), + ); +}); diff --git a/apps/server/src/cli/automaticPort.ts b/apps/server/src/cli/automaticPort.ts new file mode 100644 index 0000000000000..354de65d2519b --- /dev/null +++ b/apps/server/src/cli/automaticPort.ts @@ -0,0 +1,69 @@ +import * as NetService from "@t3tools/shared/Net"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as CliError from "effect/cli/CliError"; + +import { queryWindowsTcpListeners } from "../process/windowsTcpListeners.ts"; +import * as ProcessRunner from "../processRunner.ts"; + +const KERNEL_RELEASE_PATH = "/proc/sys/kernel/osrelease"; +const WINDOWS_NETSTAT_PATH = String.raw`C:\Windows\System32\netstat.exe`; +const FALLBACK_ATTEMPTS = 8; + +/** + * Picks the startup port when none was configured. Under WSL's default NAT networking, Windows + * reaches the server through localhost forwarding, which fails for a port a Windows process + * already listens on even though WSL can bind it. + */ +export const make = Effect.gen(function* () { + const net = yield* NetService.NetService; + const fs = yield* FileSystem.FileSystem; + const processRunner = yield* ProcessRunner.ProcessRunner; + + // Services started by systemd lack WSL_DISTRO_NAME and WSL_INTEROP; the kernel release does not. + // Hosts without this file (macOS, Windows) are not WSL. + const isWsl = fs.readFileString(KERNEL_RELEASE_PATH).pipe( + Effect.map((release) => release.toLowerCase().includes("microsoft")), + Effect.orElseSucceed(() => false), + ); + + const readWindowsHostListeners = Effect.gen(function* () { + // The automount root is configurable, so `/mnt/c` is not guaranteed. + const netstat = yield* processRunner.run({ + command: "wslpath", + args: ["-u", WINDOWS_NETSTAT_PATH], + timeout: Duration.seconds(5), + }); + const netstatPath = netstat.stdout.trim(); + if ((netstat.code !== null && netstat.code !== 0) || netstatPath.length === 0) { + return null; + } + const listeners = yield* queryWindowsTcpListeners(processRunner, netstatPath); + if (listeners === null) return null; + return new Set(listeners.map((listener) => listener.port)); + }).pipe(Effect.orElseSucceed(() => null)); + + return Effect.fn("automaticPort.select")(function* (preferred: number) { + if (!(yield* isWsl)) return yield* net.findAvailablePort(preferred); + const windows = yield* readWindowsHostListeners; + if (windows === null) { + yield* Effect.logWarning( + "Could not list Windows TCP listeners from WSL; Windows may be unable to reach the selected port. Pass --port to choose one.", + ); + return yield* net.findAvailablePort(preferred); + } + if (!windows.has(preferred) && (yield* net.isPortAvailableOnLoopback(preferred))) { + return preferred; + } + // Linux and Windows ephemeral ranges overlap, so fallback ports can collide too. + for (let attempt = 0; attempt < FALLBACK_ATTEMPTS; attempt += 1) { + const port = yield* net.reserveLoopbackPort(); + if (!windows.has(port)) return port; + } + return yield* new CliError.UserError({ + cause: + "Windows is already using every port T3 Code tried. Pass --port or set T3CODE_PORT to a port Windows is not using.", + }); + }); +}); diff --git a/apps/server/src/cli/config.test.ts b/apps/server/src/cli/config.test.ts index 297b225b6adce..5e09dda2c164f 100644 --- a/apps/server/src/cli/config.test.ts +++ b/apps/server/src/cli/config.test.ts @@ -754,6 +754,43 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => { lan: Option.none(), }); + it.effect("uses the automatic port selector only when no port is configured", () => + Effect.gen(function* () { + const baseDir = yield* FileSystem.FileSystem.pipe( + Effect.flatMap((fs) => fs.makeTempDirectoryScoped({ prefix: "t3-cli-auto-port-" })), + ); + const selected: number[] = []; + const selectAutomaticPort = (preferred: number) => + Effect.sync(() => { + selected.push(preferred); + return 50_123; + }); + const resolvePort = (flags: { + readonly mode?: Option.Option<"web" | "desktop">; + readonly port: Option.Option; + }) => + resolveServerConfig({ ...minimalWebFlags(baseDir), ...flags }, Option.none(), { + selectAutomaticPort, + }).pipe( + Effect.map((config) => config.port), + Effect.provide( + Layer.merge( + NetService.layer, + ConfigProvider.layer(ConfigProvider.fromEnv({ env: {} })), + ), + ), + ); + + expect(yield* resolvePort({ port: Option.none() })).toBe(50_123); + expect(selected).toEqual([3773]); + expect(yield* resolvePort({ port: Option.some(8788) })).toBe(8788); + expect( + yield* resolvePort({ mode: Option.some("desktop" as const), port: Option.none() }), + ).toBe(3773); + expect(selected).toEqual([3773]); + }), + ); + it.effect( "resolves each signal's endpoint through T3CODE_OTLP_*_URL, an OTEL endpoint, the bootstrap envelope, and persisted Settings, in that order", () => diff --git a/apps/server/src/cli/config.ts b/apps/server/src/cli/config.ts index f86fde0f59750..00aae0f3133b7 100644 --- a/apps/server/src/cli/config.ts +++ b/apps/server/src/cli/config.ts @@ -267,9 +267,13 @@ export const resolveServerConfig = Effect.fn("config.resolveServerConfig")(funct readonly startupPresentation?: ServerConfig.StartupPresentation; readonly forceAutoBootstrapProjectFromCwd?: boolean; readonly rejectRunningServer?: boolean; + readonly selectAutomaticPort?: ( + preferred: number, + ) => Effect.Effect; }, ) { const { findAvailablePort } = yield* NetService.NetService; + const selectAutomaticPort = options?.selectAutomaticPort ?? findAvailablePort; const path = yield* Path.Path; const fs = yield* FileSystem.FileSystem; const env = yield* EnvServerConfig; @@ -316,7 +320,7 @@ export const resolveServerConfig = Effect.fn("config.resolveServerConfig")(funct if (mode === "desktop") { return Effect.succeed(ServerConfig.DEFAULT_PORT); } - return findAvailablePort(ServerConfig.DEFAULT_PORT); + return selectAutomaticPort(ServerConfig.DEFAULT_PORT); }, }, ); diff --git a/apps/server/src/cli/policy.ts b/apps/server/src/cli/policy.ts index 99275968fd609..b56c5a0ef60cd 100644 --- a/apps/server/src/cli/policy.ts +++ b/apps/server/src/cli/policy.ts @@ -1,13 +1,17 @@ import * as Console from "effect/Console"; +import type { ManagedPolicyDocument } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as References from "effect/References"; -import { Argument, Command, GlobalFlag } from "effect/cli"; +import { Argument, Command, Flag, GlobalFlag } from "effect/cli"; import * as CliError from "effect/cli/CliError"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as ManagedPolicy from "../policy/ManagedPolicy.ts"; +import { macosProfile, windowsAdml, windowsAdmx } from "../policy/policyTemplates.ts"; import { isProcessAlive, readPersistedServerRuntimeState } from "../serverRuntimeState.ts"; import { projectLocationFlags, resolveCliAuthConfig } from "./config.ts"; @@ -103,7 +107,62 @@ const checkCommand = Command.make("check", { ), ); +const SAMPLE_POLICY: ManagedPolicyDocument = { + schemaVersion: 1, + organization: "Example Corp", + limits: { maxRuntimeMode: "auto-accept-edits", clients: ["desktop"], telemetry: false }, +}; + +const templatesCommand = Command.make("templates", { + directory: Argument.String("directory").pipe( + Argument.withDescription("Directory to write the templates to."), + ), + policy: Flag.String("policy").pipe( + Flag.withDescription("Policy file to put in the macOS profile. Default: a small sample."), + Flag.optional, + ), +}).pipe( + Command.withDescription( + "Write the Windows Group Policy template and a macOS configuration profile.", + ), + Command.withHandler( + Effect.fn("cli.policy.templates")(function* ({ directory, policy }) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + let document = SAMPLE_POLICY; + if (Option.isSome(policy)) { + if (!(yield* fs.exists(policy.value))) { + return yield* new CliError.UserError({ + cause: policy.value, + userMessage: `No policy file at ${policy.value}.`, + }); + } + const loaded = (yield* ManagedPolicy.load(policy.value)).document; + if (loaded !== undefined) document = loaded; + } + const random = yield* SecureRandom.SecureRandom; + const profile = macosProfile(document, { + profile: yield* random.uuidV4().pipe(Effect.orDie), + payload: yield* random.uuidV4().pipe(Effect.orDie), + }); + if (profile._tag === "Invalid") { + return yield* new CliError.UserError({ cause: document, userMessage: profile.reason }); + } + const files = [ + [path.join(directory, "windows", "t3code.admx"), windowsAdmx()], + [path.join(directory, "windows", "en-US", "t3code.adml"), windowsAdml()], + [path.join(directory, "macos", "t3code.mobileconfig"), profile.profile], + ] as const; + for (const [file, contents] of files) { + yield* fs.makeDirectory(path.dirname(file), { recursive: true }); + yield* fs.writeFileString(file, contents); + yield* Console.log(file); + } + }), + ), +); + export const policyCommand = Command.make("policy").pipe( Command.withDescription("Inspect the administrator's managed policy."), - Command.withSubcommands([showCommand, checkCommand]), + Command.withSubcommands([showCommand, checkCommand, templatesCommand]), ); diff --git a/apps/server/src/cli/server.ts b/apps/server/src/cli/server.ts index cb21a08140e4b..3b2768499fdb6 100644 --- a/apps/server/src/cli/server.ts +++ b/apps/server/src/cli/server.ts @@ -8,6 +8,8 @@ import * as CliError from "effect/cli/CliError"; import * as ServerConfig from "../config.ts"; import { awaitLauncherLoss } from "../launcherWatch.ts"; +import * as ProcessRunner from "../processRunner.ts"; +import * as AutomaticPort from "./automaticPort.ts"; import { type CliServerFlags, resolveServerConfig, sharedServerCommandFlags } from "./config.ts"; const encodeCommand = Schema.encodeEffect(Schema.fromJsonString(Schema.String)); @@ -30,7 +32,8 @@ const runServerCommand = Effect.fn("server.runServerCommand")(function* ( // Before config resolution, which claims a port and creates directories. const { runServer } = yield* Effect.promise(() => import("../server.ts")); const logLevel = yield* GlobalFlag.LogLevel; - const config = yield* resolveServerConfig(flags, logLevel, options); + const selectAutomaticPort = yield* AutomaticPort.make.pipe(Effect.provide(ProcessRunner.layer)); + const config = yield* resolveServerConfig(flags, logLevel, { ...options, selectAutomaticPort }); const run = runServer.pipe(Effect.provideService(ServerConfig.ServerConfig, config)); if (config.mode !== "desktop" || config.desktopBootstrapToken === undefined) return yield* run; yield* Effect.sync(() => { diff --git a/apps/server/src/cli/service.test.ts b/apps/server/src/cli/service.test.ts index 4358165958a1b..7ab52e82a3995 100644 --- a/apps/server/src/cli/service.test.ts +++ b/apps/server/src/cli/service.test.ts @@ -32,6 +32,7 @@ function makeTestService( ) { const installOptions: Array[0]> = []; const restarts: Array = []; + const uninstalls: Array = []; const service = BootService.BootService.of({ status: Effect.succeed(serviceStatus), withStopped: (operation) => operation, @@ -49,9 +50,12 @@ function makeTestService( logPath: serviceStatus.logPath, }; }), - uninstall: Effect.succeed(false), + uninstall: Effect.sync(() => { + uninstalls.push(true); + return serviceStatus.installed; + }), }); - return { service, installOptions, restarts }; + return { service, installOptions, restarts, uninstalls }; } it.layer(Layer.mergeAll(NodeServices.layer, NetService.layer))("service commands", (it) => { @@ -194,4 +198,48 @@ it.layer(Layer.mergeAll(NodeServices.layer, NetService.layer))("service commands expect(installOptions).toEqual([]); }), ); + + it.effect.each([ + { label: "this home's service", otherHome: false, unreadable: false, removed: [true] }, + { label: "another home's service", otherHome: true, unreadable: false, removed: [] }, + { label: "an unreadable service", otherHome: false, unreadable: true, removed: [true] }, + ])("uninstall removes only $label", (input) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-cli-test-" }); + const { service, uninstalls } = makeTestService({ + ...status, + installedBaseDir: input.otherHome ? "/home/me/.t3" : baseDir, + }); + vi.spyOn(BootService, "layer").mockReturnValue( + Layer.succeed( + BootService.BootService, + input.unreadable + ? { + ...service, + status: Effect.fail(new BootService.BootServiceInstallError({ cause: "EACCES" })), + } + : service, + ), + ); + + const output = yield* Command.runWith(serviceCommand, { version: packageJson.version })([ + "uninstall", + "--base-dir", + baseDir, + ]).pipe( + Effect.andThen(TestConsole.logLines), + Effect.map((lines) => lines.join("\n")), + Effect.provideService(HostProcess.Environment, {}), + Effect.provide( + Layer.mergeAll( + ConfigProvider.layer(ConfigProvider.fromEnv({ env: {} })), + Layer.fresh(TestConsole.layer), + ), + ), + ); + expect(uninstalls).toEqual(input.removed); + expect(output).toContain(input.otherHome ? "another T3 home" : "Removed"); + }), + ); }); diff --git a/apps/server/src/cli/service.ts b/apps/server/src/cli/service.ts index b78ee31c9fa4d..4ba17aa17c6de 100644 --- a/apps/server/src/cli/service.ts +++ b/apps/server/src/cli/service.ts @@ -278,6 +278,22 @@ const serviceUninstallCommand = Command.make("uninstall", projectLocationFlags). flags, Effect.gen(function* () { const service = yield* BootService.BootService; + const path = yield* Path.Path; + const { baseDir } = yield* ServerConfig.ServerConfig; + // An unreadable unit stays removable; only a unit known to serve another home is kept. + const installedBaseDir = yield* service.status.pipe( + Effect.map((status) => status.installedBaseDir), + Effect.orElseSucceed(() => undefined), + ); + if ( + installedBaseDir !== undefined && + path.resolve(installedBaseDir) !== path.resolve(baseDir) + ) { + yield* Console.log( + "The installed service runs another T3 home, so it was not removed. Pass its --base-dir.", + ); + return; + } const removed = yield* service.uninstall; yield* Console.log( removed ? "Removed the T3 Code service." : "T3 Code service is not installed.", diff --git a/apps/server/src/cli/up.test.ts b/apps/server/src/cli/up.test.ts index aa32442f451c7..10fb3b421d247 100644 --- a/apps/server/src/cli/up.test.ts +++ b/apps/server/src/cli/up.test.ts @@ -182,6 +182,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer, NetService.layer)) }, { label: "the local network opens", + prints: "Run `t3 pair --base-dir `.", installed: true, env: "T3CODE_LAN=false\n", args: ["--lan"], @@ -337,7 +338,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer, NetService.layer)) ), ); if (input.prints !== undefined) { - expect(lines.join("\n")).toContain(input.prints); + expect(lines.join("\n")).toContain(input.prints.replace("", baseDir)); } expect(exit._tag).toBe(input.restartFails === true ? "Failure" : "Success"); expect({ installs, restarts }).toEqual({ diff --git a/apps/server/src/cli/up.ts b/apps/server/src/cli/up.ts index b04b6fea7d331..5a0e820958c15 100644 --- a/apps/server/src/cli/up.ts +++ b/apps/server/src/cli/up.ts @@ -171,12 +171,12 @@ const formatSetup = (setup: Setup) => { ].join("\n"); }; -const formatNextSteps = (routes: Routes, serviceInstalled: boolean) => +const formatNextSteps = (routes: Routes, serviceInstalled: boolean, pairCommand: string) => [ ...(routes.connect ? ["On your other device, sign in to T3 Connect with the same account."] : []), - ...(routes.lan ? ["Run `t3 pair`. Use the code on your other device."] : []), + ...(routes.lan ? [`Run \`${pairCommand}\`. Use the code on your other device.`] : []), ...(routes.connect || routes.lan ? [] : [ @@ -197,6 +197,11 @@ const setUp = Effect.fn("cli.up")(function* (flags: { }) { const { baseDir } = yield* ServerConfig.ServerConfig; const service = yield* BootService.BootService; + const baseDirArgs = Option.match(flags.baseDir, { + onNone: () => [], + onSome: (dir) => ["--base-dir", dir], + }); + const { command: pairCommand } = yield* resolveHostCliCommand("pair", baseDirArgs); const before = yield* readSetup(); if (before.connect === null && Option.getOrElse(flags.connect, () => false)) { return yield* new CliError.UserError({ @@ -252,7 +257,7 @@ const setUp = Effect.fn("cli.up")(function* (flags: { yield* disconnectCloud({ clearAuthorization: false }); } if (current.lan) yield* CliService.writeServiceEnv(baseDir, { T3CODE_LAN: "false" }); - return yield* Console.log(formatNextSteps(routes, false)); + return yield* Console.log(formatNextSteps(routes, false, pairCommand)); } const { installedVersion } = before.service; const newer = @@ -327,7 +332,7 @@ const setUp = Effect.fn("cli.up")(function* (flags: { const printServeCommand = Effect.fn(function* (reason: string) { const { command } = yield* resolveHostCliCommand("serve", [ - ...Option.match(flags.baseDir, { onNone: () => [], onSome: (dir) => ["--base-dir", dir] }), + ...baseDirArgs, // Explicit either way, so a T3CODE_LAN in the shell cannot override the choice. routes.lan ? "--lan" : "--no-lan", ]); @@ -408,6 +413,7 @@ const setUp = Effect.fn("cli.up")(function* (flags: { `\nThis machine\n${formatSetup(after)}\n\n${formatNextSteps( { connect: after.connect === true, lan: after.lan }, after.service.installed, + pairCommand, )}`, ); }); diff --git a/apps/server/src/cli/update.test.ts b/apps/server/src/cli/update.test.ts index 848c2c798df41..7540c678f59d4 100644 --- a/apps/server/src/cli/update.test.ts +++ b/apps/server/src/cli/update.test.ts @@ -10,15 +10,23 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as TestClock from "effect/testing/TestClock"; +import * as TestConsole from "effect/testing/TestConsole"; import { HttpClient, HttpClientResponse } from "effect/http"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import { afterEach, vi } from "vite-plus/test"; +import packageJson from "../../package.json" with { type: "json" }; import * as BootService from "../cloud/bootService.ts"; -import { prunePinnedRuntimes } from "../cloud/pinnedRuntime.ts"; +import { pinnedRuntimePaths, prunePinnedRuntimes } from "../cloud/pinnedRuntime.ts"; import * as ManagedPolicy from "../policy/ManagedPolicy.ts"; import * as ProcessRunner from "../processRunner.ts"; -import { repointLauncher, resolveLauncherPath, resolveNewestVersion, runUpdate } from "./update.ts"; +import { + repointLauncher, + resolveLauncherPath, + resolveMirrorVersion, + resolveNewestVersion, + runUpdate, +} from "./update.ts"; // lsof listing one process, so prune sees a process list in which no runtime is running. const unrelatedLsof = ProcessRunner.ProcessRunner.of({ @@ -60,10 +68,59 @@ it.effect("authenticates each release page until the requested channel is found" }), ); +it.effect("reads the newest version from the update feed file a mirror publishes", () => + Effect.gen(function* () { + const requested: string[] = []; + const mirror = { + organization: "Example Corp", + feedUrl: "https://updates.example.com/t3code/?token=abc", + }; + const resolve = ( + channel: "stable" | "nightly" | "preview", + platform: NodeJS.Platform, + arch: string, + feed: string, + ) => + resolveMirrorVersion(mirror, channel).pipe( + Effect.map((version): string => version), + Effect.catchTags({ CliUpdateError: (error) => Effect.succeed(`error: ${error.reason}`) }), + Effect.provideService(HostProcess.Platform, platform), + Effect.provideService(HostProcess.Architecture, arch), + Effect.provideService( + HttpClient.HttpClient, + HttpClient.make((request) => { + requested.push(request.url); + return Effect.succeed(HttpClientResponse.fromWeb(request, new Response(feed))); + }), + ), + ); + + assert.equal( + yield* resolve("stable", "darwin", "arm64", "version: 1.4.0\nfiles: []\n"), + "1.4.0", + ); + assert.equal( + yield* resolve("nightly", "linux", "arm64", "version: '1.5.0-nightly.20261101.2'\n"), + "1.5.0-nightly.20261101.2", + ); + assert.deepStrictEqual(requested, [ + "https://updates.example.com/t3code/latest-mac.yml?token=abc", + "https://updates.example.com/t3code/nightly-linux-arm64.yml?token=abc", + ]); + // A feed for the wrong track is not trusted to mean the requested one. + assert.include( + yield* resolve("stable", "win32", "x64", "version: 1.5.0-nightly.20261101.2\n"), + "Could not read latest.yml from Example Corp's update mirror.", + ); + assert.include(yield* resolve("preview", "win32", "x64", ""), "has no preview builds"); + }), +); + it.layer(NodeServices.layer)("t3 update under a managed policy", (it) => { const runWithPolicy = ( updates: ManagedUpdatesPolicy, input: { readonly channel?: "stable" | "nightly"; readonly requestedVersion?: string } = {}, + httpClient = HttpClient.make(() => Effect.die("unexpected release lookup")), ) => runUpdate({ baseDir: "/unused", @@ -85,10 +142,7 @@ it.layer(NodeServices.layer)("t3 update under a managed policy", (it) => { // Refusals happen before the update touches the network, the service, or a process. Layer.mock(BootService.BootService, {}), Layer.mock(ProcessRunner.ProcessRunner, {}), - Layer.succeed( - HttpClient.HttpClient, - HttpClient.make(() => Effect.die("unexpected release lookup")), - ), + Layer.succeed(HttpClient.HttpClient, httpClient), ), ), ); @@ -118,6 +172,24 @@ it.layer(NodeServices.layer)("t3 update under a managed policy", (it) => { } }), ); + + it.effect("looks for updates on the policy's mirror, never the public releases", () => + Effect.gen(function* () { + const requested: string[] = []; + const error = yield* runWithPolicy( + { mode: "auto", feedUrl: "https://updates.example.com/t3code" }, + { channel: "stable" }, + HttpClient.make((request) => { + requested.push(new URL(request.url).origin); + return Effect.succeed( + HttpClientResponse.fromWeb(request, new Response("", { status: 404 })), + ); + }), + ); + assert.include(error.message, "from Example Corp's update mirror"); + assert.deepStrictEqual(requested, ["https://updates.example.com"]); + }), + ); }); it.layer(NodeServices.layer)("t3 update launcher", (it) => { @@ -364,3 +436,96 @@ it.layer(NodeServices.layer)("t3 update launcher", (it) => { }).pipe(Effect.scoped, Effect.provideService(HostProcess.Platform, "linux")), ); }); + +// Without terminals, update skips its restart prompt whatever runs the test. +const withoutTerminals = Effect.acquireRelease( + Effect.sync(() => + [process.stdin, process.stdout].map((stream) => { + const descriptor = Object.getOwnPropertyDescriptor(stream, "isTTY"); + Object.defineProperty(stream, "isTTY", { configurable: true, value: false }); + return { stream, descriptor }; + }), + ), + (saved) => + Effect.sync(() => { + for (const { stream, descriptor } of saved) { + if (descriptor === undefined) Reflect.deleteProperty(stream, "isTTY"); + else Object.defineProperty(stream, "isTTY", descriptor); + } + }), +); + +it.layer(NodeServices.layer)("t3 update with a background service", (it) => { + it.effect("names the version a service with a pending restart still runs", () => + Effect.gen(function* () { + yield* withoutTerminals; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-update-" }); + const version = packageJson.version; + const runtime = pinnedRuntimePaths(path, baseDir, version, "linux"); + yield* fs.makeDirectory(runtime.versionDir, { recursive: true }); + yield* fs.writeFileString(runtime.entryPath, ""); + yield* fs.writeFileString(runtime.sentinelPath, `${version}\n`); + const unitPath = path.join(baseDir, "t3code.service"); + const logPath = path.join(baseDir, "boot-service.log"); + // An earlier `t3 update` pointed the service at this version without restarting it. + const service = Layer.mock(BootService.BootService, { + status: Effect.succeed({ + supported: true, + installed: true, + current: false, + installedVersion: version, + installedBaseDir: baseDir, + problems: ["restart-pending"], + unitPath, + logPath, + }), + install: () => Effect.succeed({ program: [runtime.entryPath], baseDir, unitPath, logPath }), + }); + vi.spyOn(BootService, "layer").mockReturnValue(service); + const runner = ProcessRunner.ProcessRunner.of({ + run: (input) => + input.args?.includes("--version") === true + ? unrelatedLsof + .run(input) + .pipe(Effect.map((result) => ({ ...result, stdout: `t3 v${version}\n` }))) + : unrelatedLsof.run(input), + }); + + const output = yield* runUpdate({ + baseDir, + logsDir: baseDir, + serverRuntimeStatePath: path.join(baseDir, "server-runtime.json"), + channel: undefined, + requestedVersion: version, + allowDowngrade: false, + assumeYes: false, + }).pipe( + Effect.andThen(TestConsole.logLines), + Effect.map((lines) => lines.join("\n")), + Effect.provide( + Layer.mergeAll( + service, + Layer.succeed(ProcessRunner.ProcessRunner, runner), + ManagedPolicy.layerTest(), + Layer.succeed( + HttpClient.HttpClient, + HttpClient.make(() => Effect.die("unexpected release lookup")), + ), + Layer.fresh(TestConsole.layer), + ), + ), + ); + + assert.include( + output, + `Background service still running the version before ${version}. Run \`t3 service restart\` when you are ready to switch it to ${version}.`, + ); + }).pipe( + Effect.scoped, + Effect.provideService(HostProcess.Platform, "linux"), + Effect.provideService(HostProcess.Environment, { PATH: "" }), + ), + ); +}); diff --git a/apps/server/src/cli/update.ts b/apps/server/src/cli/update.ts index dc2e29345864c..99bf380f1c0dd 100644 --- a/apps/server/src/cli/update.ts +++ b/apps/server/src/cli/update.ts @@ -5,6 +5,8 @@ import { cliReleaseIndexPageUrl, cliReleaseChannelOf, newestCliReleaseVersion, + releaseAssetUrl, + updateFeedFileName, type CliReleaseChannel, } from "@t3tools/shared/cliRelease"; import * as Console from "effect/Console"; @@ -24,6 +26,7 @@ import { ensurePinnedRuntimeInstalled, launcherOwnsVersionsDir, pinnedRuntimeCommand, + isUnpublishedRelease, PinnedRuntimeInstallError, pinnedRuntimePaths, pinnedRuntimeVersionsDir, @@ -33,7 +36,7 @@ import { } from "../cloud/pinnedRuntime.ts"; import { compareExactServiceVersions, isExactServiceVersion } from "../cloud/serviceProtocol.ts"; import * as ManagedPolicy from "../policy/ManagedPolicy.ts"; -import { managedUpdateRefusal } from "../policy/managedUpdates.ts"; +import { managedUpdateMirror, managedUpdateRefusal } from "../policy/managedUpdates.ts"; import * as ProcessRunner from "../processRunner.ts"; import { isProcessAlive, readPersistedServerRuntimeState } from "../serverRuntimeState.ts"; import { projectLocationFlags, resolveCliAuthConfig } from "./config.ts"; @@ -94,6 +97,47 @@ export const resolveNewestVersion = Effect.fn("cli.update.resolve_newest")(funct return yield* new CliUpdateError({ reason: `No published ${channel} release was found.` }); }); +/** The version an update mirror's feed file for this platform offers on `channel`. */ +export const resolveMirrorVersion = Effect.fn("cli.update.resolve_mirror")(function* ( + mirror: { readonly organization: string; readonly feedUrl: string }, + channel: CliReleaseChannel, +) { + if (channel === "preview") { + return yield* new CliUpdateError({ + reason: `${mirror.organization}'s update mirror has no preview builds.`, + }); + } + const httpClient = yield* HttpClient.HttpClient; + const fileName = updateFeedFileName( + channel === "nightly" ? "nightly" : "latest", + yield* HostProcess.Platform, + yield* HostProcess.Architecture, + ); + const url = releaseAssetUrl(mirror.feedUrl, fileName); + // Not the URL, whose query can hold the mirror's access token. + const unreadable = new CliUpdateError({ + reason: `Could not read ${fileName} from ${mirror.organization}'s update mirror.`, + }); + const body = yield* httpClient.get(url).pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap((response) => response.text), + Effect.mapError(() => unreadable), + Effect.timeoutOrElse({ + duration: RELEASE_INDEX_TIMEOUT, + orElse: () => Effect.fail(unreadable), + }), + ); + const version = /^version:\s*["']?([^"'\s]+)/m.exec(body)?.[1]; + if ( + version === undefined || + !isExactServiceVersion(version) || + cliReleaseChannelOf(version) !== channel + ) { + return yield* unreadable; + } + return version; +}); + /** * Only a launcher that already points into this home's `runtime/versions` tree is touched; a * plain copy of the executable, or a launcher for some other install, is left alone. It is @@ -320,6 +364,7 @@ export const runUpdate = Effect.fn("cli.update.run")(function* (input: { const httpClient = yield* HttpClient.HttpClient; const service = yield* BootService.BootService; const policy = yield* ManagedPolicy.ManagedPolicy; + const mirror = managedUpdateMirror(policy.document); const currentVersion = packageJson.version; const channel = input.channel ?? cliReleaseChannelOf(currentVersion); @@ -338,9 +383,11 @@ export const runUpdate = Effect.fn("cli.update.run")(function* (input: { const progress = createUpdateProgress(); progress.status("Checking for updates..."); const targetVersion = yield* ( - input.requestedVersion === undefined - ? resolveNewestVersion(channel) - : Effect.succeed(input.requestedVersion) + input.requestedVersion !== undefined + ? Effect.succeed(input.requestedVersion) + : mirror === undefined + ? resolveNewestVersion(channel) + : resolveMirrorVersion(mirror, channel) ).pipe(Effect.ensuring(Effect.sync(progress.finish))); const targetChannel = cliReleaseChannelOf(targetVersion); @@ -455,7 +502,8 @@ export const runUpdate = Effect.fn("cli.update.run")(function* (input: { httpClient, platform, arch, - releaseBaseUrl: environment[CLI_RELEASE_BASE_URL_ENV]?.trim() || undefined, + releaseBaseUrl: mirror?.feedUrl ?? (environment[CLI_RELEASE_BASE_URL_ENV]?.trim() || undefined), + requirePublishedRelease: mirror !== undefined, validate: (paths) => runner .run({ @@ -481,17 +529,15 @@ export const runUpdate = Effect.fn("cli.update.run")(function* (input: { ), }).pipe( Effect.ensuring(Effect.sync(progress.finish)), - Effect.catchIf( - (error): error is PinnedRuntimeInstallError => - error._tag === "PinnedRuntimeInstallError" && - error.step.startsWith("downloading the t3 release checksums") && - String(error.cause).includes("404"), - () => - Effect.fail( - new CliUpdateError({ - reason: `No release archive was published for t3@${targetVersion}.`, - }), - ), + Effect.catchIf(isUnpublishedRelease, () => + Effect.fail( + new CliUpdateError({ + reason: + mirror === undefined + ? `No release archive was published for t3@${targetVersion}.` + : `${mirror.organization}'s update mirror has no t3@${targetVersion} for this machine.`, + }), + ), ), ); @@ -557,8 +603,15 @@ export const runUpdate = Effect.fn("cli.update.run")(function* (input: { } else if (serviceInstalled && serviceCurrent) { yield* Console.log(` Background service already on ${targetVersion}`); } else if (serviceInstalled) { + // A pending restart means the service already points at serviceVersion but runs an older one. + const running = + serviceVersion === undefined + ? "an unknown version" + : restartPending + ? `the version before ${serviceVersion}` + : serviceVersion; yield* Console.log( - ` Background service still running ${serviceVersion ?? "an unknown version"}. Run \`t3 service restart\` when you are ready to switch it to ${targetVersion}.`, + ` Background service still running ${running}. Run \`t3 service restart\` when you are ready to switch it to ${targetVersion}.`, ); } else if (status.installed && !servesThisHome) { yield* Console.log( diff --git a/apps/server/src/cloud/pinnedRuntime.test.ts b/apps/server/src/cloud/pinnedRuntime.test.ts index 7babf123b9d42..38e56b30388f9 100644 --- a/apps/server/src/cloud/pinnedRuntime.test.ts +++ b/apps/server/src/cloud/pinnedRuntime.test.ts @@ -16,6 +16,7 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import * as ProcessRunner from "../processRunner.ts"; import { ensurePinnedRuntimeInstalled, + isUnpublishedRelease, pinnedRuntimeCommand, pinnedRuntimePaths, PinnedRuntimeInstallError, @@ -39,7 +40,7 @@ const validChecksums = archiveHex(archiveBytes).pipe( const releaseHttpClient = (checksums: string, requests: string[] = []) => HttpClient.make((request) => { requests.push(request.url); - const body = request.url.endsWith("/SHA256SUMS") ? checksums : archiveBytes; + const body = new URL(request.url).pathname.endsWith("/SHA256SUMS") ? checksums : archiveBytes; return Effect.succeed(HttpClientResponse.fromWeb(request, new Response(body))); }); const extractingRunner = (fs: FileSystem.FileSystem, path: Path.Path, commands: string[] = []) => @@ -155,6 +156,40 @@ it.layer(NodeServices.layer)("ensurePinnedRuntimeInstalled", (it) => { }), ); + it.effect("reports a release that lists an archive it does not serve as unpublished", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-archive-missing-" }); + const checksums = yield* validChecksums; + const failsWith = (status: number) => + ensurePinnedRuntimeInstalled({ + baseDir, + version, + fs, + path, + platform: "linux", + arch: "x64", + httpClient: HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + request.url.endsWith("/SHA256SUMS") + ? new Response(checksums) + : new Response("", { status }), + ), + ), + ), + // "404" in the mirror's path must not read a server error as a missing release. + releaseBaseUrl: "https://updates.example.com/404/t3code", + runner: extractingRunner(fs, path), + validate: () => Effect.die("must not validate a missing archive"), + }).pipe(Effect.flip); + assert.isTrue(isUnpublishedRelease(yield* failsWith(404))); + assert.isFalse(isUnpublishedRelease(yield* failsWith(500))); + }), + ); + it.effect("refuses an archive whose checksum does not match the release", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -267,6 +302,42 @@ it.layer(NodeServices.layer)("ensurePinnedRuntimeInstalled", (it) => { }), ); + it.effect("reuses a cached runtime under a mirror only while the mirror lists it", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-runtime-mirror-" }); + const cached = pinnedRuntimePaths(path, baseDir, version, "linux"); + yield* fs.makeDirectory(path.dirname(cached.entryPath), { recursive: true }); + yield* fs.writeFileString(cached.entryPath, "#!/bin/sh\n"); + yield* fs.writeFileString(cached.sentinelPath, `${version}\n`); + const install = (checksums: string, requests: string[]) => + ensurePinnedRuntimeInstalled({ + baseDir, + version, + fs, + path, + platform: "linux", + arch: "x64", + httpClient: releaseHttpClient(checksums, requests), + // A signed mirror URL keeps its query on every file it serves. + releaseBaseUrl: "https://updates.example.com/t3code/?token=abc", + requirePublishedRelease: true, + runner: extractingRunner(fs, path), + validate: () => Effect.void, + }); + + const withdrawn = yield* install("", []).pipe(Effect.flip); + assert.include(withdrawn.message, `finding ${archiveName} in the t3 release checksums`); + const requests: string[] = []; + assert.equal((yield* install(yield* validChecksums, requests)).entryPath, cached.entryPath); + // Listed, so the cached runtime is used without downloading it again. + assert.deepEqual(requests, [ + `https://updates.example.com/t3code/v${version}/SHA256SUMS?token=abc`, + ]); + }), + ); + it.effect("preserves a completed runtime when validation fails", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/cloud/pinnedRuntime.ts b/apps/server/src/cloud/pinnedRuntime.ts index 96745fe85bf41..4e8df29f3026d 100644 --- a/apps/server/src/cloud/pinnedRuntime.ts +++ b/apps/server/src/cloud/pinnedRuntime.ts @@ -9,14 +9,15 @@ import * as Schema from "effect/Schema"; import * as Option from "effect/Option"; import * as Stream from "effect/Stream"; import * as Semaphore from "effect/Semaphore"; -import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http"; +import { HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http"; import { CLI_RELEASE_CHECKSUMS_FILE, cliArchiveFileName, cliArchivePlatformKey, cliArchiveTarCommand, - cliReleaseDownloadBaseUrl, + cliReleaseAssetUrl, + type CliArchivePlatformKey, parseChecksums, } from "@t3tools/shared/cliRelease"; @@ -111,6 +112,8 @@ interface PinnedRuntimeInstallInput { readonly arch: string; readonly httpClient: HttpClient.HttpClient; readonly releaseBaseUrl?: string | undefined; + /** Reuse a cached runtime only while the release source still lists it, as a policy mirror does. */ + readonly requirePublishedRelease?: boolean; readonly onProgress?: (progress: PinnedRuntimeProgress) => void; } @@ -178,28 +181,11 @@ const installFromArchive = Effect.fn("cloud.pinned_runtime.install_archive")(fun }); } const httpClient = input.httpClient; - const baseUrl = cliReleaseDownloadBaseUrl(input.version, input.releaseBaseUrl); - const fileName = cliArchiveFileName(input.version, platformKey); - input.onProgress?.({ stage: "download", received: 0, total: undefined }); - const checksums = parseChecksums( - new TextDecoder().decode( - yield* fetchReleaseAsset( - httpClient, - `${baseUrl}/${CLI_RELEASE_CHECKSUMS_FILE}`, - "downloading the t3 release checksums", - ), - ), - ); - const expected = checksums.get(fileName); - if (expected === undefined) { - return yield* new PinnedRuntimeInstallError({ - step: `finding ${fileName} in the t3 release checksums`, - }); - } + const { fileName, expected } = yield* publishedArchive(input, platformKey); const archive = yield* fetchReleaseAsset( httpClient, - `${baseUrl}/${fileName}`, + cliReleaseAssetUrl(input.version, fileName, input.releaseBaseUrl), "downloading the t3 release archive", input.onProgress, ); @@ -248,6 +234,40 @@ const installFromArchive = Effect.fn("cloud.pinned_runtime.install_archive")(fun yield* fs.remove(archivePath, { force: true }).pipe(Effect.ignore); }); +/** The release's archive for this machine and its checksum; fails when the release lists none. */ +const publishedArchive = Effect.fn("cloud.pinned_runtime.published_archive")(function* ( + input: PinnedRuntimeInstallInput, + platformKey: CliArchivePlatformKey, +) { + const fileName = cliArchiveFileName(input.version, platformKey); + const checksums = parseChecksums( + new TextDecoder().decode( + yield* fetchReleaseAsset( + input.httpClient, + cliReleaseAssetUrl(input.version, CLI_RELEASE_CHECKSUMS_FILE, input.releaseBaseUrl), + "downloading the t3 release checksums", + ), + ), + ); + const expected = checksums.get(fileName); + if (expected === undefined) { + return yield* new PinnedRuntimeInstallError({ + step: `finding ${fileName} in the t3 release checksums`, + }); + } + return { fileName, expected }; +}); + +const isPinnedRuntimeInstallError = Schema.is(PinnedRuntimeInstallError); + +/** Whether an install failed because the release source does not publish the version for this machine. */ +export const isUnpublishedRelease = (error: unknown): error is PinnedRuntimeInstallError => + isPinnedRuntimeInstallError(error) && + (error.step.startsWith("finding ") || + (error.step.startsWith("downloading the t3 release") && + HttpClientError.isHttpClientError(error.cause) && + error.cause.response?.status === 404)); + const installPinnedRuntime = Effect.fn("cloud.pinned_runtime.ensure_installed")(function* ( input: PinnedRuntimeInstallInput, ) { @@ -265,6 +285,15 @@ const installPinnedRuntime = Effect.fn("cloud.pinned_runtime.ensure_installed")( const alreadyPinned = entryExists && Option.isSome(sentinel) && sentinel.value.trim() === input.version; if (alreadyPinned) { + if (input.requirePublishedRelease === true) { + const platformKey = cliArchivePlatformKey(input.platform, input.arch); + if (platformKey === undefined) { + return yield* new PinnedRuntimeInstallError({ + step: `selecting a t3 release archive for ${input.platform}-${input.arch}`, + }); + } + yield* publishedArchive(input, platformKey); + } input.onProgress?.({ stage: "cached" }); yield* input.validate(paths); return paths; diff --git a/apps/server/src/cloud/selfUpdate.test.ts b/apps/server/src/cloud/selfUpdate.test.ts index c831e22c2c0c3..cb7b7fcc2bd35 100644 --- a/apps/server/src/cloud/selfUpdate.test.ts +++ b/apps/server/src/cloud/selfUpdate.test.ts @@ -38,9 +38,10 @@ interface HarnessOptions { } const archiveBytes = new TextEncoder().encode("not really a tarball"); -const releaseHttpClient = (order: string[], beforeDownload?: Effect.Effect) => +const releaseHttpClient = (order: string[], urls: string[], beforeDownload?: Effect.Effect) => HttpClient.make((request) => Effect.gen(function* () { + urls.push(request.url); if (request.url.endsWith("/SHA256SUMS")) { const digest = yield* Effect.promise(() => crypto.subtle.digest("SHA-256", archiveBytes)); const hex = Array.from(new Uint8Array(digest), (byte) => @@ -64,6 +65,7 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* ( const path = yield* Path.Path; const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-self-update-test-" }); const order: string[] = []; + const urls: string[] = []; const runner = ProcessRunner.ProcessRunner.of({ run: (input) => Effect.gen(function* () { @@ -143,7 +145,10 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* ( run: () => Effect.die("unexpected desktop app update run"), }, ), - Effect.provideService(HttpClient.HttpClient, releaseHttpClient(order, options.beforeDownload)), + Effect.provideService( + HttpClient.HttpClient, + releaseHttpClient(order, urls, options.beforeDownload), + ), Effect.provideService(HostProcess.Platform, "linux"), Effect.provideService(HostProcess.Architecture, "x64"), Effect.provide( @@ -153,7 +158,7 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* ( ), ), ); - return { selfUpdate, order }; + return { selfUpdate, order, urls }; }); it.layer(NodeServices.layer)("server self update", (it) => { @@ -438,11 +443,17 @@ it.layer(NodeServices.layer)("server self update", (it) => { it.effect("lets a user start an update, but only to stable without nightly", () => Effect.gen(function* () { - const { selfUpdate, order } = yield* makeHarness({ + const { selfUpdate, order, urls } = yield* makeHarness({ policy: { schemaVersion: 1, organization: "Example Corp", - limits: { updates: { mode: "auto", allowNightly: false } }, + limits: { + updates: { + mode: "auto", + allowNightly: false, + feedUrl: "https://updates.example.com/t3code/", + }, + }, }, }); expect( @@ -450,7 +461,15 @@ it.layer(NodeServices.layer)("server self update", (it) => { .reason, ).toBe("Example Corp manages T3 Code updates. Only stable releases can be installed."); expect(order).toEqual([]); + // The mirror lists only 1.1.0's archive. + expect((yield* selfUpdate.update({ targetVersion: "1.2.0" }).pipe(Effect.flip)).reason).toBe( + "Example Corp's update mirror has no t3@1.2.0 for this machine.", + ); expect((yield* selfUpdate.update({ targetVersion: "1.1.0" })).updateId).toBe("launcher-id"); + // The release comes from the policy's mirror, not GitHub. + expect(urls.length).toBeGreaterThan(0); + for (const url of urls) + expect(url).toMatch(/^https:\/\/updates\.example\.com\/t3code\/v1\.[12]\.0\//); }), ); diff --git a/apps/server/src/cloud/selfUpdate.ts b/apps/server/src/cloud/selfUpdate.ts index c14ef5384184b..9c8f54cc471aa 100644 --- a/apps/server/src/cloud/selfUpdate.ts +++ b/apps/server/src/cloud/selfUpdate.ts @@ -34,10 +34,11 @@ import * as ServerConfig from "../config.ts"; import * as DesktopAppUpdate from "../desktopUpdate/DesktopAppUpdate.ts"; import { databaseMigrationHead } from "../persistence/databaseMigrationHead.ts"; import * as ManagedPolicy from "../policy/ManagedPolicy.ts"; -import { managedUpdateRefusal } from "../policy/managedUpdates.ts"; +import { managedUpdateMirror, managedUpdateRefusal } from "../policy/managedUpdates.ts"; import * as ProcessRunner from "../processRunner.ts"; import { ensurePinnedRuntimeInstalled, + isUnpublishedRelease, pinnedRuntimeCommand, PinnedRuntimeInstallError, pinnedRuntimePaths, @@ -259,9 +260,10 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { const platform = yield* HostProcess.Platform; const arch = yield* HostProcess.Architecture; const httpClient = yield* HttpClient.HttpClient; - const releaseBaseUrl = Option.getOrUndefined( - yield* Config.String(CLI_RELEASE_BASE_URL_ENV).pipe(Config.option), - ); + const mirror = managedUpdateMirror(policy.document); + const releaseBaseUrl = + mirror?.feedUrl ?? + Option.getOrUndefined(yield* Config.String(CLI_RELEASE_BASE_URL_ENV).pipe(Config.option)); const inFlight = yield* Ref.make(false); yield* pruneRuntimesOnStart({ baseDir: serverConfig.baseDir, managed: launcher.managed }).pipe( @@ -336,6 +338,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { platform, arch, releaseBaseUrl, + requirePublishedRelease: mirror !== undefined, validate: (runtime) => runner .run({ @@ -409,7 +412,12 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { Effect.mapError((error) => error._tag === "PinnedRuntimePreflightBlockedError" ? failWith(error.reason, error) - : failWith(`Could not prepare t3@${targetVersion}.`, error), + : mirror !== undefined && isUnpublishedRelease(error) + ? failWith( + `${mirror.organization}'s update mirror has no t3@${targetVersion} for this machine.`, + error, + ) + : failWith(`Could not prepare t3@${targetVersion}.`, error), ), ); diff --git a/apps/server/src/device/DeviceService.test.ts b/apps/server/src/device/DeviceService.test.ts index 8c7b400a2c5de..a551ffdf5a875 100644 --- a/apps/server/src/device/DeviceService.test.ts +++ b/apps/server/src/device/DeviceService.test.ts @@ -18,6 +18,7 @@ import * as Exit from "effect/Exit"; import * as Deferred from "effect/Deferred"; import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; @@ -534,6 +535,11 @@ describe("device setup consent", () => { ); }); +const layerSecretResolver = Layer.succeed(SecretResolver.SecretResolver, { + read: (references) => Effect.succeed(references.map(() => undefined)), + invalidate: () => Effect.void, +}); + it.effect("publishes boot progress and does not restore sessions after support is disabled", () => Effect.gen(function* () { const started = yield* Deferred.make(); @@ -1437,7 +1443,12 @@ it.effect("reconciles SSH host settings without publishing no-op changes", () => }).pipe( Effect.scoped, Effect.provide( - Layer.mergeAll(ServerSettings.layerTest(), NodeServices.layer, NodeCrypto.layer), + Layer.mergeAll( + ServerSettings.layerTest(), + layerSecretResolver, + NodeServices.layer, + NodeCrypto.layer, + ), ), ), ); @@ -1496,6 +1507,7 @@ it.effect("a failed tool update reports npm's exit code but not its output", () Effect.provide( Layer.mergeAll( ServerSettings.layerTest({ enableDeviceSupport: true }), + layerSecretResolver, ServerConfig.layerTest(process.cwd(), { prefix: "t3-device-update-" }), NodeCrypto.layer, NetService.layer, diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 2f6df7dace573..b18cf8a516a98 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -68,12 +68,14 @@ import { isLocalSshDeviceHost, remoteSshDeviceHosts } from "./localSshDeviceHost import { readDeviceDetail, runDeviceAction } from "./DeviceActions.ts"; import * as ProcessRunner from "../processRunner.ts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import * as SecretResolverLayer from "../secrets/SecretResolver.ts"; import * as DeviceHost from "./DeviceHost.ts"; import * as SshDeviceHost from "./SshDeviceHost.ts"; import * as EasDeviceHost from "./EasDeviceHost.ts"; import * as SimlockDeviceHost from "./SimlockDeviceHost.ts"; import * as HostProcess from "@t3tools/shared/HostProcess"; -import { makeExpoApi, resolveExpoCredential } from "./ExpoApi.ts"; +import { makeExpoApi } from "./ExpoApi.ts"; import * as Exit from "effect/Exit"; import * as LocalDeviceHost from "./LocalDeviceHost.ts"; @@ -1220,6 +1222,7 @@ export const make = Effect.gen(function* () { const path = yield* Path.Path; const runner = yield* ProcessRunner.ProcessRunner; const settings = yield* ServerSettings.ServerSettingsService; + const secrets = yield* SecretResolver.SecretResolver; const scope = yield* Scope.Scope; const hosts = new Map([ [localHost.id, localHost], @@ -1537,7 +1540,10 @@ export const make = Effect.gen(function* () { ? "" : ((yield* settings.getSettings).easDeviceHosts.find((host) => host.id === input.hostId) ?.accessToken ?? ""); - const credential = yield* resolveExpoCredential(input.accessToken ?? saved); + const credential = yield* EasDeviceHost.resolveSavedExpoCredential( + secrets, + input.accessToken ?? saved, + ); const viewer = yield* makeExpoApi(yield* HttpClient.HttpClient, credential).viewer; return { signedInAs: viewer.name, credentialSource: credential.source, apps: viewer.apps }; }, @@ -1618,7 +1624,10 @@ export const make = Effect.gen(function* () { }; }); -export const layer = Layer.effect(DeviceService, make).pipe(Layer.provide(LocalDeviceHost.layer)); +export const layer = Layer.effect(DeviceService, make).pipe( + Layer.provide(LocalDeviceHost.layer), + Layer.provide(SecretResolverLayer.layer), +); export const stateStream = (service: DeviceService["Service"]): Stream.Stream => Stream.unwrap( diff --git a/apps/server/src/device/EasDeviceHost.test.ts b/apps/server/src/device/EasDeviceHost.test.ts index b5825d3f973e7..47d74e65b4743 100644 --- a/apps/server/src/device/EasDeviceHost.test.ts +++ b/apps/server/src/device/EasDeviceHost.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it } from "@effect/vitest"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { type EasDeviceHostConfig, ThreadId } from "@t3tools/contracts"; -import * as HostProcess from "@t3tools/shared/HostProcess"; +import { type EasDeviceHostConfig, SecretProviderId, ThreadId } from "@t3tools/contracts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; @@ -13,6 +13,7 @@ import * as Path from "effect/Path"; import * as Scope from "effect/Scope"; import * as TestClock from "effect/testing/TestClock"; import { HttpClient, HttpClientResponse } from "effect/http"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; @@ -228,9 +229,24 @@ const fakeExpo = ( return { http, named, stopped, allowPreview, allowCreate, releases, previews }; }; +const makeSecretResolver = (values: Record = {}) => + Layer.succeed(SecretResolver.SecretResolver, { + read: (references) => + Effect.succeed(references.map((reference) => values[reference.reference])), + invalidate: () => Effect.void, + }); + +const layerWith = (secrets: Layer.Layer) => + Layer.mergeAll( + ProcessRunner.layer, + ServerConfig.layerTest(process.cwd(), { prefix: "t3-eas-host-" }), + secrets, + ).pipe(Layer.provideMerge(NodeServices.layer)); + const layer = Layer.mergeAll( ProcessRunner.layer, ServerConfig.layerTest(process.cwd(), { prefix: "t3-eas-host-" }), + makeSecretResolver(), ).pipe(Layer.provideMerge(NodeServices.layer)); const makeHost = ( @@ -299,6 +315,64 @@ describe("EAS Simulator host", () => { }).pipe(Effect.scoped, Effect.provide(layer)), ); + it.effect("reads its token from the secret provider the setting names", () => + Effect.gen(function* () { + const expo = fakeExpo(); + const source = { provider: SecretProviderId.make("homelab"), reference: "expo#token" }; + const host = yield* makeHost(expo, config({ accessToken: source }), yield* Scope.make()); + const ready = yield* host.ensureReady(() => Effect.void); + const [catalogDevice] = (yield* ready.devices.list).devices; + expect(yield* ready.devices.boot(catalogDevice!)).toBe("session-1"); + }).pipe( + Effect.scoped, + Effect.provide(layerWith(makeSecretResolver({ "expo#token": "expo-token" }))), + ), + ); + + it.effect("fails instead of using another Expo login when its secret cannot be read", () => + Effect.gen(function* () { + const expo = fakeExpo(); + const source = { provider: SecretProviderId.make("homelab"), reference: "expo#token" }; + const host = yield* makeHost(expo, config({ accessToken: source }), yield* Scope.make()).pipe( + Effect.provideService(HostProcess.Environment, { EXPO_TOKEN: "expo-token" }), + ); + const ready = yield* host.ensureReady(() => Effect.void); + const [catalogDevice] = (yield* ready.devices.list).devices; + const failure = yield* ready.devices.boot(catalogDevice!).pipe(Effect.flip); + expect(failure.message).toContain('secret provider "homelab"'); + expect(expo.named("T3CreateEasSimulatorSession")).toHaveLength(0); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.effect("still stops its sessions when the secret becomes unreadable after they start", () => { + const readable = { value: true }; + return Effect.gen(function* () { + const expo = fakeExpo(); + const source = { provider: SecretProviderId.make("homelab"), reference: "expo#token" }; + const scope = yield* Scope.make(); + const host = yield* makeHost(expo, config({ accessToken: source }), scope); + const ready = yield* host.ensureReady(() => Effect.void); + const [catalogDevice] = (yield* ready.devices.list).devices; + expect(yield* ready.devices.boot(catalogDevice!)).toBe("session-1"); + + readable.value = false; + yield* ready.devices.shutdown({ ...catalogDevice!, id: "session-1" }); + expect(expo.stopped()).toEqual(["session-1"]); + yield* Scope.close(scope, Exit.void); + }).pipe( + Effect.scoped, + Effect.provide( + layerWith( + Layer.succeed(SecretResolver.SecretResolver, { + read: (references) => + Effect.succeed(references.map(() => (readable.value ? "expo-token" : undefined))), + invalidate: () => Effect.void, + }), + ), + ), + ); + }); + it.effect("runs an Android session through serve-emu beside an iOS one", () => Effect.gen(function* () { const expo = fakeExpo(); diff --git a/apps/server/src/device/EasDeviceHost.ts b/apps/server/src/device/EasDeviceHost.ts index e82638dbd76fa..719cfbf435a8a 100644 --- a/apps/server/src/device/EasDeviceHost.ts +++ b/apps/server/src/device/EasDeviceHost.ts @@ -11,6 +11,7 @@ import { DeviceOperationError, type DevicePlatform, type EasDeviceHostConfig, + type SettingsSecretValue, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; @@ -32,6 +33,8 @@ import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as ServerConfig from "../config.ts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import { requireSettingsSecret } from "../secrets/settingsSecretValue.ts"; import { releaseAgentDeviceSession } from "./AgentDeviceTarget.ts"; import * as DeviceHost from "./DeviceHost.ts"; import { findEasProjectId } from "./easProjectId.ts"; @@ -140,11 +143,21 @@ const decodePathSegment = (segment: string): string | null => { } }; +/** + * The Expo credential a host's setting holds or names. Every session bills the account behind + * it, so a named secret that cannot be read fails rather than falling back to another login. + */ +export const resolveSavedExpoCredential = ( + secrets: SecretResolver.SecretResolver["Service"], + saved: SettingsSecretValue, +) => requireSettingsSecret(secrets, saved).pipe(Effect.flatMap(resolveExpoCredential)); + export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDeviceHostConfig) { const http = yield* HttpClient.HttpClient; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const server = yield* ServerConfig.ServerConfig; + const secrets = yield* SecretResolver.SecretResolver; const services = yield* Effect.context< FileSystem.FileSystem | Path.Path | Effect.Services> >(); @@ -198,13 +211,24 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice ...(cause === undefined ? {} : { cause }), }); - const api: Effect.Effect = resolveExpoCredential( + let lastApi: ExpoApi | undefined; + const api: Effect.Effect = resolveSavedExpoCredential( + secrets, config.accessToken, ).pipe( Effect.map((credential) => makeExpoApi(http, credential)), + Effect.tap((expo) => + Effect.sync(() => { + lastApi = expo; + }), + ), Effect.provide(services), Effect.mapError((error) => unavailable(error.message, error)), ); + /** Stopping a billed session must not depend on the secret still being readable; starting one must. */ + const apiForStop = api.pipe( + Effect.catch((error) => (lastApi === undefined ? Effect.fail(error) : Effect.succeed(lastApi))), + ); /** A missing ledger is empty; an unreadable one is an error, never an empty list. */ const readLedger = fs.readFileString(ledgerPath).pipe( @@ -626,7 +650,7 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice if (stale.length === 0 && unresolved.size === 0 && uncertain.size === 0 && !reapDue) return; const reap = reapDue || uncertain.size > 0; if (reap) reapedAt = now; - const expo = yield* api.pipe( + const expo = yield* apiForStop.pipe( // Without a credential no sweep can look; a record Expo would have // listed by now still stops counting, so the limit is not held forever. Effect.tapError(() => @@ -723,7 +747,7 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice Option.match(sessionFor(device.id), { onNone: () => Effect.void, onSome: (session) => - api.pipe( + apiForStop.pipe( Effect.flatMap((expo) => stopTracked(expo, session)), Effect.asVoid, Effect.mapError( @@ -853,7 +877,7 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice // Interrupted starts stop the sessions they created before this returns. yield* FiberSet.clear(starts); yield* FiberSet.clear(background); - const expo = yield* api; + const expo = yield* apiForStop; yield* Effect.forEach( [...sessions.values()], (session) => stopTracked(expo, session).pipe(Effect.ignore), @@ -884,7 +908,7 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice // Each server signs in on its own, so one with no Expo credential answers for itself. Reading // the credential is local; whether Expo accepts it is only known once a session starts. const platformAvailability = (platform: DevicePlatform) => - resolveExpoCredential(config.accessToken).pipe( + resolveSavedExpoCredential(secrets, config.accessToken).pipe( Effect.provide(services), Effect.as({ platform, available: true }), Effect.catch((error) => diff --git a/apps/server/src/device/SshDeviceHost.test.ts b/apps/server/src/device/SshDeviceHost.test.ts index 80b669732eb37..6600425331c8f 100644 --- a/apps/server/src/device/SshDeviceHost.test.ts +++ b/apps/server/src/device/SshDeviceHost.test.ts @@ -11,6 +11,7 @@ import * as Stream from "effect/Stream"; import * as HttpClient from "effect/http/HttpClient"; import * as HttpClientResponse from "effect/http/HttpClientResponse"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import * as TestClock from "effect/testing/TestClock"; import * as ServerConfig from "../config.ts"; import * as DeviceHost from "./DeviceHost.ts"; import * as SshDeviceHost from "./SshDeviceHost.ts"; @@ -309,3 +310,79 @@ it.effect("lists Android devices without probing the host again once it reports yield* host.stop; }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); + +it.effect( + "keeps the SSH forward through slow health probes and reconnects once the hub stays down", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped(); + let forwardSpawns = 0; + let hubAnswers = true; + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if (command._tag !== "StandardCommand") return yield* Effect.die("Unexpected command"); + const forwarding = command.args.includes("-L"); + // A fresh forward reaches a working hub again. + if (forwarding && ++forwardSpawns > 1) hubAnswers = true; + const output = JSON.stringify({ + nodePath: "/node", + platforms: [{ platform: "ios", available: true }], + hubPort: 1234, + helpers: { serveSimAxSettings: null, serveSimCli: null }, + }); + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(123), + stdout: Stream.make(new TextEncoder().encode(output)), + stderr: Stream.empty, + all: Stream.empty, + exitCode: forwarding ? Effect.never : Effect.succeed(ChildProcessSpawner.ExitCode(0)), + isRunning: Effect.succeed(forwarding), + kill: () => Effect.void, + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + }), + ); + const host = yield* SshDeviceHost.make({ + kind: "ssh", + id: "test", + label: "Test", + target: "test.example", + }).pipe( + Effect.provide(Layer.mergeAll(ServerConfig.layerTest(home, home), Net.layer)), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService( + HttpClient.HttpClient, + HttpClient.make((request) => + hubAnswers + ? Effect.succeed(HttpClientResponse.fromWeb(request, new Response("ok"))) + : Effect.never, + ), + ), + ); + yield* host.ensureReady(() => Effect.void); + expect(forwardSpawns).toBe(1); + + // A loaded host can miss a probe or two without being gone; the forward stays up. + hubAnswers = false; + yield* TestClock.adjust("30 seconds"); + hubAnswers = true; + yield* TestClock.adjust("2 minutes"); + expect(forwardSpawns).toBe(1); + + // A hub that keeps missing probes is still replaced. + hubAnswers = false; + // Recovery reset the miss count, so two new misses do not reconnect yet. + yield* TestClock.adjust("30 seconds"); + expect(forwardSpawns).toBe(1); + yield* TestClock.adjust("30 seconds"); + expect(forwardSpawns).toBe(2); + yield* host.stop; + }).pipe( + Effect.scoped, + Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer, TestClock.layer())), + ), +); diff --git a/apps/server/src/device/SshDeviceHost.ts b/apps/server/src/device/SshDeviceHost.ts index 789b700308b8d..ef051cb90756c 100644 --- a/apps/server/src/device/SshDeviceHost.ts +++ b/apps/server/src/device/SshDeviceHost.ts @@ -27,6 +27,8 @@ import * as DeviceHost from "./DeviceHost.ts"; import { makeDeviceHubOperations } from "./DeviceHubOperations.ts"; import { quoteRemoteArg, remoteDeviceEnvironment, remoteDeviceScript } from "./sshDeviceScript.ts"; +const UNHEALTHY_PROBE_LIMIT = 3; + const Probe = Schema.Struct({ nodePath: Schema.String, tools: Schema.optional(DeviceToolVersions), @@ -365,7 +367,10 @@ export const make = Effect.fn("SshDeviceHost.make")(function* ( : next; yield* onStatus("ready"); // Reconnect also repairs helpers that died while SSH itself stayed connected. + // A loaded host can miss a probe or two while still working, and reconnecting + // then drops the forward in use, so only consecutive misses count. const unhealthy = Effect.gen(function* () { + let misses = 0; while (true) { yield* Effect.sleep("10 seconds"); const alive = yield* http.get(`${next.hub.origin}/readyz`).pipe( @@ -380,7 +385,8 @@ export const make = Effect.fn("SshDeviceHost.make")(function* ( Effect.orElseSucceed(() => false), ) : true; - if (!alive || !daemonAlive) return; + misses = alive && daemonAlive ? 0 : misses + 1; + if (misses >= UNHEALTHY_PROBE_LIMIT) return; } }); yield* Effect.gen(function* () { diff --git a/apps/server/src/git/GitManager.test.ts b/apps/server/src/git/GitManager.test.ts index 12a09e8c439cc..d3ac96bd397fa 100644 --- a/apps/server/src/git/GitManager.test.ts +++ b/apps/server/src/git/GitManager.test.ts @@ -21,6 +21,7 @@ import * as PlatformError from "effect/PlatformError"; import * as Result from "effect/Result"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import { ChildProcessSpawner } from "effect/process"; import { expect } from "vite-plus/test"; import type { @@ -1091,6 +1092,78 @@ it.layer(layerGitManagerTest)("GitManager", (it) => { }), ); + it.effect("a settled branch PR is rechecked only when the branch moves or every 15 minutes", () => + Effect.gen(function* () { + const repoDir = yield* makeTempDir("t3code-git-manager-"); + yield* initRepo(repoDir); + const remoteDir = yield* createBareRemote(); + yield* runGit(repoDir, ["remote", "add", "origin", remoteDir]); + yield* runGit(repoDir, ["push", "-u", "origin", "main"]); + yield* runGit(repoDir, ["checkout", "-b", "feature/settled"]); + yield* runGit(repoDir, ["push", "-u", "origin", "feature/settled"]); + const pullRequest = (number: number, state: "open" | "merged"): FakeChangeRequest => ({ + number, + title: "Settled", + url: `https://github.com/pingdotgg/codething-mvp/pull/${number}`, + baseRefName: "main", + headRefName: "feature/settled", + state, + updatedAt: `2026-04-0${number}T15:00:00Z`, + }); + const gitCommands: string[] = []; + const { manager } = yield* makeManager({ + gitCommands, + providerScenario: { + prListSequence: [ + [pullRequest(1, "merged")], + [pullRequest(1, "merged")], + [pullRequest(1, "merged")], + ], + prListOtherwise: [pullRequest(2, "open")], + }, + }); + const lookup = manager.branchPullRequest({ cwd: repoDir, branch: "feature/settled" }); + const freshLookups = () => gitCommands.filter((args) => args === "remote").length; + + expect((yield* lookup)?.state).toBe("merged"); + gitCommands.length = 0; + expect((yield* lookup)?.state).toBe("merged"); + expect((yield* lookup)?.state).toBe("merged"); + expect(gitCommands.map((args) => args.split(" ", 1)[0]).toSorted()).toEqual([ + "config", + "config", + "for-each-ref", + "for-each-ref", + ]); + expect(freshLookups()).toBe(0); + + NodeFS.writeFileSync(NodePath.join(repoDir, "settled.txt"), "new work\n"); + yield* runGit(repoDir, ["add", "settled.txt"]); + yield* runGit(repoDir, ["commit", "-m", "New work"]); + gitCommands.length = 0; + expect((yield* lookup)?.state).toBe("merged"); + expect(freshLookups()).toBe(1); + gitCommands.length = 0; + yield* lookup; + expect(freshLookups()).toBe(0); + + // Remembered git state expires after 5 minutes; that rereads git without a recheck. + yield* TestClock.adjust("10 minutes"); + gitCommands.length = 0; + expect((yield* lookup)?.state).toBe("merged"); + expect(freshLookups()).toBe(1); + + yield* TestClock.adjust("5 minutes"); + gitCommands.length = 0; + expect((yield* lookup)?.state).toBe("merged"); + expect(freshLookups()).toBe(2); + + // A new PR from the reused branch shows on the next host query. + yield* TestClock.adjust("5 minutes"); + expect((yield* lookup)?.number).toBe(2); + }), + ); + it.effect("branch PR lookup uses a saved tracked branch without changing checkout", () => Effect.gen(function* () { const repoDir = yield* makeTempDir("t3code-git-manager-"); @@ -1789,6 +1862,244 @@ it.layer(layerGitManagerTest)("GitManager", (it) => { }), ); + function squashMergeIntoRemoteMain(repoDir: string, branch: string) { + return Effect.gen(function* () { + yield* runGit(repoDir, ["push", "-u", "origin", "main"]); + yield* runGit(repoDir, ["checkout", "-b", branch]); + NodeFS.writeFileSync(NodePath.join(repoDir, "squash.txt"), "one\ntwo\n"); + yield* runGit(repoDir, ["add", "squash.txt"]); + yield* runGit(repoDir, ["commit", "-m", "Feature work"]); + yield* runGit(repoDir, ["push", "-u", "origin", branch]); + // The host squash-merges: a new commit on main with the branch's tree, not the branch's commit. + const squash = yield* runGit(repoDir, [ + "commit-tree", + "HEAD^{tree}", + "-p", + "origin/main", + "-m", + "Feature work (#217)", + ]); + yield* runGit(repoDir, ["push", "origin", `${squash.stdout.trim()}:refs/heads/main`]); + return (yield* runGit(repoDir, ["rev-parse", "HEAD"])).stdout.trim(); + }); + } + + const squashMergedPr = (branch: string, headSha: string): FakeChangeRequest => ({ + number: 217, + title: "Feature work", + url: "https://github.com/pingdotgg/t3code/pull/217", + baseRefName: "main", + headRefName: branch, + headSha, + state: "merged", + mergedAt: "2026-04-02T15:00:00Z", + updatedAt: "2026-04-02T15:00:00Z", + }); + + it.effect("status reports nothing to ship after a squash merge deleted the remote branch", () => + Effect.gen(function* () { + const repoDir = yield* makeTempDir("t3code-git-manager-"); + yield* initRepo(repoDir); + const remoteDir = yield* createBareRemote(); + yield* runGit(repoDir, ["remote", "add", "origin", remoteDir]); + const branch = "feature/squash-deleted"; + const headSha = yield* squashMergeIntoRemoteMain(repoDir, branch); + yield* runGit(repoDir, ["push", "origin", "--delete", branch]); + yield* runGit(repoDir, ["fetch", "--prune", "origin"]); + + const { manager } = yield* makeManager({ + providerScenario: { prListSequence: [[squashMergedPr(branch, headSha)]] }, + }); + + const status = yield* manager.status({ cwd: repoDir }); + + expect(status.pr?.state).toBe("merged"); + expect(status.hasUpstream).toBe(false); + expect({ + aheadCount: status.aheadCount, + aheadOfDefaultCount: status.aheadOfDefaultCount, + }).toEqual({ aheadCount: 0, aheadOfDefaultCount: 0 }); + }), + ); + + it.effect("status reports nothing to ship after a squash merge kept the remote branch", () => + Effect.gen(function* () { + const repoDir = yield* makeTempDir("t3code-git-manager-"); + yield* initRepo(repoDir); + const remoteDir = yield* createBareRemote(); + yield* runGit(repoDir, ["remote", "add", "origin", remoteDir]); + const branch = "feature/squash-kept"; + const headSha = yield* squashMergeIntoRemoteMain(repoDir, branch); + yield* runGit(repoDir, ["fetch", "--prune", "origin"]); + + const { manager } = yield* makeManager({ + providerScenario: { prListSequence: [[squashMergedPr(branch, headSha)]] }, + }); + + const status = yield* manager.status({ cwd: repoDir }); + + expect(status.pr?.state).toBe("merged"); + expect(status.hasUpstream).toBe(true); + expect({ + aheadCount: status.aheadCount, + aheadOfDefaultCount: status.aheadOfDefaultCount, + }).toEqual({ aheadCount: 0, aheadOfDefaultCount: 0 }); + }), + ); + + it.effect("status still reports work to ship for a commit added after a squash merge", () => + Effect.gen(function* () { + const repoDir = yield* makeTempDir("t3code-git-manager-"); + yield* initRepo(repoDir); + const remoteDir = yield* createBareRemote(); + yield* runGit(repoDir, ["remote", "add", "origin", remoteDir]); + const branch = "feature/squash-follow-up"; + const headSha = yield* squashMergeIntoRemoteMain(repoDir, branch); + NodeFS.writeFileSync(NodePath.join(repoDir, "follow-up.txt"), "after merge\n"); + yield* runGit(repoDir, ["add", "follow-up.txt"]); + yield* runGit(repoDir, ["commit", "-m", "Follow-up work"]); + yield* runGit(repoDir, ["fetch", "--prune", "origin"]); + + const { manager } = yield* makeManager({ + providerScenario: { prListSequence: [[squashMergedPr(branch, headSha)]] }, + }); + + const status = yield* manager.status({ cwd: repoDir }); + + expect(status.pr?.state).toBe("merged"); + expect(status.aheadCount).toBe(1); + expect(status.aheadOfDefaultCount).toBe(2); + }), + ); + + it.effect("status still reports work against main after a merge into another base", () => + Effect.gen(function* () { + const repoDir = yield* makeTempDir("t3code-git-manager-"); + yield* initRepo(repoDir); + const initialSha = (yield* runGit(repoDir, ["rev-parse", "HEAD"])).stdout.trim(); + const remoteDir = yield* createBareRemote(); + yield* runGit(repoDir, ["remote", "add", "origin", remoteDir]); + const branch = "feature/release-only"; + const headSha = yield* squashMergeIntoRemoteMain(repoDir, branch); + // Move the merge to release and leave main without it. + const squashSha = (yield* runGit(repoDir, ["rev-parse", "origin/main"])).stdout.trim(); + yield* runGit(repoDir, ["push", "origin", `${squashSha}:refs/heads/release`]); + yield* runGit(repoDir, ["push", "--force", "origin", `${initialSha}:refs/heads/main`]); + yield* runGit(repoDir, ["fetch", "origin"]); + + const { manager } = yield* makeManager({ + providerScenario: { + prListSequence: [[{ ...squashMergedPr(branch, headSha), baseRefName: "release" }]], + }, + }); + + const status = yield* manager.status({ cwd: repoDir }); + + expect(status.pr?.state).toBe("merged"); + expect(status.aheadOfDefaultCount).toBe(1); + }), + ); + + it.effect("status keeps a squash-merged branch shipped when a later PR lookup fails", () => + Effect.gen(function* () { + const repoDir = yield* makeTempDir("t3code-git-manager-"); + yield* initRepo(repoDir); + const remoteDir = yield* createBareRemote(); + yield* runGit(repoDir, ["remote", "add", "origin", remoteDir]); + const branch = "feature/squash-lookup-fails"; + const headSha = yield* squashMergeIntoRemoteMain(repoDir, branch); + yield* runGit(repoDir, ["push", "origin", "--delete", branch]); + yield* runGit(repoDir, ["fetch", "--prune", "origin"]); + + const { manager } = yield* makeManager({ + providerScenario: { + prListSequence: [[squashMergedPr(branch, headSha)]], + failWith: "Rate limited", + failAfterCalls: 1, + }, + }); + + const first = yield* manager.status({ cwd: repoDir }); + yield* manager.invalidateStatus(repoDir); + const second = yield* manager.status({ cwd: repoDir }); + + expect(first.aheadOfDefaultCount).toBe(0); + expect(second.pr?.state).toBe("merged"); + expect({ + aheadCount: second.aheadCount, + aheadOfDefaultCount: second.aheadOfDefaultCount, + }).toEqual({ aheadCount: 0, aheadOfDefaultCount: 0 }); + }), + ); + + it.effect("status keeps real divergence for a squash-merged branch that tracks its base", () => + Effect.gen(function* () { + const repoDir = yield* makeTempDir("t3code-git-manager-"); + yield* initRepo(repoDir); + const remoteDir = yield* createBareRemote(); + yield* runGit(repoDir, ["remote", "add", "origin", remoteDir]); + const branch = "feature/squash-tracks-main"; + const headSha = yield* squashMergeIntoRemoteMain(repoDir, branch); + yield* runGit(repoDir, ["fetch", "origin"]); + yield* runGit(repoDir, ["branch", "--set-upstream-to", "origin/main"]); + + const { manager } = yield* makeManager({ + providerScenario: { prListSequence: [[squashMergedPr(branch, headSha)]] }, + }); + + const status = yield* manager.status({ cwd: repoDir }); + + expect(status.pr?.state).toBe("merged"); + expect({ aheadCount: status.aheadCount, behindCount: status.behindCount }).toEqual({ + aheadCount: 1, + behindCount: 1, + }); + }), + ); + + it.effect("status reports nothing to ship after a squash merge into a non-origin default", () => + Effect.gen(function* () { + const repoDir = yield* makeTempDir("t3code-git-manager-"); + yield* initRepo(repoDir); + yield* runGit(repoDir, ["branch", "-m", "develop"]); + const remoteDir = yield* createBareRemote(); + yield* runGit(repoDir, ["remote", "add", "upstream", remoteDir]); + yield* runGit(repoDir, ["push", "-u", "upstream", "develop"]); + yield* runGit(repoDir, ["remote", "set-head", "upstream", "develop"]); + const branch = "feature/squash-develop"; + yield* runGit(repoDir, ["checkout", "-b", branch]); + NodeFS.writeFileSync(NodePath.join(repoDir, "squash.txt"), "one\ntwo\n"); + yield* runGit(repoDir, ["add", "squash.txt"]); + yield* runGit(repoDir, ["commit", "-m", "Feature work"]); + yield* runGit(repoDir, ["push", "-u", "upstream", branch]); + const headSha = (yield* runGit(repoDir, ["rev-parse", "HEAD"])).stdout.trim(); + const squash = yield* runGit(repoDir, [ + "commit-tree", + "HEAD^{tree}", + "-p", + "upstream/develop", + "-m", + "Feature work (#217)", + ]); + yield* runGit(repoDir, ["push", "upstream", `${squash.stdout.trim()}:refs/heads/develop`]); + yield* runGit(repoDir, ["fetch", "upstream"]); + + const { manager } = yield* makeManager({ + providerScenario: { + prListSequence: [[{ ...squashMergedPr(branch, headSha), baseRefName: "develop" }]], + }, + }); + + const status = yield* manager.status({ cwd: repoDir }); + + expect(status.pr?.state).toBe("merged"); + expect({ + aheadCount: status.aheadCount, + aheadOfDefaultCount: status.aheadOfDefaultCount, + }).toEqual({ aheadCount: 0, aheadOfDefaultCount: 0 }); + }), + ); + it.each([ [ "https://github.example.com/team/repository/pull/42?tab=files", @@ -2579,6 +2890,46 @@ it.layer(layerGitManagerTest)("GitManager", (it) => { }), ); + it.effect("keeps signature output out of recent commit subjects", () => + Effect.gen(function* () { + const repoDir = yield* makeTempDir("t3code-git-manager-"); + yield* initRepo(repoDir); + const signingKey = NodePath.join(yield* makeTempDir("t3code-git-signing-key-"), "key"); + NodeChildProcess.execFileSync("ssh-keygen", [ + "-q", + "-t", + "ed25519", + "-N", + "", + "-f", + signingKey, + ]); + yield* runGit(repoDir, ["config", "gpg.format", "ssh"]); + // A global signer such as 1Password's cannot use the fixture's generated key. + yield* runGit(repoDir, ["config", "gpg.ssh.program", "ssh-keygen"]); + yield* runGit(repoDir, ["config", "user.signingkey", `${signingKey}.pub`]); + yield* runGit(repoDir, ["config", "log.showSignature", "true"]); + yield* runGit(repoDir, ["commit", "--amend", "-S", "--no-edit"]); + NodeFS.writeFileSync(NodePath.join(repoDir, "README.md"), "hello again\n"); + let commitInstructions: string | undefined; + + const { manager } = yield* makeManager({ + textGeneration: { + generateCommitMessage: (input) => { + commitInstructions = input.policy?.commitInstructions; + return Effect.succeed({ subject: "Update readme", body: "" }); + }, + }, + }); + + yield* runStackedAction(manager, { cwd: repoDir, action: "commit" }); + + expect(commitInstructions).toMatch( + /Recent commit subjects from this repository:\nInitial commit$/, + ); + }), + ); + it.effect("falls back when the dedicated source control writer is unavailable", () => Effect.gen(function* () { const repoDir = yield* makeTempDir("t3code-git-manager-"); diff --git a/apps/server/src/git/GitManager.ts b/apps/server/src/git/GitManager.ts index d1f48bfed3681..7ec4b4effbdaa 100644 --- a/apps/server/src/git/GitManager.ts +++ b/apps/server/src/git/GitManager.ts @@ -171,6 +171,7 @@ const PR_LOOKUP_FAILURE_BASE_TTL = Duration.seconds(20); const PR_LOOKUP_FAILURE_MAX_TTL = Duration.minutes(15); const PR_LOOKUP_CACHE_CAPACITY = 2_048; const PR_LOOKUP_GIT_STATE_TTL = Duration.minutes(5); +const SETTLED_PR_LOOKUP_RECHECK_INTERVAL = Duration.minutes(15); const PR_THREAD_CONTEXT_BYTES = 16_000; const PR_THREAD_COMMAND_OUTPUT_CHARS = 2_000; const isSourceControlProviderError = Schema.is(SourceControlProviderError); @@ -791,7 +792,7 @@ export const make = Effect.gen(function* () { .execute({ operation: "GitManager.readRecentCommitSubjects", cwd, - args: ["log", "-n", "20", "--no-merges", "--pretty=format:%s"], + args: ["log", "-n", "20", "--no-merges", "--no-show-signature", "--pretty=format:%s"], }) .pipe( Effect.map((result) => @@ -1258,6 +1259,7 @@ export const make = Effect.gen(function* () { const getPrLookup = (key: string) => detachStackFrame(Cache.get(prLookupCache, key)); interface LastKnownPr { readonly pr: ReturnType | null; + readonly headSha: string | null; readonly upstreamRef: string | null; readonly headBranch: string; readonly remoteName: string | null; @@ -1279,7 +1281,7 @@ export const make = Effect.gen(function* () { const resolveLastKnownPr = ( branchKey: string, current: Pick, - ): ReturnType | null => { + ): LastKnownPr | null => { const lastKnown = lastKnownPrByBranchKey.get(branchKey); if (!lastKnown) return null; if (lastKnown.headBranch !== current.headBranch) { @@ -1288,7 +1290,7 @@ export const make = Effect.gen(function* () { // Config read failures become null; require both remote URLs before treating a mismatch as real. if (lastKnown.headRemoteUrlKey !== null && current.headRemoteUrlKey !== null) { - return lastKnown.headRemoteUrlKey === current.headRemoteUrlKey ? lastKnown.pr : null; + return lastKnown.headRemoteUrlKey === current.headRemoteUrlKey ? lastKnown : null; } if ( @@ -1297,9 +1299,9 @@ export const make = Effect.gen(function* () { lastKnown.remoteName !== null && current.remoteName !== null ) { - return lastKnown.remoteName === current.remoteName ? lastKnown.pr : null; + return lastKnown.remoteName === current.remoteName ? lastKnown : null; } - return lastKnown.pr; + return lastKnown; }; const lookupStatusPr = Effect.fn("lookupStatusPr")(function* ( cwd: string, @@ -1324,17 +1326,18 @@ export const make = Effect.gen(function* () { } return yield* getPrLookup(cacheKey).pipe( Effect.map(({ latest, headContext }) => { - if (!latest) return { pr: null, headContext }; + if (!latest) return { pr: null, headSha: null, headContext }; // Merged/closed matches are usually reverse-merge history, not the thread's PR context. if (details.isDefaultBranch && latest.state !== "open") { - return { pr: null, headContext }; + return { pr: null, headSha: null, headContext }; } - return { pr: toStatusPr(latest), headContext }; + return { pr: toStatusPr(latest), headSha: latest.headSha ?? null, headContext }; }), - Effect.tap(({ pr, headContext }) => + Effect.tap(({ pr, headSha, headContext }) => Effect.sync(() => rememberLastKnownPr(branchKey, { pr, + headSha, upstreamRef: details.upstreamRef, headBranch: headContext.headBranch, remoteName: headContext.remoteName, @@ -1342,7 +1345,7 @@ export const make = Effect.gen(function* () { }), ), ), - Effect.map(({ pr }) => pr), + Effect.map(({ pr, headSha }) => ({ pr, headSha })), Effect.catch((error) => { const fallback = lastKnownPrByBranchKey.get(branchKey)?.pr ? resolveLookupHeadContext(cwd, details).pipe( @@ -1374,6 +1377,10 @@ export const make = Effect.gen(function* () { : {}), }), Effect.andThen(fallback), + Effect.map((lastKnown) => ({ + pr: lastKnown?.pr ?? null, + headSha: lastKnown?.headSha ?? null, + })), ); }), ); @@ -1389,7 +1396,7 @@ export const make = Effect.gen(function* () { return null; } - const pr = + const { pr, headSha } = details.branch !== null ? yield* lookupStatusPr( cwd, @@ -1401,13 +1408,35 @@ export const make = Effect.gen(function* () { }, options?.refreshMissingPullRequest, ) - : null; + : { pr: null, headSha: null }; + // Squash and rebase merges land new commits, so HEAD never becomes reachable from the base. + const shippedByMergedPr = + pr?.state === "merged" && + headSha !== null && + (yield* Effect.gen(function* () { + const defaultBranch = + details.defaultBranch ?? + (yield* gitCore.resolvePrimaryRemoteName(cwd).pipe( + Effect.flatMap((remoteName) => gitCore.resolveDefaultBranchName(cwd, remoteName)), + Effect.orElseSucceed(() => null), + )); + const mergedIntoDefault = + defaultBranch === null + ? pr.baseRef === "main" || pr.baseRef === "master" + : pr.baseRef === defaultBranch; + if (!mergedIntoDefault) return false; + return yield* gitCore.resolveCommit({ cwd, revision: "HEAD" }).pipe( + Effect.map(({ commitSha }) => commitSha === headSha), + Effect.orElseSucceed(() => false), + ); + })); return { hasUpstream: details.hasUpstream, - aheadCount: details.aheadCount, + // A real upstream delta stays; only the no-upstream fallback counts against the base. + aheadCount: shippedByMergedPr && !details.hasUpstream ? 0 : details.aheadCount, behindCount: details.behindCount, - aheadOfDefaultCount: details.aheadOfDefaultCount, + aheadOfDefaultCount: shippedByMergedPr ? 0 : details.aheadOfDefaultCount, pr, } satisfies VcsStatusRemoteResult; }); @@ -2425,6 +2454,10 @@ export const make = Effect.gen(function* () { prLookupGitStateByKey.set(entryKey, { readAtMs, value }); return value; }); + const settledPrLookupByBranchKey = new Map< + string, + { readonly url: string; readonly fingerprint: string; readonly confirmedAtMs: number } + >(); const forgetPrLookupGitState = (cwd: string) => normalizeStatusCacheKey(cwd).pipe( Effect.map((cacheCwd) => { @@ -2470,6 +2503,39 @@ export const make = Effect.gen(function* () { }; }, ); + // objectname lets a settled PR result notice new commits on its branch. + const readPrLookupBranchRef = (cacheCwd: string, branch: string, fresh: boolean) => + rememberPrLookupGitState( + cacheCwd, + `branchRef\u0000${branch}`, + fresh, + gitCore + .execute({ + operation: "GitManager.branchPullRequest.branchRef", + cwd: cacheCwd, + args: [ + "for-each-ref", + "--format=%(refname)%00%(upstream:short)%00%(upstream:remotename)%00%(upstream:remoteref)%00%(objectname)", + `refs/heads/${branch}`, + ], + }) + .pipe(Effect.map((result) => result.stdout)), + ); + const readSettledPrLookupFingerprint = (cacheCwd: string, branch: string) => + Effect.all( + [ + readPrLookupBranchRef(cacheCwd, branch, true), + gitCore + .execute({ + operation: "GitManager.branchPullRequest.remoteUrls", + cwd: cacheCwd, + args: ["config", "--get-regexp", "^remote\\..*\\.url$"], + allowNonZeroExit: true, + }) + .pipe(Effect.map((result) => result.stdout)), + ], + { concurrency: "unbounded" }, + ).pipe(Effect.map((parts) => parts.join("\u0000"))); const lookupBranchPullRequest = Effect.fn("lookupBranchPullRequest")(function* ( cacheCwd: string, branch: string, @@ -2493,20 +2559,7 @@ export const make = Effect.gen(function* () { .filter((remoteName) => remoteName.length > 0); const [firstRemoteName] = remoteNames; if (firstRemoteName === undefined) return null; - const branchRef = yield* remember( - `branchRef\u0000${branch}`, - gitCore - .execute({ - operation: "GitManager.branchPullRequest.branchRef", - cwd: cacheCwd, - args: [ - "for-each-ref", - "--format=%(refname)%00%(upstream:short)%00%(upstream:remotename)%00%(upstream:remoteref)", - `refs/heads/${branch}`, - ], - }) - .pipe(Effect.map((result) => result.stdout)), - ); + const branchRef = yield* readPrLookupBranchRef(cacheCwd, branch, options.freshGitState); const expectedRefName = `refs/heads/${branch}`; const exactBranch = (branchRef ?? "") .split("\n") @@ -2657,10 +2710,38 @@ export const make = Effect.gen(function* () { freshGitState, }); if (pullRequest === null || pullRequest.state === "open" || freshGitState) return pullRequest; - return yield* lookupBranchPullRequest(cacheCwd, branch, { + // A confirmed closed or merged result stays final until the branch or remotes change, + // with a periodic fresh lookup so a new PR from a reused branch still shows up. + const settledKey = `${cacheCwd}\u0000${branch}`; + const now = yield* Clock.currentTimeMillis; + const settled = settledPrLookupByBranchKey.get(settledKey); + // Read before any fresh lookup, so a change made during it forces another one. + const fingerprint = yield* readSettledPrLookupFingerprint(cacheCwd, branch); + if ( + settled !== undefined && + settled.url === pullRequest.url && + settled.fingerprint === fingerprint && + now - settled.confirmedAtMs < Duration.toMillis(SETTLED_PR_LOOKUP_RECHECK_INTERVAL) + ) { + return pullRequest; + } + const confirmed = yield* lookupBranchPullRequest(cacheCwd, branch, { refresh: false, freshGitState: true, }); + settledPrLookupByBranchKey.delete(settledKey); + if (confirmed !== null && confirmed.state !== "open") { + if (settledPrLookupByBranchKey.size >= PR_LOOKUP_CACHE_CAPACITY) { + const oldestKey = settledPrLookupByBranchKey.keys().next().value; + if (oldestKey !== undefined) settledPrLookupByBranchKey.delete(oldestKey); + } + settledPrLookupByBranchKey.set(settledKey, { + url: confirmed.url, + fingerprint, + confirmedAtMs: now, + }); + } + return confirmed; }); const invalidateLocalStatus: GitManager["Service"]["invalidateLocalStatus"] = Effect.fn( "invalidateLocalStatus", diff --git a/apps/server/src/guest/GuestShareService.test.ts b/apps/server/src/guest/GuestShareService.test.ts index a22e6182c7738..cfd8ca1363fa4 100644 --- a/apps/server/src/guest/GuestShareService.test.ts +++ b/apps/server/src/guest/GuestShareService.test.ts @@ -965,6 +965,82 @@ it.effect("serves the shared thread's files to its guest, never another thread's }).pipe(Effect.scoped, Effect.provide(layerTest)), ); +it.effect("gives a fork's guest the files on messages the fork shows from its source", () => + Effect.gen(function* () { + const { orchestrator } = yield* shareThread; + const guestShares = yield* GuestShareService.GuestShareService; + const { attachmentsDir } = yield* ServerConfig.ServerConfig; + const fileSystem = yield* FileSystem.FileSystem; + const ownerImage = Effect.fn(function* (name: string) { + const image = { + type: "image" as const, + id: ChatAttachmentId.make(createAttachmentId(threadId)!), + name, + mimeType: "image/png", + sizeBytes: PNG.length, + }; + yield* fileSystem.writeFile( + resolveAttachmentPath({ attachmentsDir, attachment: image })!, + PNG, + ); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`owner:${name}`), + threadId, + messageId: MessageId.make(`message:${name}`), + text: name, + attachments: [image], + dispatchMode: { type: "defer_start" }, + createdBy: "user", + creationSource: "web", + }); + return image; + }); + const inherited = yield* ownerImage("before.png"); + const run = (yield* orchestrator.getThreadRecords(threadId, ["runs"])).runs[0]!; + const now = yield* DateTime.now; + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make("event:run-done"), + type: "run.updated", + threadId, + runId: run.id, + providerInstanceId: instanceId, + occurredAt: now, + payload: { ...run, status: "completed", completedAt: now }, + }, + ], + }); + const forkId = ThreadId.make("thread:guest-share-fork"); + yield* orchestrator.dispatch({ + type: "thread.fork", + commandId: CommandId.make("fork"), + sourceThreadId: threadId, + targetThreadId: forkId, + sourcePoint: { type: "run", runId: run.id }, + createdBy: "user", + creationSource: "web", + }); + const later = yield* ownerImage("after.png"); + const { url } = yield* guestShares.create({ threadId: forkId }); + const { token: session } = yield* guestShares.redeem({ + credential: url.slice(url.indexOf("#") + 1), + name: "Sam", + }); + + assert.equal( + (yield* guestShares.attachment(session, inherited.id)).attachment.id, + inherited.id, + ); + // The source's messages after the fork point are not part of the fork. + assert.equal( + (yield* Effect.flip(guestShares.attachment(session, later.id))).reason, + "unavailable", + ); + }).pipe(Effect.provide(layerTest)), +); + it.effect("serves an image on the host only when an agent reply the guest sees shows it", () => Effect.gen(function* () { const { orchestrator, guestShares } = yield* shareThread; diff --git a/apps/server/src/guest/GuestShareService.ts b/apps/server/src/guest/GuestShareService.ts index 8298e10e4df5a..fd9a0efc3b66b 100644 --- a/apps/server/src/guest/GuestShareService.ts +++ b/apps/server/src/guest/GuestShareService.ts @@ -715,7 +715,18 @@ const make = Effect.gen(function* () { .find((candidate) => candidate.id === attachmentId), ), Effect.mapError(() => unavailable), - )); + )) ?? + // A fork shows its source's earlier messages, which stay in the source thread. + (thread.forkedFrom === null + ? undefined + : yield* orchestrator.getTimelinePage(thread.id, { attachmentId, limit: 1 }).pipe( + Effect.map(({ items }) => + items + .flatMap(({ item }) => (item.type === "user_message" ? item.attachments : [])) + .find((candidate) => candidate.id === attachmentId), + ), + Effect.mapError(() => unavailable), + )); const path = found === undefined ? null : resolveAttachmentPath({ attachmentsDir, attachment: found }); if (found === undefined || path === null) return yield* unavailable; diff --git a/apps/server/src/htmlRender/HtmlRender.test.ts b/apps/server/src/htmlRender/HtmlRender.test.ts index b2bc080ac863a..fa465ea011834 100644 --- a/apps/server/src/htmlRender/HtmlRender.test.ts +++ b/apps/server/src/htmlRender/HtmlRender.test.ts @@ -23,6 +23,7 @@ import * as ServerConfig from "../config.ts"; import * as HtmlRender from "./HtmlRender.ts"; import { HtmlRenderBrowserError } from "./headlessChrome.ts"; import * as PreviewBrowser from "../preview/PreviewBrowser.ts"; +import * as PreviewBrowserHost from "../preview/PreviewBrowserHost.ts"; const TEST_BROWSER_ENV = "T3CODE_TEST_HEADLESS_SHELL"; @@ -385,21 +386,57 @@ fs.createReadStream(null, { fd: 3 }).on("end", quit).on("data", (chunk) => { for (const { id, method, sessionId } of messages.map((raw) => JSON.parse(raw))) { if (method === "Browser.close") quit(); reply({ id, sessionId, result: results[method] ?? {} }); - if (method === "Page.navigate") reply({ sessionId, method: "Page.loadEventFired", params: {} }); + if (method === "Page.navigate") { + const value = "x" + "\u{1F600}".repeat(300); + const params = { type: "log", args: [{ type: "string", value }] }; + reply({ sessionId, method: "Runtime.consoleAPICalled", params }); + reply({ sessionId, method: "Page.loadEventFired", params: {} }); + } } }); `, { mode: 0o755 }, ); - yield* HtmlRender.HtmlRender.use((htmlRender) => + const preview = yield* HtmlRender.HtmlRender.use((htmlRender) => htmlRender.preview({ html: "

x

" }), ).pipe(Effect.provide(layerHtmlRender(executable))); + const logged = preview.consoleMessages[0]?.text ?? ""; + expect(logged.isWellFormed()).toBe(true); + expect(logged).toHaveLength(500); const exitedBy = yield* fileSystem.readFileString(exitRecord).pipe(Effect.option); expect(Option.getOrElse(exitedBy, () => "killed by a signal")).toBe("asked to close"); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); + + it.live("names this server's home in the browser setup command", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-fake-browser-", + }); + const executable = path.join(directory, "chrome-headless-shell"); + yield* fileSystem.writeFileString( + executable, + `#!${process.execPath}\nprocess.stderr.write(${JSON.stringify(PreviewBrowserHost.NO_SANDBOX_SIGNATURE)});\nprocess.exit(1);\n`, + { mode: 0o755 }, + ); + + const { error, baseDir } = yield* Effect.gen(function* () { + const { baseDir } = yield* ServerConfig.ServerConfig; + const htmlRender = yield* HtmlRender.HtmlRender; + const error = yield* htmlRender.preview({ html: "

x

" }).pipe(Effect.flip); + return { error, baseDir }; + }).pipe( + Effect.provide(layerHtmlRender(executable)), + Effect.provideService(HostProcess.UserId, 1000), + ); + expect(error).toBeInstanceOf(PreviewBrowserHost.PreviewBrowserSandboxError); + expect(error.message).toContain(`browser setup --base-dir ${baseDir}`); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); }, ); diff --git a/apps/server/src/htmlRender/HtmlRender.ts b/apps/server/src/htmlRender/HtmlRender.ts index cd3a7f3abe582..0147a39ab8eff 100644 --- a/apps/server/src/htmlRender/HtmlRender.ts +++ b/apps/server/src/htmlRender/HtmlRender.ts @@ -418,7 +418,11 @@ const make = Effect.gen(function* () { const browsers = yield* Semaphore.make(MAX_CONCURRENT_BROWSERS); // Keep Chrome sandboxed unless explicitly disabled; root must opt out to run. const noSandbox = PreviewBrowserHost.sandboxDisabled(yield* HostProcess.Environment); - const setupCommand = yield* resolveRootCliCommand(PreviewBrowserHost.SETUP_SUBCOMMAND); + // sudo resets the environment, so the command has to name this server's home. + const setupCommand = yield* resolveRootCliCommand(PreviewBrowserHost.SETUP_SUBCOMMAND, [ + "--base-dir", + config.baseDir, + ]); const launching = ( executable: string, diff --git a/apps/server/src/htmlRender/headlessChrome.ts b/apps/server/src/htmlRender/headlessChrome.ts index 13ce0c6845740..5e2a5bb0d5891 100644 --- a/apps/server/src/htmlRender/headlessChrome.ts +++ b/apps/server/src/htmlRender/headlessChrome.ts @@ -316,10 +316,13 @@ const launchBrowser = Effect.fnUntraced(function* (input: { return yield* Effect.void; } const text = consoleMessage.text.replaceAll(PAGE_URL, PAGE_NAME); + // Half an emoji serializes to a lone surrogate escape, which provider JSON parsers reject. + const last = text.charCodeAt(MAX_CONSOLE_TEXT_CHARS - 1); + const end = + last >= 0xd800 && last <= 0xdbff ? MAX_CONSOLE_TEXT_CHARS - 1 : MAX_CONSOLE_TEXT_CHARS; page.consoleMessages.push({ level: consoleMessage.level, - text: - text.length > MAX_CONSOLE_TEXT_CHARS ? `${text.slice(0, MAX_CONSOLE_TEXT_CHARS)}…` : text, + text: text.length > MAX_CONSOLE_TEXT_CHARS ? `${text.slice(0, end)}…` : text, }); return yield* Effect.void; }); diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index e8e1e81d04a84..7b54d282ca800 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -896,9 +896,9 @@ it.effect("bounds the snapshot text even when nothing but logs and the title are Effect.gen(function* () { const oversized = { ...snapshotResult, - title: "t".repeat(70_000), + title: `t${"😀".repeat(35_000)}`, interactiveElements: [], - consoleEntries: [{ level: "log", text: "x".repeat(70_000), timestamp: "t" }], + consoleEntries: [{ level: "log", text: `x${"😀".repeat(35_000)}`, timestamp: "t" }], }; yield* serveSnapshots("mcp-bounded-logs-client", oversized); @@ -913,8 +913,10 @@ it.effect("bounds the snapshot text even when nothing but logs and the title are readonly title: string; readonly consoleEntries: ReadonlyArray<{ readonly text: string }>; }; - expect(parsed.title.length).toBe(2_049); - expect(parsed.consoleEntries[0]?.text.length).toBe(501); + expect(parsed.title.isWellFormed()).toBe(true); + expect(parsed.title.length).toBe(2_048); + expect(parsed.consoleEntries[0]?.text.isWellFormed()).toBe(true); + expect(parsed.consoleEntries[0]?.text.length).toBe(500); const notice = snapshot.content[2]; const noticeText = notice?.type === "text" ? notice.text : ""; expect(noticeText).toContain("url or title after 2048 characters"); diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index ef2004572c74d..3ae92bcaeb11a 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -259,8 +259,12 @@ const MAX_SNAPSHOT_IDENTIFIER_CHARS = 2_048; const encodeJsonText = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); const utf8Length = (text: string) => Buffer.byteLength(text, "utf8"); -const cutText = (text: string, max: number) => - text.length > max ? `${text.slice(0, max)}…` : text; +const cutText = (text: string, max: number) => { + if (text.length <= max) return text; + // Half an emoji serializes to a lone surrogate escape, which provider JSON parsers reject. + const last = text.charCodeAt(max - 1); + return `${text.slice(0, last >= 0xd800 && last <= 0xdbff ? max - 1 : max)}…`; +}; const cutEntryStrings = (entry: A): A => typeof entry === "object" && entry !== null diff --git a/apps/server/src/mcp/McpInvocationContext.ts b/apps/server/src/mcp/McpInvocationContext.ts index 181ac3a9cfb2c..842d18ad9c8bc 100644 --- a/apps/server/src/mcp/McpInvocationContext.ts +++ b/apps/server/src/mcp/McpInvocationContext.ts @@ -12,7 +12,7 @@ import { import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; -import { formatThreadLink } from "@t3tools/shared/threadLinks"; +import { formatThreadLink, hostedThreadLink } from "@t3tools/shared/threadLinks"; const ALL_MCP_CAPABILITIES = [ "preview", @@ -45,8 +45,8 @@ export interface McpClientCaller { * another environment passing on the limits of the agent it calls for. */ readonly narrowedTo?: ClientModes | undefined; - /** The origin an outside client reached; its thread links open the web app there. */ - readonly webOrigin?: string | undefined; + /** An outside client's thread links go through this hosted app, as Copy thread's do. */ + readonly hostedAppUrl?: string | undefined; } /** @@ -135,16 +135,15 @@ export interface McpInvocationScope { /** * T3 Code renders `t3-thread://` links as thread chips. Other MCP clients cannot open them, so - * they get a web URL on the origin they reached instead. + * they get the hosted link Copy thread copies, which opens the thread in either app or a browser. */ export const threadLinkFor = ( scope: Pick, threadId: ThreadId, ): string => { - const origin = scope.client?.webOrigin; - return origin === undefined - ? formatThreadLink({ environmentId: scope.environmentId, threadId }) - : `${origin}/${encodeURIComponent(scope.environmentId)}/${encodeURIComponent(threadId)}`; + const hostedAppUrl = scope.client?.hostedAppUrl; + const ref = { environmentId: scope.environmentId, threadId }; + return hostedAppUrl === undefined ? formatThreadLink(ref) : hostedThreadLink(ref, hostedAppUrl); }; export class McpInvocationContext extends Context.Service< diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index 36ccf46442aa9..5a2ca00404ccc 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -1405,6 +1405,7 @@ describe("OrchestratorMcpService provider resolution", () => { Effect.gen(function* () { const disabledAntigravityInstanceId = ProviderInstanceId.make("antigravity-alt"); const disabledWithoutAdapterInstanceId = ProviderInstanceId.make("codex-without-adapter"); + const projectDisabledInstanceId = ProviderInstanceId.make("codex-project-off"); const forkOnlyInstanceId = ProviderInstanceId.make("forkOnly"); const forkShadow = yield* buildUnavailableProviderSnapshot({ driverKind: "forkOnly", @@ -1450,6 +1451,11 @@ describe("OrchestratorMcpService provider resolution", () => { model: "gpt-5.4", enabled: false, }), + providerSnapshot({ + instanceId: projectDisabledInstanceId, + driver: ProviderDriverKind.make("codex"), + model: "gpt-5.4", + }), forkShadow, ]; const layerDependencies = Layer.mergeAll( @@ -1458,12 +1464,34 @@ describe("OrchestratorMcpService provider resolution", () => { Layer.mock(ThreadManagementService.ThreadManagementService)({ getThreadRecords: () => Effect.succeed(parentProjection([])), }), - ServerSettings.layerTest(), + ServerSettings.layerTest({ + providerInstances: { + [ProviderInstanceId.make("pi")]: { + driver: ProviderDriverKind.make("pi"), + enabled: true, + }, + [ProviderInstanceId.make("acpRegistry")]: { + driver: ProviderDriverKind.make("acpRegistry"), + enabled: true, + }, + [antigravityInstanceId]: { + driver: ProviderDriverKind.make("antigravity"), + enabled: true, + }, + [projectDisabledInstanceId]: { driver: ProviderDriverKind.make("codex") }, + }, + projectSettingsOverrides: { + [projectId]: { + providerInstances: { [projectDisabledInstanceId]: { enabled: false } }, + }, + }, + }), Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.sync(() => providers), }), adapterRegistryLayer([ codexInstanceId, + projectDisabledInstanceId, ProviderInstanceId.make("claudeAgent"), ProviderInstanceId.make("pi"), ProviderInstanceId.make("acpRegistry"), @@ -1535,6 +1563,11 @@ describe("OrchestratorMcpService provider resolution", () => { assert.isFalse(disabled!.canRunCrossProviderChildTask); assert.deepEqual(disabled!.constraints, ["Provider instance is disabled."]); + const projectDisabled = byId.get(projectDisabledInstanceId); + assert.isDefined(projectDisabled); + assert.isFalse(projectDisabled!.canRunChildTask); + assert.deepEqual(projectDisabled!.constraints, ["Provider instance is disabled."]); + const disabledWithoutAdapter = byId.get(disabledWithoutAdapterInstanceId); assert.isDefined(disabledWithoutAdapter); assert.isFalse(disabledWithoutAdapter!.canRunChildTask); diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 521710ba66383..b4018436f3ee2 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -67,6 +67,7 @@ import { type ServerProviderModel, ThreadId, isOrchestrationV2WorkActive, + resolveProjectProviderInstanceEnabled, } from "@t3tools/contracts"; import { getProviderOptionCurrentValue } from "@t3tools/shared/model"; import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError"; @@ -2451,8 +2452,16 @@ const make = Effect.gen(function* () { runtimeMode: limits.runtimeMode, interactionMode: limits.interactionMode, providers: providers.map((provider) => { + // Snapshots list instances any project enables; delegate_task checks the caller's project. + const enabledForCaller = + parent === undefined || + resolveProjectProviderInstanceEnabled( + settings, + parent.thread.projectId, + provider.instanceId, + ); const constraints = providerConstraints( - provider, + enabledForCaller ? provider : { ...provider, enabled: false }, orchestrationCapableInstanceIds.has(provider.instanceId), ); const { listed, omitted } = listCapabilityModels({ diff --git a/apps/server/src/mcp/WorktreeMcpService.test.ts b/apps/server/src/mcp/WorktreeMcpService.test.ts index 688b4bee44156..996680f44e9bf 100644 --- a/apps/server/src/mcp/WorktreeMcpService.test.ts +++ b/apps/server/src/mcp/WorktreeMcpService.test.ts @@ -19,6 +19,7 @@ import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as PlatformError from "effect/PlatformError"; import * as Schema from "effect/Schema"; import * as GitWorkflowService from "../git/GitWorkflowService.ts"; @@ -96,6 +97,7 @@ interface HarnessOptions { readonly currentBranch?: string | null; readonly notARepo?: boolean; readonly newWorktreesStartFromOrigin?: boolean; + readonly projectNewWorktreesStartFromOrigin?: boolean; readonly setupScript?: "started" | "no-script" | "fails" | "dies"; readonly dispatchFails?: boolean; readonly dispatchDies?: boolean; @@ -114,6 +116,7 @@ interface HarnessOptions { readonly removeWorktreeFails?: boolean; readonly deleteLocalBranchFails?: boolean; readonly createWorktreeGate?: Effect.Effect; + readonly existsFails?: boolean; } const makeHarness = (options: HarnessOptions = {}) => { @@ -233,7 +236,7 @@ const makeHarness = (options: HarnessOptions = {}) => { ), ), ); - const listRefs = vi.fn((input: { readonly query?: string | undefined }) => + const listRefs = vi.fn((input: { readonly cwd: string; readonly query?: string | undefined }) => Effect.succeed({ refs: options.existingBranchWorktreePath === undefined @@ -302,6 +305,15 @@ const makeHarness = (options: HarnessOptions = {}) => { } satisfies Partial), ServerSettings.layerTest({ newWorktreesStartFromOrigin: options.newWorktreesStartFromOrigin ?? false, + ...(options.projectNewWorktreesStartFromOrigin === undefined + ? {} + : { + projectSettingsOverrides: { + [projectId]: { + newWorktreesStartFromOrigin: options.projectNewWorktreesStartFromOrigin, + }, + }, + }), }), Layer.mock(GitWorkflowService.GitWorkflowService)({ listRefs, @@ -321,6 +333,19 @@ const makeHarness = (options: HarnessOptions = {}) => { } satisfies Partial), NodeServices.layer, NodeCrypto.layer, + options.existsFails + ? FileSystem.layerNoop({ + exists: (path) => + Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "exists", + pathOrDescriptor: path, + }), + ), + }) + : Layer.empty, ), ), ); @@ -328,6 +353,7 @@ const makeHarness = (options: HarnessOptions = {}) => { return { layer, scope, + listRefs, dispatch, sendToThread, fetchWorktreeBaseCommit, @@ -627,6 +653,15 @@ describe("t3_worktree_handoff", () => { }); }); + it.effect("defaults startFromOrigin to the project's override", () => { + const harness = makeHarness({ projectNewWorktreesStartFromOrigin: true }); + return Effect.gen(function* () { + const result = yield* runHandoff(harness, { branch: "feature/project-origin" }); + expect(harness.fetchWorktreeBaseCommit).toHaveBeenCalled(); + expect(result.startedFromOrigin).toBe(true); + }); + }); + it.effect("removes the created worktree when the thread update dies with a defect", () => { const harness = makeHarness({ dispatchDies: true }); return Effect.gen(function* () { @@ -784,6 +819,7 @@ describe("t3_worktree_status", () => { worktreePath: "/worktrees/project/existing", branch: "feature/existing", }, + projectNewWorktreesStartFromOrigin: true, }); return Effect.gen(function* () { const result = yield* runStatus(harness); @@ -791,8 +827,39 @@ describe("t3_worktree_status", () => { attached: true, worktreePath: "/worktrees/project/existing", branch: "feature/existing", - defaultStartFromOrigin: false, + defaultStartFromOrigin: true, }); }); }); }); + +describe("t3_worktree_list", () => { + const listRefs = (harness: ReturnType, worktreePath: string | null) => + Effect.gen(function* () { + const service = yield* WorktreeMcpService.WorktreeMcpService; + return yield* service.listRefs({ worktreePath, workspaceRoot }); + }).pipe(Effect.provide(harness.layer)); + + it.effect("lists refs from the thread's worktree, or the project root once it was removed", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const worktreePath = yield* fs.makeTempDirectoryScoped({ prefix: "worktree-list-" }); + const harness = makeHarness(); + yield* listRefs(harness, worktreePath); + yield* listRefs(harness, `${worktreePath}/removed`); + expect(harness.listRefs.mock.calls.map(([input]) => input.cwd)).toEqual([ + worktreePath, + workspaceRoot, + ]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("fails instead of falling back when the worktree cannot be checked", () => { + const harness = makeHarness({ existsFails: true }); + return Effect.gen(function* () { + const exit = yield* Effect.exit(listRefs(harness, "/worktrees/project/unreadable")); + expectTypedFailure(exit, { _tag: "PlatformError" }); + expect(harness.listRefs).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/apps/server/src/mcp/WorktreeMcpService.ts b/apps/server/src/mcp/WorktreeMcpService.ts index 9da20ba832ec9..ef5e1dbe29fb4 100644 --- a/apps/server/src/mcp/WorktreeMcpService.ts +++ b/apps/server/src/mcp/WorktreeMcpService.ts @@ -1,6 +1,7 @@ import * as Path from "effect/Path"; import { CommandId, + type GitCommandError, MessageId, type ProjectId, WorktreeMcpFailure, @@ -9,13 +10,18 @@ import { type WorktreeMcpHandoffResult, type WorktreeMcpSetupScriptStatus, type WorktreeMcpStatusResult, + type VcsListRefsInput, + type VcsListRefsResult, } from "@t3tools/contracts"; +import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import type * as PlatformError from "effect/PlatformError"; import * as GitWorkflowService from "../git/GitWorkflowService.ts"; import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts"; @@ -35,6 +41,12 @@ export class WorktreeMcpService extends Context.Service< readonly status: ( scope: McpInvocationScope, ) => Effect.Effect; + readonly listRefs: ( + input: Omit & { + readonly worktreePath: string | null; + readonly workspaceRoot: string; + }, + ) => Effect.Effect; } >()("t3/mcp/WorktreeMcpService") {} @@ -58,6 +70,7 @@ const asOperationFailed = (prefix: string) => const make = Effect.gen(function* () { const path = yield* Path.Path; const random = yield* SecureRandom.SecureRandom; + const fileSystem = yield* FileSystem.FileSystem; const threadManagement = yield* ThreadManagementService.ThreadManagementService; const projects = yield* ProjectService.ProjectService; const serverSettings = yield* ServerSettings.ServerSettingsService; @@ -118,10 +131,14 @@ const make = Effect.gen(function* () { ), ); - const readDefaultStartFromOrigin = serverSettings.getSettings.pipe( - Effect.map((settings) => settings.newWorktreesStartFromOrigin), - asOperationFailed("Unable to read server settings"), - ); + const readDefaultStartFromOrigin = (projectId: ProjectId) => + serverSettings.getSettings.pipe( + Effect.map( + (settings) => + resolveProjectSettings(settings, projectId).settings.newWorktreesStartFromOrigin, + ), + asOperationFailed("Unable to read server settings"), + ); const handoffIds = (scope: McpThreadInvocationScope) => random.uuidV4().pipe( @@ -145,6 +162,7 @@ const make = Effect.gen(function* () { input: WorktreeMcpHandoffInput, projectCwd: string, projectBranch: string | null, + projectId: ProjectId, ) { const baseRef = input.baseRef ?? projectBranch; if (baseRef === null) { @@ -153,7 +171,7 @@ const make = Effect.gen(function* () { "Could not determine the current branch of the project workspace (detached HEAD?). Pass baseRef explicitly.", ); } - const startFromOrigin = input.startFromOrigin ?? (yield* readDefaultStartFromOrigin); + const startFromOrigin = input.startFromOrigin ?? (yield* readDefaultStartFromOrigin(projectId)); if (!startFromOrigin) return { baseRef, startFromOrigin, startRef: baseRef }; const resolvedRemoteBase = yield* gitWorkflow .fetchWorktreeBaseCommit({ cwd: projectCwd, refName: baseRef }) @@ -246,7 +264,7 @@ const make = Effect.gen(function* () { existingWorktreePath === null ? { created: true as const, - base: yield* resolveNewWorktreeBase(input, projectCwd, localStatus.refName), + base: yield* resolveNewWorktreeBase(input, projectCwd, localStatus.refName, project.id), } : { created: false as const, path: existingWorktreePath }; @@ -456,7 +474,7 @@ const make = Effect.gen(function* () { const projection = yield* loadThread(scope); const project = yield* loadProject(scope, projection.thread.projectId); - const defaultStartFromOrigin = yield* readDefaultStartFromOrigin; + const defaultStartFromOrigin = yield* readDefaultStartFromOrigin(project.id); const result: WorktreeMcpStatusResult = { attached: projection.thread.worktreePath !== null, @@ -469,7 +487,18 @@ const make = Effect.gen(function* () { }, ); - return WorktreeMcpService.of({ handoff, status }); + const listRefs: WorktreeMcpService["Service"]["listRefs"] = Effect.fn( + "WorktreeMcpService.listRefs", + )(function* ({ worktreePath, workspaceRoot, ...refs }) { + // Storage cleanup removes a worktree but keeps its path on the thread. + const cwd = + worktreePath !== null && (yield* fileSystem.exists(worktreePath)) + ? worktreePath + : workspaceRoot; + return yield* gitWorkflow.listRefs({ ...refs, cwd }); + }); + + return WorktreeMcpService.of({ handoff, status, listRefs }); }); export const layer: Layer.Layer< @@ -477,6 +506,7 @@ export const layer: Layer.Layer< never, | SecureRandom.SecureRandom | Path.Path + | FileSystem.FileSystem | ThreadManagementService.ThreadManagementService | ProjectService.ProjectService | ServerSettings.ServerSettingsService diff --git a/apps/server/src/mcp/toolkits/project/handlers.test.ts b/apps/server/src/mcp/toolkits/project/handlers.test.ts index 97e0e903bf4d3..6c2c5607cd468 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.test.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.test.ts @@ -371,7 +371,7 @@ const clientLaunchHarness = (input: { readonly workspaceRoot?: string; readonly checkNewWorktree?: ThreadLaunch.ThreadLaunchService["Service"]["checkNewWorktree"]; readonly launchError?: ThreadLaunch.ThreadLaunchPullRequestError; - readonly webOrigin?: string; + readonly hostedAppUrl?: string; /** The orchestrator refuses the new thread's mode, as under a managed ceiling. */ readonly refusedByCeiling?: true; }) => { @@ -412,7 +412,7 @@ const clientLaunchHarness = (input: { sessionId: "session-1", label: "Claude Code", access: input.runtimeModeCeiling, - webOrigin: input.webOrigin, + hostedAppUrl: input.hostedAppUrl, }, issuedAt: 0, capabilities: new Set(["orchestration" as const]), @@ -511,18 +511,18 @@ it.effect( readonly runtimeMode?: RuntimeMode; readonly targeted?: false; readonly overrideModel?: ModelSelection; - readonly webOrigin?: string; + readonly hostedAppUrl?: string; readonly refusedByCeiling?: true; }, ) { - const { targeted, overrideModel, webOrigin, refusedByCeiling, ...input } = params; + const { targeted, overrideModel, hostedAppUrl, refusedByCeiling, ...input } = params; const launched: Array = []; const { projectId, modelSelection, dependencies } = clientLaunchHarness({ runtimeModeCeiling, defaultRuntimeMode, launched, ...(overrideModel === undefined ? {} : { overrideModel }), - ...(webOrigin === undefined ? {} : { webOrigin }), + ...(hostedAppUrl === undefined ? {} : { hostedAppUrl }), ...(refusedByCeiling === undefined ? {} : { refusedByCeiling }), }); const toolkit = yield* ProjectToolkit.pipe( @@ -556,11 +556,11 @@ it.effect( const aboveCeiling = yield* launchWith("auto-accept-edits", "full-access", { title: "Fix", - webOrigin: "https://box.example.ts.net", + hostedAppUrl: "https://t3.example.com", }); expect(aboveCeiling.launched[0]?.runtimeMode).toBe("auto-accept-edits"); expect(aboveCeiling.result).toMatchObject({ - link: `https://box.example.ts.net/environment/${encodeURIComponent(aboveCeiling.launched[0]?.threadId ?? "")}`, + link: `https://t3.example.com/threads/environment/${encodeURIComponent(aboveCeiling.launched[0]?.threadId ?? "")}`, }); const requested = yield* launchWith("full-access", "approval-required", { diff --git a/apps/server/src/mcp/toolkits/worktree/handlers.ts b/apps/server/src/mcp/toolkits/worktree/handlers.ts index c6fabe18c6fb8..2e78dd2a4f996 100644 --- a/apps/server/src/mcp/toolkits/worktree/handlers.ts +++ b/apps/server/src/mcp/toolkits/worktree/handlers.ts @@ -1,6 +1,5 @@ import { OrchestratorMcpFailure } from "@t3tools/contracts"; import * as Option from "effect/Option"; -import * as GitWorkflow from "../../../git/GitWorkflowService.ts"; import * as Project from "../../../project/ProjectService.ts"; import { readThread, unavailable } from "../../threadAccess.ts"; import * as Effect from "effect/Effect"; @@ -30,9 +29,13 @@ const handlers = { code: "invalid_request", message: "The project was not found.", }); - const git = yield* GitWorkflow.GitWorkflowService; - return yield* git - .listRefs({ ...refs, cwd: thread.worktreePath ?? project.value.workspaceRoot }) + const service = yield* WorktreeMcpService.WorktreeMcpService; + return yield* service + .listRefs({ + ...refs, + worktreePath: thread.worktreePath, + workspaceRoot: project.value.workspaceRoot, + }) .pipe(Effect.mapError(unavailable)); }), ), diff --git a/apps/server/src/mcp/toolkits/worktree/tools.ts b/apps/server/src/mcp/toolkits/worktree/tools.ts index e0c82d8da35ad..6cd574e8c9384 100644 --- a/apps/server/src/mcp/toolkits/worktree/tools.ts +++ b/apps/server/src/mcp/toolkits/worktree/tools.ts @@ -10,7 +10,6 @@ import { ThreadId, } from "@t3tools/contracts"; import * as Schema from "effect/Schema"; -import * as GitWorkflowService from "../../../git/GitWorkflowService.ts"; import * as ProjectService from "../../../project/ProjectService.ts"; import * as ThreadManagementService from "../../../orchestration-v2/ThreadManagementService.ts"; import { Tool, Toolkit } from "effect/ai"; @@ -78,7 +77,7 @@ const WorktreeListTool = Tool.make("t3_worktree_list", { McpInvocationContext.McpInvocationContext, ThreadManagementService.ThreadManagementService, ProjectService.ProjectService, - GitWorkflowService.GitWorkflowService, + WorktreeMcpService.WorktreeMcpService, ], }) .annotate(Tool.Readonly, true) diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index 4fc1e3970b68b..9385d35a0e370 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -342,6 +342,71 @@ describe("acpToolCallDiffPatch", () => { "", ].join("\n"), ); + assert.equal( + acpToolCallDiffPatch([ + { + type: "diff", + changes: [{ operation: "modify", path: "/repo/a.ts" }], + patch: { + format: "git_patch", + text: "diff --git a/repo/a.ts b/repo/a.ts\n@@ -1 +1 @@\n-a\n+A\n", + }, + }, + { + type: "diff", + changes: [{ operation: "modify", path: "/repo/b.ts" }], + patch: { + format: "git_patch", + text: "diff --git a/repo/b.ts b/repo/b.ts\n@@ -1 +1 @@\n-b\n+B\n", + }, + }, + { type: "diff", path: "/repo/c.ts", oldText: "c\n", newText: "C\n" }, + ]), + [ + "diff --git a/repo/a.ts b/repo/a.ts", + "@@ -1 +1 @@", + "-a", + "+A", + "", + "diff --git a/repo/b.ts b/repo/b.ts", + "@@ -1 +1 @@", + "-b", + "+B", + "", + 'diff --git "a//repo/c.ts" "b//repo/c.ts"', + "--- /repo/c.ts", + "+++ /repo/c.ts", + "@@ -1,1 +1,1 @@", + "-c", + "+C", + "", + ].join("\n"), + ); + assert.equal( + acpToolCallDiffPatch([ + { type: "diff", path: "/repo/new.txt", oldText: null, newText: "hello\n" }, + { + type: "diff", + changes: [{ operation: "modify", path: "/repo/a.ts" }], + patch: { format: "git_patch", text: "@@ -1 +1 @@\n-a\n+A\n" }, + }, + ]), + [ + 'diff --git "a//repo/new.txt" "b//repo/new.txt"', + "--- /dev/null", + "+++ /repo/new.txt", + "@@ -0,0 +1,1 @@", + "+hello", + "", + 'diff --git "a//repo/a.ts" "b//repo/a.ts"', + '--- "a//repo/a.ts"', + '+++ "b//repo/a.ts"', + "@@ -1 +1 @@", + "-a", + "+A", + "", + ].join("\n"), + ); }); }); @@ -4533,6 +4598,140 @@ describe("AcpAdapterV2", () => { }).pipe(Effect.provide(layerTest), Effect.scoped), ); + it.effect.each([ + { + label: "answers Always allow with the agent's allow-always option", + options: [ + { optionId: "allow-once", name: "Allow once", kind: "allow_once" }, + { optionId: "allow-always", name: "Always allow", kind: "allow_always" }, + { optionId: "reject-once", name: "Reject", kind: "reject_once" }, + ], + optionId: "allow-always", + }, + { + label: "answers Always allow with allow-once when the agent offers no allow-always option", + options: [ + { optionId: "allow-once", name: "Allow once", kind: "allow_once" }, + { optionId: "reject-once", name: "Reject", kind: "reject_once" }, + ], + optionId: "allow-once", + }, + ] as const)("$label", ({ options, optionId }) => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const path = yield* Path.Path; + const selfInvocation = yield* resolveSelfInvocation(); + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let requestPermission: Parameters[0] | undefined; + let runtimeInput: AcpAdapterV2RuntimeInput | undefined; + const makeRuntime = makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_HANG_PROMPT_FOREVER: "1" }, + wrapRuntime: (runtime) => ({ + ...runtime, + handleRequestPermission: (handler) => + Effect.sync(() => { + requestPermission = handler; + }).pipe(Effect.andThen(runtime.handleRequestPermission(handler))), + }), + }); + const instanceId = ProviderInstanceId.make("acp-test-always-allow"); + const adapter = yield* makeAcpAdapterV2({ + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + interruptPromptOnCancel: true, + makeRuntime: (input) => + Effect.sync(() => { + runtimeInput = input; + }).pipe(Effect.andThen(makeRuntime(input))), + }, + selfInvocation, + clientTerminals: {}, + }); + const threadId = ThreadId.make("thread-acp-always-allow"); + const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "approval-required", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-always-allow"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const turnFiber = yield* runtime + .startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + }), + ) + .pipe(Effect.forkScoped); + if (requestPermission === undefined) { + return yield* Effect.die("ACP runtime must register a permission handler"); + } + + const permissionFiber = yield* requestPermission( + { + sessionId: "mock-session-1", + toolCall: { toolCallId: "always-allow-tool", title: "Run tests", kind: "execute" }, + options, + }, + { requestId: "always-allow-request", method: "session/request_permission" }, + ).pipe(Effect.forkScoped); + const pending = Option.getOrThrow( + yield* runtime.events.pipe( + Stream.filter( + (event) => + event.type === "runtime_request.updated" && event.runtimeRequest.status === "pending", + ), + Stream.runHead, + ), + ); + if ( + pending.type !== "runtime_request.updated" || + pending.runtimeRequest.providerTurnId === null + ) { + return yield* Effect.die("Expected an ACP approval request"); + } + const responseFiber = yield* runtime + .respondToRuntimeRequest({ + requestId: pending.runtimeRequest.id, + decision: "acceptForSession", + }) + .pipe(Effect.forkScoped); + const { outcome } = yield* Fiber.join(permissionFiber); + if (runtimeInput?.onOutgoingResponse === undefined) { + return yield* Effect.die("ACP runtime must expose native response acknowledgements"); + } + yield* runtimeInput.onOutgoingResponse("always-allow-request"); + yield* Fiber.join(responseFiber); + + yield* runtime.interruptTurn({ + providerThread, + providerTurnId: pending.runtimeRequest.providerTurnId, + }); + yield* Fiber.join(turnFiber); + assert.deepEqual(outcome, { outcome: "selected", optionId }); + }).pipe(Effect.provide(layerTest), Effect.scoped), + ); + it.effect("replaces the ACP session and clears conversation state on rollback", () => Effect.gen(function* () { const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; @@ -4554,10 +4753,16 @@ describe("AcpAdapterV2", () => { browserToolsAvailable: false, }); const runtimeInputs: Array = []; + const nativeRuntimes: Array = []; + let sessionEventCount = 0; const makeRuntime = makeMockRuntime({ childProcessSpawner, mockAgentPath, environment: { T3_ACP_PROMPT_DELAY_MS: "100" }, + wrapRuntime: (nativeRuntime) => { + nativeRuntimes.push(nativeRuntime); + return nativeRuntime; + }, }); const adapter = yield* makeAcpAdapterV2({ instanceId, @@ -4568,6 +4773,10 @@ describe("AcpAdapterV2", () => { Effect.sync(() => runtimeInputs.push(runtimeInput)).pipe( Effect.andThen(makeRuntime(runtimeInput)), ), + onSessionEvent: () => + Effect.sync(() => { + sessionEventCount += 1; + }), }, selfInvocation, }); @@ -4662,6 +4871,7 @@ describe("AcpAdapterV2", () => { }); assert.deepEqual(snapshotAfterRollback.providerTurns, []); assert.deepEqual(snapshotAfterRollback.messages, []); + const sessionEventsBeforeSecondTurn = sessionEventCount; yield* runtime.startTurn( makeTurnInput({ @@ -4688,6 +4898,8 @@ describe("AcpAdapterV2", () => { } } assert.equal(secondStatus, "completed"); + yield* nativeRuntimes[1]!.drainEvents; + assert.isAbove(sessionEventCount, sessionEventsBeforeSecondTurn); }).pipe(Effect.provide(layerTest), Effect.scoped), ); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index ffdf6e2ba0058..70dfdad84b7d8 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -755,7 +755,7 @@ describe("ClaudeAdapterV2 context usage", () => { output_tokens: 1, cache_creation: { ephemeral_1h_input_tokens: 0, ephemeral_5m_input_tokens: 300 }, }, - CLAUDE_TEST_MODEL_SELECTION, + null, "2026-08-29T00:03:00.000Z", "", DateTime.makeUnsafe("2026-08-29T00:00:00.000Z"), @@ -3150,6 +3150,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { readonly cache_creation_input_tokens: number; readonly cache_read_input_tokens: number; }; + readonly modelUsage?: SDKResultMessage["modelUsage"]; }) => claudeSdkFrame({ type: "result", @@ -3167,7 +3168,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { cache_creation_input_tokens: 0, cache_read_input_tokens: 0, }, - modelUsage: {}, + modelUsage: input.modelUsage ?? {}, permission_denials: [], uuid: input.uuid, session_id: WAKE_NATIVE_SESSION, @@ -7097,9 +7098,15 @@ describe("ClaudeAdapterV2 background wake turns", () => { it.effect("never records a subagent uuid for a turn with no main-thread assistant message", () => Effect.gen(function* () { + const forkCalls: Array = []; const harness = yield* makeWakeHarnessWithOptions({ close: (sdkMessages) => Queue.shutdown(sdkMessages), freshQueueOnReopen: true, + forkSession: (input) => + Effect.sync(() => { + forkCalls.push(input); + return { sessionId: "forked-after-compact-rollback" }; + }), }); const now = yield* DateTime.now; const subagentUuid = "00000000-0000-4000-8000-000000000644"; @@ -7180,6 +7187,14 @@ describe("ClaudeAdapterV2 background wake turns", () => { pastSilentTurn.providerThread.nativeConversationHeadRef?.nativeId, earlierCursor, ); + // Forks omit rolled-back turns, which Claude's transcript still holds past the pinned head. + yield* harness.runtime.forkThread({ + sourceProviderThread: pastSilentTurn.providerThread, + sourceProviderTurns: turnsAcrossSilent.slice(0, 2), + providerTurnId: providerTurn.id, + targetThreadId: ThreadId.make("thread-claude-fork-after-compact-rollback"), + }); + assert.equal(forkCalls[0]?.options.upToMessageId, earlierCursor); const interruptedBeforeCompact = yield* rollbackTo( [ { @@ -7864,6 +7879,67 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect("measures a model outside the catalog against the window Claude reports", () => + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const modelSelection = { + instanceId: ProviderInstanceId.make(ClaudeAdapterV2.CLAUDE_PROVIDER), + model: "gateway-model", + } satisfies ModelSelection; + assert.strictEqual(harness.runtime.getModelContextWindow?.(modelSelection), undefined); + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-reported-context-window"), + text: "Continue.", + attachments: [], + modelSelection, + }), + ); + const usage = { ...zeroUsage, input_tokens: 9_667, output_tokens: 2 }; + const frames = [ + usageStreamFrame({ type: "message_start", message: { id: "msg_gateway", usage } }), + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000680", + result: "OK", + modelUsage: { + "gateway-model": { + inputTokens: 9_667, + outputTokens: 2, + cacheReadInputTokens: 0, + cacheCreationInputTokens: 0, + webSearchRequests: 0, + costUSD: 0, + contextWindow: 1_048_576, + maxOutputTokens: 32_000, + }, + }, + }), + ]; + for (const frame of frames) yield* Queue.offer(harness.sdkMessages, frame); + const terminal = yield* Queue.take(harness.terminalReceipts); + assert.equal(terminal.status, "completed"); + const windows = harness.events.flatMap((event) => + event.type === "provider_turn.updated" && event.providerTurn.tokenUsage + ? [event.providerTurn.tokenUsage.maxTokens] + : [], + ); + assert.deepEqual(windows, [null, 1_048_576]); + assert.strictEqual(harness.runtime.getModelContextWindow?.(modelSelection), 1_048_576); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + it.effect("merges cumulative stream usage into the message_start counts", () => Effect.gen(function* () { const harness = yield* makeWakeHarnessWithOptions({ @@ -8105,6 +8181,10 @@ describe("ClaudeAdapterV2 background wake turns", () => { providerThread: harness.providerThread, now, attemptId: RunAttemptId.make("claude-native-compact-attempt"), + modelSelection: { + instanceId: ProviderInstanceId.make(ClaudeAdapterV2.CLAUDE_PROVIDER), + model: "gateway-model", + }, text: " /COMPACT ", attachments: [], }), @@ -8124,6 +8204,18 @@ describe("ClaudeAdapterV2 background wake turns", () => { makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000202", result: "Compacted conversation.", + modelUsage: { + "gateway-model": { + inputTokens: 1, + outputTokens: 1, + cacheReadInputTokens: 0, + cacheCreationInputTokens: 0, + webSearchRequests: 0, + costUSD: 0, + contextWindow: 200_000, + maxOutputTokens: 32_000, + }, + }, }), ); yield* Queue.take(harness.terminalReceipts); @@ -8144,6 +8236,12 @@ describe("ClaudeAdapterV2 background wake turns", () => { event.providerTurn.tokenUsage?.usedTokens === 400, ); assert.isDefined(watermark); + const windows = harness.events.flatMap((event) => + event.type === "provider_turn.updated" && event.providerTurn.tokenUsage + ? [event.providerTurn.tokenUsage.maxTokens] + : [], + ); + assert.deepEqual(windows, [null, 200_000]); const completed = harness.events.findLast( (event) => event.type === "provider_turn.updated", ); @@ -8157,6 +8255,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { outputTokens: 1, cachedInputTokens: 0, cacheCreationTokens: 0, + otherUsage: { inputTokens: 0, outputTokens: 0 }, }); } }).pipe( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 5c5ca4e2ede47..9b47ae5521fd5 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -27,6 +27,7 @@ import { addClaudeLedgerUsage, claudeLedgerDelta, claudeModelLedger, + claudeReportedContextWindows, normalizeClaudeTurnTokenUsage, type ClaudeLedgerUsage, type ClaudeModelLedger, @@ -232,12 +233,6 @@ import { export const CLAUDE_PROVIDER = ProviderDriverKind.make("claudeAgent"); const CLAUDE_AGENT_SDK_QUERY_PROTOCOL = "claude-agent-sdk.query" as const; -function claudeContextWindow(modelSelection: ModelSelection): number | null { - return ( - resolveClaudeCatalogContextWindowTokens(BUNDLED_CLAUDE_MODEL_CATALOG, modelSelection) ?? null - ); -} - interface ClaudeCacheCreation { readonly ephemeral_1h_input_tokens: number; readonly ephemeral_5m_input_tokens: number; @@ -284,7 +279,7 @@ export function claudeProviderTurnTokenUsage( readonly cache_creation?: ClaudeCacheCreation | null; readonly output_tokens: number; }, - modelSelection: ModelSelection, + maxTokens: number | null, updatedAt: string, autoCompactWindow = "", requestedAt?: DateTime.Utc, @@ -292,7 +287,6 @@ export function claudeProviderTurnTokenUsage( const inputTokens = claudeMessageInputTokens(usage); const outputTokens = usage.output_tokens; const promptCacheTtlMs = claudePromptCacheTtlMs(usage.cache_creation); - const maxTokens = claudeContextWindow(modelSelection); // The cache lifetime starts with the request, not when its response ends. const promptCacheExpiresAt = promptCacheTtlMs === undefined || requestedAt === undefined @@ -2139,7 +2133,8 @@ const resolveClaudeForkUpToMessageId = Effect.fn("ClaudeAdapterV2.resolveForkUpT .slice(boundaryIndex + 1) .filter(isTerminalProviderTurn); if (terminalTurnsAfterBoundary.length === 0) { - return undefined; + // Rollback pins the head but leaves the rolled-back turns in Claude's transcript. + return yield* getNativeConversationHeadId(input.sourceProviderThread); } return yield* new ProviderAdapter.ProviderAdapterForkThreadError({ @@ -4541,6 +4536,7 @@ interface ActiveClaudeTurnContext { outputFinal: boolean; reasoningOutputTokens?: number; } | null; + compactionTokens: number | undefined; readonly toolCalls: Map; // The tool call that carries each model call's usage, by Anthropic message // id, for this turn's root and native subagent frames. A workflow member's @@ -5118,6 +5114,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( BUNDLED_CLAUDE_MODEL_CATALOG, adapterOptions.settings.customModels, ); + // Results report the window Claude Code compacts against; outside the catalog it is the only source. + const reportedContextWindows = new Map(); + const claudeContextWindow = (modelSelection: ModelSelection): number | null => + resolveClaudeCatalogContextWindowTokens(BUNDLED_CLAUDE_MODEL_CATALOG, modelSelection) ?? + reportedContextWindows.get(compileClaudeModelSelection(modelSelection).apiModelId) ?? + null; const continuationRequests = adapterOptions.continuationRequests ?? { offer: () => Effect.void, }; @@ -5354,6 +5356,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const queryContext = yield* Ref.make(null); // Wake buffers replay result frames, so ledger growth is measured once on arrival. const resultLedgerUsage = new WeakMap(); + const resultsReportingNewContextWindow = new WeakSet(); // A worktree handoff changes the project settings even within one session. let bypassCache: { cwd: string | null; availability: ClaudeBypassAvailability } | null = null; @@ -5652,21 +5655,36 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( // Usage without input is a placeholder, not a context measurement. const emitContextUsage = Effect.fnUntraced(function* ( context: ActiveClaudeTurnContext, - usage: ClaudeMessageUsage, + usage: ClaudeMessageUsage | number, requestStarted = false, ) { const now = yield* DateTime.now; // Only message_start marks the request start; an assistant snapshot // arrives after generation, so it leaves the cache expiry unknown. if (requestStarted && context.streamUsage !== null) context.streamUsage.requestedAt = now; - const tokenUsage = claudeProviderTurnTokenUsage( - usage, - context.input.modelSelection, - DateTime.formatIso(now), - adapterOptions.settings.autoCompactWindow, - context.streamUsage?.requestedAt, - ); - if (tokenUsage.inputTokens <= 0) return; + const maxTokens = claudeContextWindow(context.input.modelSelection); + const tokenUsage = + typeof usage === "number" + ? { + usedTokens: usage, + maxTokens, + ...claudeAutoCompactThreshold( + adapterOptions.settings.autoCompactWindow, + maxTokens, + ), + updatedAt: DateTime.formatIso(now), + } + : claudeProviderTurnTokenUsage( + usage, + maxTokens, + DateTime.formatIso(now), + adapterOptions.settings.autoCompactWindow, + context.streamUsage?.requestedAt, + ); + if (typeof usage !== "number") { + if (claudeMessageInputTokens(usage) <= 0) return; + context.compactionTokens = undefined; + } yield* emitProviderEvent({ type: "provider_turn.updated", driver: CLAUDE_PROVIDER, @@ -9629,38 +9647,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( typeof postTokens === "number" && Number.isFinite(postTokens) && postTokens > 0 ? Math.round(postTokens) : undefined; - if (afterTokenCount !== undefined) { - const maxTokens = claudeContextWindow(context.input.modelSelection); - yield* emitProviderEvent({ - type: "provider_turn.updated", - driver: CLAUDE_PROVIDER, - threadId: context.input.threadId, - providerTurn: { - id: context.providerTurnId, - providerThreadId: context.input.providerThread.id, - nodeId: context.input.rootNodeId, - runAttemptId: context.input.attemptId, - nativeTurnRef: { - driver: CLAUDE_PROVIDER, - nativeId: context.nativeTurnId, - strength: "strong", - }, - ordinal: context.providerTurnOrdinal, - status: "running", - startedAt: context.startedAt, - completedAt: null, - tokenUsage: { - usedTokens: afterTokenCount, - maxTokens, - ...claudeAutoCompactThreshold( - adapterOptions.settings.autoCompactWindow, - maxTokens, - ), - updatedAt: DateTime.formatIso(now), - }, - }, - }); - } + context.compactionTokens = afterTokenCount; + if (afterTokenCount !== undefined) yield* emitContextUsage(context, afterTokenCount); yield* emitProviderEvent({ type: "turn_item.updated", driver: CLAUDE_PROVIDER, @@ -10487,6 +10475,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } if (message.type === "result") { + const contextUsage = context.streamUsage?.usage ?? context.compactionTokens; + if (resultsReportingNewContextWindow.has(message) && contextUsage !== undefined) { + yield* emitContextUsage(context, contextUsage); + } const completedAt = yield* DateTime.now; const interrupted = (yield* Ref.get(interruptedTurns)).has(context.providerTurnId); const wasSteered = (yield* Ref.get(steeredTurns)).has(context.providerTurnId); @@ -11592,6 +11584,13 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( growth === undefined && context.modelLedger !== undefined ? undefined : ledger; + for (const [model, window] of claudeReportedContextWindows( + message.modelUsage, + )) { + if (reportedContextWindows.get(model) === window) continue; + reportedContextWindows.set(model, window); + resultsReportingNewContextWindow.add(message); + } } return handleSdkMessage({ query: querySession, message }); }), @@ -11765,6 +11764,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( blocks: new Map(), }, streamUsage: null, + compactionTokens: undefined, toolCalls: new Map(), modelCallCarriers: new Map(), compaction: null, @@ -12358,8 +12358,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( driver: CLAUDE_PROVIDER, providerSessionId: input.providerSessionId, providerSession: session, - getModelContextWindow: (selection) => - resolveClaudeCatalogContextWindowTokens(BUNDLED_CLAUDE_MODEL_CATALOG, selection), + getModelContextWindow: (selection) => claudeContextWindow(selection) ?? undefined, events: Stream.fromEffectRepeat(Queue.take(events)), backgroundTasksEndWithProcess: true, hasPendingBackgroundWork: Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeTaskOutputTail.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeTaskOutputTail.test.ts index 8da21b20236e2..b70b7322d5e63 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeTaskOutputTail.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeTaskOutputTail.test.ts @@ -20,7 +20,7 @@ const makeTaskFile = Effect.gen(function* () { return { root, tasks, file: path.join(tasks, `${TASK_ID}.output`) }; }); -// TestClock cannot interleave with real filesystem I/O; use the live clock. +// TestClock cannot interleave with real filesystem I/O; tests that wait on a poll use the live clock. describe("tailClaudeTaskOutput", () => { it.live("follows a file that appears after the task starts, across split characters", () => Effect.scoped( @@ -48,6 +48,32 @@ describe("tailClaudeTaskOutput", () => { ).pipe(Effect.provide(NodeServices.layer)), ); + it.effect("drops a split character when the file is truncated and rewritten", () => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const { root, tasks, file } = yield* makeTaskFile; + yield* fileSystem.makeDirectory(tasks, { recursive: true }); + // End the first read inside the two-byte é. + yield* fileSystem.writeFile(file, new TextEncoder().encode("café").subarray(0, 4)); + const chunks = yield* Queue.unbounded(); + const stop = yield* Deferred.make(); + yield* tailClaudeTaskOutput({ + root, + sessionId: SESSION_ID, + taskId: TASK_ID, + stop, + onChunk: (chunk) => Queue.offer(chunks, chunk), + }).pipe(Effect.forkScoped); + assert.equal(yield* Queue.take(chunks), "caf"); + + yield* fileSystem.writeFileString(file, "ok\n"); + yield* Deferred.succeed(stop, undefined); + assert.equal(yield* Queue.take(chunks), "ok\n"); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + it.live("skips ahead when the command writes faster than it is read", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeTaskOutputTail.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeTaskOutputTail.ts index 0bd369eb4df3a..77505d24a5bbf 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeTaskOutputTail.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeTaskOutputTail.ts @@ -88,7 +88,10 @@ export const tailClaudeTaskOutput = Effect.fn("ClaudeTaskOutputTail.tail")(funct Effect.map((info) => Number(info.size)), Effect.orElseSucceed(() => offset), ); - if (size < offset) offset = 0; + if (size < offset) { + offset = 0; + decoder = new TextDecoder("utf-8", { fatal: false }); + } if (size > offset) { const start = size - offset > MAX_READ_BYTES ? size - MAX_READ_BYTES : offset; const bytes = yield* Effect.scoped( diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 1f7ef6ad5a9ed..524d9e3563142 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -72,6 +72,7 @@ import * as ProviderReplayHarness from "../testkit/ProviderReplayHarness.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import type * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; import * as ModelManifest from "../../provider/ModelManifest.ts"; +import * as PortScanner from "../../preview/PortScanner.ts"; import { resolveProviderCompatibility } from "../../provider/providerCompatibility.ts"; import * as CodexAdapterV2 from "./CodexAdapterV2.ts"; import { withCodexReplayChildMetadata } from "./CodexAdapterV2.testkit.ts"; @@ -887,6 +888,66 @@ describe("CodexAdapterV2 process exit", () => { ), ), ); + + it.effect("counts the app-server as a running agent until its session closes", () => { + let runningAgents = 0; + const factoryLayer = CodexAdapterV2.layerAppServerClientFactory.pipe( + Layer.provide( + Layer.mock(PortScanner.PortDiscovery)({ + trackAgent: () => { + runningAgents += 1; + return () => { + runningAgents -= 1; + }; + }, + }), + ), + Layer.provide( + Layer.succeed(ProviderEventLoggers.ProviderEventLoggers, { + native: undefined, + canonical: undefined, + }), + ), + Layer.provide( + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(999_999_999), + exitCode: Effect.never, + isRunning: Effect.succeed(true), + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + stdin: Sink.drain, + stdout: Stream.never, + stderr: Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }), + ), + ), + ), + ), + ); + return Effect.gen(function* () { + const factory = yield* CodexAdapterV2.CodexAppServerClientFactory; + yield* Effect.scoped( + factory + .open({ + instanceId: CodexAdapterV2.CODEX_DEFAULT_INSTANCE_ID, + threadId: ThreadId.make("thread-codex-agent"), + providerSessionId: ProviderSessionId.make("session-codex-agent"), + runtimePolicy: CODEX_TEST_RUNTIME_POLICY, + settings: DEFAULT_CODEX_SETTINGS, + environment: {}, + }) + .pipe(Effect.tap(() => Effect.sync(() => assert.equal(runningAgents, 1)))), + ); + assert.equal(runningAgents, 0); + }).pipe(Effect.provide(factoryLayer)); + }); }); describe("CodexAdapterV2 session initialize", () => { @@ -3260,6 +3321,78 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); + it.effect("reports a thread another Codex client is writing to as in use", () => + Effect.scoped( + Effect.gen(function* () { + const nativeThreadId = "thread-open-elsewhere"; + const harness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ + scenario: "codex-resume-active-writer", + entries: [ + ...codexReplayPreamble({ + nativeThreadId, + nativeTurnId: "unused-turn", + prompt: "unused-prompt", + }).slice(0, 5), + { + type: "expect_outbound", + label: "thread/resume", + frame: { + id: 3, + method: "thread/resume", + params: { + threadId: nativeThreadId, + excludeTurns: true, + cwd: CODEX_TEST_RUNTIME_POLICY.cwd, + model: CODEX_TEST_MODEL_SELECTION.model, + config: CodexAdapterV2.CODEX_THREAD_CONFIG, + }, + }, + }, + { + type: "emit_inbound", + label: "active writer", + frame: { + id: 3, + error: { + code: -32600, + message: `thread ${nativeThreadId} already has an active writer`, + }, + }, + }, + ], + }), + ); + + const error = yield* Effect.flip( + harness.runtime.resumeThread({ + providerThread: harness.providerThread, + modelSelection: CODEX_TEST_MODEL_SELECTION, + runtimePolicy: CODEX_TEST_RUNTIME_POLICY, + }), + ); + + assert.equal(error._tag, "ProviderAdapterThreadInUseError"); + assert.nestedPropertyVal(error, "providerThreadId", harness.providerThread.id); + assert.include(error.message, "open in another Codex client"); + assert.nestedPropertyVal( + error, + "cause.errorMessage", + `thread ${nativeThreadId} already has an active writer`, + ); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + it.effect("resolves retryable app-server errors on resumed provider activity", () => Effect.scoped( Effect.gen(function* () { @@ -8928,6 +9061,84 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); + it.effect("unarchives an archived Codex source thread before forking it", () => + Effect.gen(function* () { + const nativeThreadId = "archived-fork-source"; + const forkThreadId = "archived-fork-target"; + const forkParams = { + threadId: nativeThreadId, + lastTurnId: "native-turn-first", + config: CodexAdapterV2.CODEX_THREAD_CONFIG, + }; + const exchange = ( + id: number, + method: string, + params: unknown, + reply: { readonly result: unknown } | { readonly error: unknown }, + ): Array => [ + { type: "expect_outbound", label: method, frame: { id, method, params } }, + { type: "emit_inbound", label: method, frame: { id, ...reply } }, + ]; + const transcript = makeCodexReplayTranscript({ + scenario: "codex-archived-fork", + entries: [ + ...codexReplayPreamble({ + nativeThreadId, + nativeTurnId: "archived-fork-turn", + prompt: "unused", + }).slice(0, 5), + ...exchange(3, "thread/fork", forkParams, { + error: { + code: -32600, + message: `session ${nativeThreadId} is archived. Run \`codex unarchive ${nativeThreadId}\` to unarchive it first.`, + }, + }), + ...exchange( + 4, + "thread/unarchive", + { threadId: nativeThreadId }, + { result: codexReplayThreadResult({ nativeThreadId, forkedFromId: null }) }, + ), + ...exchange(5, "thread/fork", forkParams, { + result: codexReplayThreadResult({ + nativeThreadId: forkThreadId, + forkedFromId: nativeThreadId, + }), + }), + ], + }); + const harness = yield* makeCodexReplayHarness(transcript); + const now = yield* DateTime.now; + const firstTurn = codexReplaySourceTurn({ + id: "provider-turn-first", + ordinal: 1, + nativeId: "native-turn-first", + providerThreadId: harness.providerThread.id, + now, + }); + + const forkedProviderThread = yield* harness.runtime.forkThread({ + sourceProviderThread: harness.providerThread, + sourceProviderTurns: [firstTurn], + providerTurnId: firstTurn.id, + targetThreadId: ThreadId.make("thread-archived-fork-target"), + }); + + assert.equal(forkedProviderThread.nativeThreadRef?.nativeId, forkThreadId); + assert.equal(forkedProviderThread.forkedFrom?.providerTurnId, firstTurn.id); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + describe("native goals", () => { const nativeThreadId = "goal-thread"; const objective = "Ship the feature"; diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 72270fc78019d..90cc63dcb1807 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -361,6 +361,21 @@ function toProtocolError( }); } +const CODEX_THREAD_IN_USE_MESSAGE = + "This conversation is open in another Codex client, such as the Codex app, the Codex CLI, or an IDE extension. Codex lets one client write to a conversation at a time. Quit that client, then send your message again."; + +const isCodexAppServerRequestError = Schema.is(CodexErrors.CodexAppServerRequestError); + +// Codex holds a per-thread writer lock for as long as a thread stays loaded in +// any app-server sharing the same CODEX_HOME, and reports a conflict this way. +function isCodexActiveWriterError(error: unknown): boolean { + return ( + isCodexAppServerRequestError(error) && + error.code === -32600 && + error.errorMessage.includes("already has an active writer") + ); +} + function codexTimestamp(seconds: number | null | undefined): DateTime.Utc { return seconds === null || seconds === undefined ? DateTime.nowUnsafe() @@ -1847,13 +1862,16 @@ function isSensitiveCodexProtocolKey(key: string): boolean { export const layerAppServerClientFactory: Layer.Layer< CodexAppServerClientFactory, never, - ChildProcessSpawner.ChildProcessSpawner | ProviderEventLoggers.ProviderEventLoggers + | ChildProcessSpawner.ChildProcessSpawner + | PortScanner.PortDiscovery + | ProviderEventLoggers.ProviderEventLoggers > = Layer.effect( CodexAppServerClientFactory, Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const { native: nativeEventLogger } = yield* ProviderEventLoggers.ProviderEventLoggers; const agentScope = yield* AgentScope; + const portDiscovery = yield* PortScanner.PortDiscovery; return CodexAppServerClientFactory.of({ open: Effect.fn("CodexAdapterV2.open")(function* (input) { @@ -1881,6 +1899,10 @@ export const layerAppServerClientFactory: Layer.Layer< }), ), ); + // Its shells carry each thread's T3CODE_THREAD_ID, so port discovery reads listeners' threads. + yield* Effect.acquireRelease(Effect.sync(portDiscovery.trackAgent), (release) => + Effect.sync(release), + ); const protocolLogger = makeCodexAppServerProtocolLogger({ nativeEventLogger, threadId: input.threadId, @@ -7572,14 +7594,20 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }, (effect, threadInput) => effect.pipe( - Effect.mapError( - (cause) => - new ProviderAdapter.ProviderAdapterResumeThreadError({ - driver: CODEX_PROVIDER, - providerSessionId: input.providerSessionId, - providerThreadId: threadInput.providerThread.id, - cause, - }), + Effect.mapError((cause) => + isCodexActiveWriterError(cause) + ? new ProviderAdapter.ProviderAdapterThreadInUseError({ + driver: CODEX_PROVIDER, + providerThreadId: threadInput.providerThread.id, + detail: CODEX_THREAD_IN_USE_MESSAGE, + cause, + }) + : new ProviderAdapter.ProviderAdapterResumeThreadError({ + driver: CODEX_PROVIDER, + providerSessionId: input.providerSessionId, + providerThreadId: threadInput.providerThread.id, + cause, + }), ), ), ), @@ -8360,24 +8388,32 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( function* (threadInput) { const threadId = yield* getNativeThreadId(threadInput.sourceProviderThread); const boundary = yield* resolveCodexForkBoundary(threadInput); + const fork = client.request("thread/fork", { + threadId, + ...(boundary.lastTurnId === undefined ? {} : { lastTurnId: boundary.lastTurnId }), + ...codexThreadRuntimeParams({ + threadId: threadInput.targetThreadId, + mcpSession: yield* mcpSessions.read(threadInput.targetThreadId), + ...(threadInput.modelSelection === undefined + ? {} + : { modelSelection: threadInput.modelSelection }), + ...(threadInput.runtimePolicy === undefined + ? {} + : { runtimePolicy: threadInput.runtimePolicy }), + }), + }); const response = yield* ensureInitialized.pipe( Effect.andThen( - client.request("thread/fork", { - threadId, - ...(boundary.lastTurnId === undefined - ? {} - : { lastTurnId: boundary.lastTurnId }), - ...codexThreadRuntimeParams({ - threadId: threadInput.targetThreadId, - mcpSession: yield* mcpSessions.read(threadInput.targetThreadId), - ...(threadInput.modelSelection === undefined - ? {} - : { modelSelection: threadInput.modelSelection }), - ...(threadInput.runtimePolicy === undefined - ? {} - : { runtimePolicy: threadInput.runtimePolicy }), + fork.pipe( + Effect.catchTags({ + CodexAppServerRequestError: (cause) => + /\bsession \S+ is archived\b|\bcodex unarchive\b/i.test(cause.errorMessage) + ? client.raw + .request("thread/unarchive", { threadId }) + .pipe(Effect.andThen(fork)) + : Effect.fail(cause), }), - }), + ), ), Effect.mapError( (cause) => diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts index c334af4fad113..f9819070fd885 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts @@ -2453,7 +2453,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { // session.command returns 204 without an inbox id; the stream drops before session.inbox.enqueued provides it. const { runtime, thread } = yield* resumed([ out("command.list", ""), - reply("command.list", { location: { directory: WORK }, data: [{ name: "hello" }] }), + reply("command.list", { location: { directory: WORK }, data: [{ name: "git/hello" }] }), out("message.list", { sessionID: SESSION, order: "desc", limit: "1" }), reply("message.list", { data: [ @@ -2461,7 +2461,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { ], cursor: {}, }), - out("session.command", { sessionID: SESSION, name: "hello", text: "WORLD" }), + out("session.command", { sessionID: SESSION, name: "git/hello", text: "WORLD" }), reply("session.command", null), { type: "runtime_exit", status: "success" } as const, out("event.subscribe"), @@ -2498,7 +2498,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { ); yield* runtime.startTurn({ ...turnInput(thread), - message: { ...turnInput(thread).message, text: "/hello WORLD" }, + message: { ...turnInput(thread).message, text: "/git/hello WORLD" }, }); const collected = yield* Fiber.join(events); const texts = collected.flatMap((event) => @@ -2759,6 +2759,56 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }).pipe(Effect.scoped), ); + it.effect("stores a shell command that exits non-zero as failed", () => + Effect.gen(function* () { + const tool = { sessionID: SESSION, assistantMessageID: "msg_assistant", id: "call-exit" }; + const { runtime, thread } = yield* resumed([ + out("session.prompt", { sessionID: SESSION, text: "" }), + promptAccepted, + event("session.tool.input.started", { ...tool, name: "shell" }), + event("session.tool.called", { ...tool, input: { command: "vp test" }, executed: false }), + { + type: "emit_inbound", + frame: { + type: "sdk.event", + event: { + id: "evt_toolsuccess_call-exit", + created: 1, + type: "session.tool.success", + data: { + ...tool, + content: [{ type: "text", text: "1 failed\n" }], + metadata: { status: "completed", truncated: false, exit: 1 }, + executed: false, + }, + durable: { aggregateID: SESSION, seq: 1, version: 2 }, + }, + }, + }, + event("session.execution.succeeded", { sessionID: SESSION }), + ]); + const events = yield* runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + Effect.forkScoped, + ); + yield* runtime.startTurn(turnInput(thread)); + const collected = yield* Fiber.join(events); + const item = collected + .flatMap((event) => (event.type === "turn_item.updated" ? [event.turnItem] : [])) + .findLast((item) => item.nativeItemRef?.nativeId?.endsWith(tool.id)); + assert.equal(item?.type, "command_execution"); + assert.equal(item?.status, "failed"); + assert.equal(item?.type === "command_execution" ? item.exitCode : null, 1); + assert.equal( + collected + .flatMap((event) => (event.type === "node.updated" ? [event.node] : [])) + .findLast((node) => node.id === item?.nodeId)?.status, + "failed", + ); + }).pipe(Effect.scoped), + ); + it.effect("re-registers T3 MCP each turn and drops its instructions when adding fails", () => Effect.gen(function* () { const mcpSessions = yield* McpProviderSessions.McpProviderSessions; diff --git a/apps/server/src/orchestration-v2/ContextHandoffBudget.test.ts b/apps/server/src/orchestration-v2/ContextHandoffBudget.test.ts index c4068702e3a57..71d9cf3dadaf9 100644 --- a/apps/server/src/orchestration-v2/ContextHandoffBudget.test.ts +++ b/apps/server/src/orchestration-v2/ContextHandoffBudget.test.ts @@ -137,6 +137,19 @@ describe("handoff budget", () => { assert.equal(command?.kind, "command_execution"); assert.include(command!.text, "Exit code: 1"); assert.include(command!.text, "界".repeat(300)); + assert.equal( + historicalMessage({ + ...base, + type: "file_change", + fileName: "src/a.ts", + changes: [ + { operation: "update", path: "src/a.ts" }, + { operation: "add", path: "src/b.ts" }, + { operation: "update", path: "src/new.ts", oldPath: "src/old.ts" }, + ], + })?.text, + "File change: src/a.ts, src/b.ts, src/new.ts", + ); assert.isNull( historicalMessage({ ...base, diff --git a/apps/server/src/orchestration-v2/ContextHandoffService.test.ts b/apps/server/src/orchestration-v2/ContextHandoffService.test.ts index 0282819d81b56..78958c2e317d5 100644 --- a/apps/server/src/orchestration-v2/ContextHandoffService.test.ts +++ b/apps/server/src/orchestration-v2/ContextHandoffService.test.ts @@ -210,6 +210,36 @@ it.effect("caps the side chat context block without summarizing discarded parent }); assert.isTrue(framingOnly.startsWith("You are in a side chat beside the main thread.")); assert.notInclude(framingOnly, "Main thread activity"); + + const multiFilePatch = ContextHandoffService.makeSideChatContext({ + parentThreadId, + digest: [ + { + id: TurnItemId.make("turn-item:patch"), + threadId: parentThreadId, + runId: null, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 60, + status: "completed", + title: null, + startedAt: null, + completedAt: null, + updatedAt: DateTime.makeUnsafe(0), + type: "file_change", + fileName: "src/a.ts", + changes: [ + { operation: "update", path: "src/a.ts" }, + { operation: "add", path: "src/b.ts" }, + { operation: "update", path: "src/new.ts", oldPath: "src/old.ts" }, + ], + }, + ], + }); + assert.include(multiFilePatch, "- File change: src/a.ts, src/b.ts, src/new.ts"); }).pipe(Effect.provide(layerTest)), ); diff --git a/apps/server/src/orchestration-v2/ContextHandoffService.ts b/apps/server/src/orchestration-v2/ContextHandoffService.ts index dcfffa018c8c0..10f2701446268 100644 --- a/apps/server/src/orchestration-v2/ContextHandoffService.ts +++ b/apps/server/src/orchestration-v2/ContextHandoffService.ts @@ -213,7 +213,7 @@ function summarizeDeltaItem(item: OrchestrationV2TurnItem): string | null { case "command_execution": return `- Command: ${compactText(item.input)}`; case "file_change": - return `- File change: ${item.fileName}`; + return `- File change: ${compactText(item.changes?.map((change) => change.path).join(", ") ?? item.fileName)}`; case "checkpoint": return `- Checkpoint: ${item.files.length} files`; case "handoff": diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index 540ddad789ada..55ef659e4749c 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -160,7 +160,7 @@ const seedParentWithTerminalTask = (input: { readonly runId: RunId; readonly rootNodeId: NodeId; readonly taskId: NodeId; - readonly deliveryState: "delivered" | "claimed" | "acknowledged" | "disposed"; + readonly deliveryState: "pending" | "delivered" | "claimed" | "acknowledged" | "disposed"; readonly completionWake?: "always" | "settled_only"; readonly deliveryTaskIds?: ReadonlyArray; readonly parentRunStatus?: "running" | "completed"; @@ -1051,6 +1051,7 @@ it.layer(layerTest)("delegated completion delivery repairs", (it) => { { command: "acknowledge", before: "running", after: "failed", capture: false }, { command: "acknowledge", before: "waiting", after: "completed", capture: true }, { command: "accept", before: "waiting", after: "completed", capture: true }, + { command: "wake-policy", before: "running", after: "completed", capture: true }, ] as const)( "keeps a $after run that settled between $command's read and commit", ({ command, before, after, capture }) => @@ -1064,14 +1065,20 @@ it.layer(layerTest)("delegated completion delivery repairs", (it) => { const taskId = NodeId.make(`node:${race}-task`); const checkpointId = CheckpointId.make(`checkpoint:${race}`); const messageId = MessageId.make(`message:delegated-delivery:${threadId}`); + const commandType = { + acknowledge: "delegated_task.completion-delivery.acknowledge", + accept: "notification.delivery.accept", + "wake-policy": "delegated_task.wake-policy", + }[command]; yield* seedParentWithTerminalTask({ threadId, projectId: ProjectId.make(`project:${race}`), runId, rootNodeId: NodeId.make(`node:${race}-root`), taskId, - deliveryState: "claimed", - deliveryTaskIds: [taskId], + ...(command === "wake-policy" + ? { deliveryState: "pending" } + : { deliveryState: "claimed", deliveryTaskIds: [taskId] }), now, }); const parent = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!; @@ -1112,18 +1119,18 @@ it.layer(layerTest)("delegated completion delivery repairs", (it) => { const { delegatedCompletion: _cohort, ...settledRun } = parent; const commit = sink.commitCommand; let settled = false; + let planned: OrchestrationV2Run["delegatedCompletion"]; yield* Effect.acquireUseRelease( Effect.sync(() => vi.spyOn(sink, "commitCommand").mockImplementation((input) => Effect.gen(function* () { - if ( - !settled && - input.commandType === - (command === "accept" - ? "notification.delivery.accept" - : "delegated_task.completion-delivery.acknowledge") - ) { + if (!settled && input.commandType === commandType) { settled = true; + const plannedRun = input.events.find((event) => event.type === "run.updated"); + planned = + plannedRun?.type === "run.updated" + ? plannedRun.payload.delegatedCompletion + : undefined; // The provider turn ends (or its checkpoint lands) after the command read the run. yield* commit({ commandId: CommandId.make(`command:${race}`), @@ -1161,13 +1168,21 @@ it.layer(layerTest)("delegated completion delivery repairs", (it) => { threadId, messageId, } - : { - type: "delegated_task.completion-delivery.acknowledge", - commandId: CommandId.make(`command:${race}:task-status`), - parentThreadId: threadId, - taskId, - observedByRunId: runId, - }, + : command === "wake-policy" + ? { + type: "delegated_task.wake-policy", + commandId: CommandId.make(`command:${race}:wake-policy`), + parentThreadId: threadId, + taskId, + completionWake: "always", + } + : { + type: "delegated_task.completion-delivery.acknowledge", + commandId: CommandId.make(`command:${race}:task-status`), + parentThreadId: threadId, + taskId, + observedByRunId: runId, + }, ), (spy) => Effect.sync(() => spy.mockRestore()), ); @@ -1175,12 +1190,22 @@ it.layer(layerTest)("delegated completion delivery repairs", (it) => { const finished = yield* orchestrator.getThreadProjection(threadId); assert.equal( finished.subagents.find((row) => row.id === taskId)?.completionDelivery?.state, - command === "accept" ? "delivered" : "acknowledged", + { acknowledge: "acknowledged", accept: "delivered", "wake-policy": "claimed" }[command], ); const run = finished.runs.find((candidate) => candidate.id === runId)!; assert.deepEqual( - { status: run.status, completedAt: run.completedAt, checkpointId: run.checkpointId }, - { status: after, completedAt: now, checkpointId: capture ? checkpointId : null }, + { + status: run.status, + completedAt: run.completedAt, + checkpointId: run.checkpointId, + delegatedCompletion: run.delegatedCompletion, + }, + { + status: after, + completedAt: now, + checkpointId: capture ? checkpointId : null, + delegatedCompletion: planned, + }, ); }), ); diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index c5c51646584aa..475e0a0e4ae7b 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -62,6 +62,7 @@ import * as EventSink from "./EventSink.ts"; import * as EventStore from "./EventStore.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProjectionMaintenance from "./ProjectionMaintenance.ts"; +import * as ProjectionReplay from "./testkit/ProjectionReplay.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProjectStore from "./ProjectStore.ts"; import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts"; @@ -85,6 +86,9 @@ const layerCommandReceiptStoreProvided = CommandReceiptStore.layer.pipe( ); const layerIsolatedOutbox = Layer.fresh(EffectOutbox.layer.pipe(Layer.provideMerge(layerDatabase))); const layerProjectionMaintenanceProvided = ProjectionMaintenance.layer.pipe( + Layer.provide(layerDatabase), +); +const layerProjectionReplayProvided = ProjectionReplay.layer.pipe( Layer.provide(layerStoresProvided), ); const layerTest = Layer.mergeAll( @@ -94,6 +98,7 @@ const layerTest = Layer.mergeAll( layerCommandReceiptStoreProvided, IdAllocator.layer, layerProjectionMaintenanceProvided, + layerProjectionReplayProvided, ); const providerInstanceId = ProviderInstanceId.make("codex"); @@ -148,85 +153,6 @@ function threadCreatedEvent(input: { }; } -it.effect("rebuilds event history one bounded page at a time", () => - Effect.gen(function* () { - const eventStore = yield* EventStore.EventStoreV2; - const projectionStore = yield* ProjectionStore.ProjectionStoreV2; - const now = yield* DateTime.now; - const threadId = ThreadId.make("thread:foundation-paged-rebuild"); - const thread = makeThread(threadId, now); - const eventCount = 1_005; - yield* eventStore.append({ - events: [ - threadCreatedEvent({ id: "event:paged-rebuild:0", thread, now }), - ...Array.from({ length: eventCount - 1 }, (_, index) => ({ - id: EventId.make(`event:paged-rebuild:${index + 1}`), - type: "thread.metadata-updated" as const, - threadId, - occurredAt: now, - payload: { ...thread, title: `Rebuilt update ${index + 1}` }, - })), - ], - }); - let applied = 0; - let failAfterFirstPage = false; - const appliedAtRead: Array = []; - const layerObservedStores = Layer.mergeAll( - Layer.succeed(EventStore.EventStoreV2, { - ...eventStore, - read: (input) => - Stream.suspend(() => { - appliedAtRead.push(applied); - if (failAfterFirstPage && applied >= 500) { - return Stream.fail( - new EventStore.EventStoreReadEventsError({ afterSequence: input?.afterSequence }), - ); - } - return eventStore.read(input); - }), - }), - Layer.succeed(ProjectionStore.ProjectionStoreV2, { - ...projectionStore, - apply: (event) => - projectionStore.apply(event).pipe( - Effect.tap(() => - Effect.sync(() => { - applied += 1; - }), - ), - ), - }), - ); - const rebuild = Effect.gen(function* () { - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; - return yield* maintenance.rebuild; - }).pipe( - Effect.provide( - Layer.fresh(ProjectionMaintenance.layer.pipe(Layer.provide(layerObservedStores))), - ), - ); - const rebuilt = yield* rebuild; - assert.isTrue(rebuilt.valid); - assert.equal(applied, eventCount); - assert.deepEqual(appliedAtRead, [0, 500, 1_000]); - assert.equal( - (yield* projectionStore.getThreadProjection(threadId)).thread.title, - "Rebuilt update 1004", - ); - applied = 0; - failAfterFirstPage = true; - const failed = yield* Effect.exit(rebuild); - assert.equal(failed._tag, "Failure"); - assert.equal(applied, 500); - assert.equal( - (yield* projectionStore.getThreadProjection(threadId)).thread.title, - "Rebuilt update 1004", - ); - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; - assert.isTrue((yield* maintenance.verify).valid); - }).pipe(Effect.provide(layerTest)), -); - const seedCompaction = Effect.fnUntraced(function* () { const sql = yield* SqlClient.SqlClient; yield* sql`INSERT INTO orchestration_v2_legacy_imports ( @@ -240,6 +166,13 @@ const seedCompaction = Effect.fnUntraced(function* () { ) SELECT 'event-' || n, 'thread', 'thread', n, CASE WHEN n IN (1, 51, 101) THEN 'message.updated' ELSE 'thread.visited' END, '2026-10-17', 'server', '{"id":"message"}', '{}', 2 FROM history`; + // A pass a day ago marked the whole log; projections cover it. + yield* sql`INSERT INTO orchestration_v2_projection_metadata ( + projection_name, schema_version, last_sequence, updated_at + ) VALUES + ('event-log-prune-mark', 2, 151, '1969-12-30T00:00:00.000Z'), + ('thread-projections', 2, 151, '1969-12-30T00:00:00.000Z') + ON CONFLICT(projection_name) DO UPDATE SET last_sequence = excluded.last_sequence`; yield* sql`WITH RECURSIVE history(n) AS ( SELECT 1 UNION ALL SELECT n + 1 FROM history WHERE n < 100 ) INSERT INTO orchestration_command_receipts ( @@ -284,11 +217,12 @@ it.effect("discovers on the reader worker and deletes on the writer", () => }), ); const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; - const result = yield* maintenance.compactEventStore(); - assert.equal(result.deletedEventCount, 148); + const result = yield* maintenance.pruneEventStore(); + assert.equal(result.deletedEventCount, 150); assert.equal(result.deletedReceiptCount, 100); + // Only the in-transaction check for rows appended during the pass reads on the writer. assert.deepEqual( - reads.filter((query) => /\bSELECT\b/.test(query)), + reads.filter((query) => /\bSELECT\b/.test(query) && !query.includes("AS prompt_sequence")), [], ); assert(writes.some((query) => query.includes("DELETE FROM orchestration_events"))); @@ -296,7 +230,6 @@ it.effect("discovers on the reader worker and deletes on the writer", () => }).pipe( Effect.provide( ProjectionMaintenance.layer.pipe( - Layer.provide(Layer.merge(EventStore.layer, ProjectionStore.layer)), Layer.provideMerge(SqlitePersistence.layerFromPath(filename)), ), ), @@ -318,7 +251,7 @@ it.effect("paces scheduled compaction while preserving retention state and fixed const tracer = Tracer.make({ span(options) { const span = new Tracer.NativeSpan(options); - if (span.name === "ProjectionMaintenance.compactEventStore") compactionSpans.push(span); + if (span.name === "ProjectionMaintenance.pruneEventStore") compactionSpans.push(span); const end = span.end.bind(span); span.end = (time, exit) => { end(time, exit); @@ -377,15 +310,13 @@ it.effect("paces scheduled compaction while preserving retention state and fixed }, }), ); - assert.deepEqual(pauses, Array(5).fill(270)); + assert.deepEqual(pauses, Array(8).fill(270)); assert.deepEqual( compactionSpans.map((span) => span.status._tag), ["Ended"], ); + // 151 stopped being the stream head when 152 committed mid-pass; 152 is past the bound. assert.deepEqual(yield* sql`SELECT sequence FROM orchestration_events ORDER BY sequence`, [ - { sequence: 1 }, - { sequence: 101 }, - { sequence: 151 }, { sequence: 152 }, ]); assert.deepEqual(yield* sql`SELECT command_id FROM orchestration_command_receipts`, [ @@ -768,7 +699,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; const projectionStore = yield* ProjectionStore.ProjectionStoreV2; - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const maintenance = yield* ProjectionReplay.ProjectionReplay; const now = yield* DateTime.now; const firstThreadId = ThreadId.make("thread:foundation-shared-session:first"); const secondThreadId = ThreadId.make("thread:foundation-shared-session:second"); @@ -829,7 +760,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { }), ); - it.effect("compacts superseded state events, imported v1 events, and legacy receipts", () => + it.effect("prunes covered history, imported v1 events, and legacy receipts", () => Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; @@ -983,9 +914,12 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { ON CONFLICT(command_id) DO NOTHING `; - const summary = yield* maintenance.compactEventStore(); + // The first pass only marks the head and drops imported legacy rows. + const marked = yield* maintenance.pruneEventStore(); + yield* TestClock.adjust("1 day"); + const summary = yield* maintenance.pruneEventStore(); assert.isAtLeast(summary.deletedEventCount, 504); - assert.isAtLeast(summary.deletedReceiptCount, 1); + assert.equal(marked.deletedReceiptCount + summary.deletedReceiptCount, 1); const remaining = yield* sql<{ readonly event_id: string }>` SELECT event_id @@ -996,18 +930,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { `; assert.deepEqual( remaining.map((row) => row.event_id), - [ - "event:foundation-compact:create", - "event:foundation-compact:meta", - "event:foundation-compact:message-1", - "event:foundation-compact:node-1", - "event:foundation-compact:item-1", - "event:foundation-compact:visit-2", - "event:foundation-compact:message-2", - "event:foundation-compact:node-2", - "event:foundation-compact:item-2", - "event:foundation-compact:tags-2", - ], + ["event:foundation-compact:create", "event:foundation-compact:tags-2"], ); const remainingV1 = yield* sql<{ readonly stream_id: string }>` @@ -1031,8 +954,6 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { ["command:foundation-compact:pending"], ); - assert.isTrue((yield* maintenance.verify).valid); - assert.isTrue((yield* maintenance.rebuild).valid); assert.deepEqual(yield* projections.getThreadProjection(threadId), beforeCompaction); }), ); @@ -1041,7 +962,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; const projectionStore = yield* ProjectionStore.ProjectionStoreV2; - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const maintenance = yield* ProjectionReplay.ProjectionReplay; const sql = yield* SqlClient.SqlClient; const now = yield* DateTime.now; const parentThreadId = ThreadId.make("thread:foundation-cross-thread:parent"); @@ -3152,7 +3073,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; const projectionStore = yield* ProjectionStore.ProjectionStoreV2; - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const maintenance = yield* ProjectionReplay.ProjectionReplay; const sql = yield* SqlClient.SqlClient; const now = yield* DateTime.now; const threadId = ThreadId.make("thread:foundation-many-items"); @@ -3384,10 +3305,7 @@ it.effect.each(["events", "receipts"] as const)( const path = yield* Path.Path; const directory = yield* fs.makeTempDirectoryScoped({ prefix: "compaction-reader-" }); const database = SqlitePersistence.layerFromPath(path.join(directory, "statev2.sqlite")); - const maintenanceLayer = ProjectionMaintenance.layer.pipe( - Layer.provide(Layer.merge(EventStore.layer, ProjectionStore.layer)), - Layer.provideMerge(database), - ); + const maintenanceLayer = ProjectionMaintenance.layer.pipe(Layer.provideMerge(database)); yield* Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; @@ -3434,7 +3352,7 @@ it.effect.each(["events", "receipts"] as const)( : Effect.succeed(statement); }; const compacted = yield* Deferred.make(); - const compaction = yield* maintenance.compactEventStore().pipe( + const compaction = yield* maintenance.pruneEventStore().pipe( Effect.provideService(Statement.CurrentTransformer, observeDiscovery), Effect.provideService(Statement.SpanPropagationEnabled, true), Effect.tap(() => Deferred.succeed(compacted, undefined)), diff --git a/apps/server/src/orchestration-v2/IdleCompaction.test.ts b/apps/server/src/orchestration-v2/IdleCompaction.test.ts index 3c13452e98e7f..a11a534b32614 100644 --- a/apps/server/src/orchestration-v2/IdleCompaction.test.ts +++ b/apps/server/src/orchestration-v2/IdleCompaction.test.ts @@ -163,6 +163,22 @@ describe("idleCompactionWindow", () => { const compacted = records(cache); compacted.messages = [{ ...compacted.messages[0]!, text: "/compact" }]; expect(idleCompactionWindow(compacted, NOW_MS)).toBeNull(); + const steered = records(cache); + steered.runs = [ + ...steered.runs, + { + ...steered.runs[0]!, + id: RunId.make("run-2"), + ordinal: 2, + userMessageId: MessageId.make("message-2"), + status: "cancelled", + startedAt: null, + }, + ]; + expect(idleCompactionWindow(steered, NOW_MS)?.runId).toBe(runId); + // A turn Claude itself reported as cancelled did run, so it stays the latest. + steered.runs = steered.runs.map((run) => ({ ...run, startedAt: at(-MINUTE_MS) })); + expect(idleCompactionWindow(steered, NOW_MS)).toBeNull(); const archived = records(cache); archived.thread = { ...archived.thread, archivedAt: at() }; expect(idleCompactionWindow(archived, NOW_MS)).toBeNull(); @@ -323,9 +339,17 @@ describe("IdleCompactionService", () => { ordinal: 2, userMessageId: laterMessageId, }; + const cancelledRun = { + ...earlierRun, + id: RunId.make("run-3"), + ordinal: 3, + userMessageId: MessageId.make("message-3"), + status: "cancelled" as const, + startedAt: null, + }; const data: OrchestrationV2ThreadProjection = { ...base, - runs: [earlierRun, laterRun], + runs: [earlierRun, laterRun, cancelledRun], messages: [ ...base.messages, { ...base.messages[0]!, id: laterMessageId, runId: laterRunId, text: "Keep going" }, diff --git a/apps/server/src/orchestration-v2/IdleCompaction.ts b/apps/server/src/orchestration-v2/IdleCompaction.ts index a89894045c298..c4d586817551f 100644 --- a/apps/server/src/orchestration-v2/IdleCompaction.ts +++ b/apps/server/src/orchestration-v2/IdleCompaction.ts @@ -51,10 +51,13 @@ export function idleCompactionWindow( if (thread.snoozedUntil != null && DateTime.toEpochMillis(thread.snoozedUntil) > nowMs) return null; if (!isListedThreadLineage(thread.lineage)) return null; - const latestRun = runs.reduce<(typeof runs)[number] | null>( - (latest, run) => (latest === null || run.ordinal > latest.ordinal ? run : latest), - null, - ); + // A removed or steered queued message leaves a cancelled run that never started. + const latestRun = runs + .filter((run) => run.status !== "cancelled" || run.startedAt !== null) + .reduce<(typeof runs)[number] | null>( + (latest, run) => (latest === null || run.ordinal > latest.ordinal ? run : latest), + null, + ); if (latestRun?.status !== "completed" || latestRun.completedAt === null) return null; const message = messages.find((candidate) => candidate.id === latestRun.userMessageId); // Never compact a compaction, so one idle spell compacts at most once. diff --git a/apps/server/src/orchestration-v2/IdleCompactionService.ts b/apps/server/src/orchestration-v2/IdleCompactionService.ts index 3b3b788b183a5..c7035de4ec584 100644 --- a/apps/server/src/orchestration-v2/IdleCompactionService.ts +++ b/apps/server/src/orchestration-v2/IdleCompactionService.ts @@ -88,10 +88,12 @@ export const make = Effect.gen(function* () { ) ) return; - const latestRun = records.runs.reduce<(typeof records.runs)[number] | undefined>( - (latest, run) => (latest === undefined || run.ordinal > latest.ordinal ? run : latest), - undefined, - ); + const latestRun = records.runs + .filter((run) => run.status !== "cancelled" || run.startedAt !== null) + .reduce<(typeof records.runs)[number] | undefined>( + (latest, run) => (latest === undefined || run.ordinal > latest.ordinal ? run : latest), + undefined, + ); // A late update for an earlier run replaced the latest run's timer, so take it over. if (latestRun?.status === "completed" && latestRun.id !== runId) { runId = latestRun.id; diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index e6e83831a7556..186cde505c6bc 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -444,6 +444,12 @@ export interface OrchestratorV2Shape { }, OrchestratorV2Error >; + readonly getPullRequestSnapshot: ( + options: Parameters[0], + ) => Effect.Effect< + Effect.Success>, + OrchestratorV2Error + >; readonly getShellSnapshot: ( options?: ShellSnapshotOptions, ) => Effect.Effect; @@ -1193,6 +1199,35 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio return withId; }); + // Detaching kills background work running inside the provider process. + // Probe even with an empty roster: Claude's native subagents never reach + // pendingBackgroundTasks. Sessions shared across threads probe per thread. + const sessionHasBackgroundWork = Effect.fn("Orchestrator.sessionHasBackgroundWork")(function* ( + detaching: OrchestrationV2ThreadProjection["providerSessions"][number], + providerThreads: OrchestrationV2ThreadProjection["providerThreads"], + ) { + const session = yield* providerSessions + .get(detaching.id) + .pipe(Effect.orElseSucceed(() => Option.none())); + if (Option.isNone(session)) return false; + const runtime = session.value; + const probes = + !detaching.capabilities.sessions.supportsMultipleProviderThreadsPerSession && + runtime.hasPendingBackgroundWork !== undefined + ? [runtime.hasPendingBackgroundWork] + : providerThreads.flatMap((providerThread) => + providerThread.providerSessionId === detaching.id && + runtime.hasPendingBackgroundWorkForThread !== undefined + ? [runtime.hasPendingBackgroundWorkForThread(providerThread)] + : [], + ); + for (const probe of probes) { + // A failed probe cannot confirm that the work ended. + if (yield* probe.pipe(Effect.catchCause(() => Effect.succeed(true)))) return true; + } + return false; + }); + // Runs in every command transaction; avoid reading thousands of finished leaf nodes. const readCommandProjection = (threadId: ThreadId) => projectionStore @@ -1853,6 +1888,42 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ), ) : null; + const releasableSessions = projection.providerSessions.filter( + (session) => + switchPlan?.releaseProviderSessionIds.includes(session.id) && + session.status !== "stopped" && + session.status !== "error", + ); + const busySessionIds = new Set(); + for (const session of releasableSessions) { + if (yield* sessionHasBackgroundWork(session, projection.providerThreads)) { + busySessionIds.add(session.id); + } + } + // Nobody waits on a queued start to refuse it. A handoff runs in its own session, so a + // busy one stays up; a restart resumes the busy session's native thread, so wait instead. + if (busySessionIds.size > 0 && switchPlan?.transition.type === "restart_and_resume") { + yield* Effect.logInfo( + "Holding the queue: the replaced provider session still runs background work", + { threadId, runId: queuedRun.id, providerSessionIds: Array.from(busySessionIds) }, + ); + yield* writeSystemEvents( + projection.runs + .filter((run) => run.status === "queued") + .map((run) => ({ + type: "run.updated" as const, + threadId, + runId: run.id, + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { ...run, queueHeld: true }, + })), + ); + return; + } + const sessionsToDetach = releasableSessions.filter( + (session) => !busySessionIds.has(session.id), + ); const canResumeAcrossInstances = switchPlan?.instanceChanged === true && switchPlan.transition.type === "restart_and_resume" && @@ -2208,12 +2279,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }, ] : []; - const sessionsToDetach = projection.providerSessions.filter( - (session) => - switchPlan?.releaseProviderSessionIds.includes(session.id) && - session.status !== "stopped" && - session.status !== "error", - ); yield* writeSystemEvents( [ ...(selectionChanged @@ -3781,6 +3846,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio settledOverride: thread.settledOverride === "settled" ? "active" : thread.settledOverride, settledAt: thread.settledOverride === "settled" ? null : thread.settledAt, + ...(thread.settledOverride === "settled" ? { unsettledAt: now } : {}), snoozedUntil: null, snoozedAt: null, lastSnoozeWakeAt: thread.snoozedUntil == null ? thread.lastSnoozeWakeAt : now, @@ -6725,40 +6791,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio session.status !== "stopped" && session.status !== "error", ); - // Detaching kills background work running inside the provider process. - // Probe even with an empty roster: Claude's native subagents never reach - // pendingBackgroundTasks. Sessions shared across threads probe per thread. for (const detaching of sessionsToDetach) { - const session = yield* providerSessions - .get(detaching.id) - .pipe(Effect.orElseSucceed(() => Option.none())); - if (Option.isNone(session)) { - continue; - } - const runtime = session.value; - const probes = - !detaching.capabilities.sessions.supportsMultipleProviderThreadsPerSession && - runtime.hasPendingBackgroundWork !== undefined - ? [runtime.hasPendingBackgroundWork] - : projection.providerThreads.flatMap((providerThread) => - providerThread.providerSessionId === detaching.id && - runtime.hasPendingBackgroundWorkForThread !== undefined - ? [runtime.hasPendingBackgroundWorkForThread(providerThread)] - : [], - ); - for (const probe of probes) { - const stillRunning = yield* probe.pipe( - // A failed probe cannot confirm that the work ended. - Effect.catchCause(() => Effect.succeed(true)), - ); - if (stillRunning) { - return yield* new OrchestratorDispatchError({ - commandId: command.commandId, - commandType: command.type, - cause: - "The previous provider is still running background agents or commands, and switching would end them. Wait for them to finish, or press Stop, then send the message again.", - }); - } + if (yield* sessionHasBackgroundWork(detaching, projection.providerThreads)) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: + "The previous provider is still running background agents or commands, and switching would end them. Wait for them to finish, or press Stop, then send the message again.", + }); } } @@ -7745,7 +7785,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ...(command.context ? { context: command.context } : {}), attachments: command.attachments, ...(run.contextHandoffId === null - ? modelSwitchSince(projection, modelSelection, sourceRun?.modelSelection) + ? modelSwitchSince( + projection, + modelSelection, + // Side chats do not show the parent's runs. + sideChatFork ? undefined : sourceRun?.modelSelection, + ) : {}), }; const activeHandoff = portableForkHandoff ?? mergeBackHandoff ?? providerSwitchHandoff; @@ -9088,6 +9133,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio payload: completionPlan.message, }); } + return completionPlan.parentRun?.id; }); const dispatchCreatedThreadRecord = Effect.fn("orchestrationV2.dispatch.createdThreadRecord")( @@ -14101,7 +14147,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio yield* dispatchRemoteDelegatedTaskStop(command, events); break; case "delegated_task.wake-policy": - yield* dispatchDelegatedTaskWakePolicy(command, events); + updateDelegatedCompletionOnlyRunId = yield* dispatchDelegatedTaskWakePolicy( + command, + events, + ); break; case "delegated_task.completion-delivery.acknowledge": case "delegated_task.completion-delivery.dispose": @@ -14828,6 +14877,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))); }, ), + getPullRequestSnapshot: Effect.fn("OrchestratorV2.getPullRequestSnapshot")(function (options) { + return projectionStore + .getPullRequestSnapshot(options) + .pipe(Effect.mapError(shellProjectionError)); + }), getShellSnapshot: Effect.fn("OrchestratorV2.getShellSnapshot")(function (options) { return projectionStore.getShellSnapshot(options).pipe(Effect.mapError(shellProjectionError)); }), diff --git a/apps/server/src/orchestration-v2/ProjectionMaintenance.test.ts b/apps/server/src/orchestration-v2/ProjectionMaintenance.test.ts new file mode 100644 index 0000000000000..1f42953cc4ec3 --- /dev/null +++ b/apps/server/src/orchestration-v2/ProjectionMaintenance.test.ts @@ -0,0 +1,385 @@ +import { assert, it } from "@effect/vitest"; +import { + EventId, + NodeId, + PlanId, + type OrchestrationV2AppThread, + type OrchestrationV2DomainEvent, + type OrchestrationV2TurnItem, + ProjectId, + ProviderDriverKind, + ProviderInstanceId, + ThreadId, + TurnItemId, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as Statement from "effect/sql/Statement"; + +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as EventSink from "./EventSink.ts"; +import * as EventStore from "./EventStore.ts"; +import * as ProjectionMaintenance from "./ProjectionMaintenance.ts"; +import * as ProjectionStore from "./ProjectionStore.ts"; +import * as OrchestrationEventStore from "../persistence/OrchestrationEventStore.ts"; + +const database = SqlitePersistence.layerMemory; +const eventStoreProvided = EventStore.layer.pipe(Layer.provideMerge(database)); +const projectionStoreProvided = ProjectionStore.layer.pipe(Layer.provideMerge(database)); +const stores = Layer.mergeAll(database, eventStoreProvided, projectionStoreProvided); +const layerTest = Layer.mergeAll( + stores, + OrchestrationEventStore.layer.pipe(Layer.provide(database)), + EventSink.layer.pipe(Layer.provide(stores)), + ProjectionMaintenance.layer.pipe(Layer.provide(stores)), +); + +const providerInstanceId = ProviderInstanceId.make("codex"); +const providerDriver = ProviderDriverKind.make("codex"); +const modelSelection = { instanceId: providerInstanceId, model: "gpt-5.4" }; + +function makeThread(threadId: ThreadId, now: DateTime.Utc): OrchestrationV2AppThread { + return { + id: threadId, + projectId: ProjectId.make(`project:${threadId}`), + title: "Turn item compaction", + providerInstanceId, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + activeProviderThreadId: null, + lineage: { rootThreadId: threadId, parentThreadId: null, relationshipToParent: null }, + forkedFrom: null, + createdBy: "user", + creationSource: "web", + createdAt: now, + updatedAt: now, + archivedAt: null, + settledOverride: null, + settledAt: null, + lastVisitedAt: null, + deletedAt: null, + }; +} + +function threadCreated(thread: OrchestrationV2AppThread, now: DateTime.Utc) { + return { + id: EventId.make("event:turn-item-compaction:thread"), + type: "thread.created", + threadId: thread.id, + providerInstanceId, + occurredAt: now, + payload: thread, + } satisfies OrchestrationV2DomainEvent; +} + +const itemSpecs = [ + { key: "command", type: "command_execution", ordinal: 10 }, + { key: "reasoning", type: "reasoning", ordinal: 20 }, + { key: "todo", type: "todo_list", ordinal: 30 }, + { key: "commanded", type: "command_execution", ordinal: 40 }, +] as const; + +function itemEvent( + threadId: ThreadId, + now: DateTime.Utc, + spec: (typeof itemSpecs)[number], + revision: number, + eventId = `event:turn-item-compaction:${spec.key}:${revision}`, +): OrchestrationV2DomainEvent { + const id = TurnItemId.make(`turn-item:compaction:${spec.key}`); + const growing = `${spec.key}:${revision}:` + "x".repeat((revision + 1) * 192); + const completed = revision === 69; + const base = { + id, + threadId, + runId: null, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: spec.ordinal, + status: completed ? ("completed" as const) : ("running" as const), + title: spec.key, + startedAt: now, + completedAt: completed ? now : null, + updatedAt: now, + }; + let payload: OrchestrationV2TurnItem; + switch (spec.type) { + case "command_execution": + payload = { + ...base, + type: "command_execution", + input: `run ${spec.key}`, + output: growing, + }; + break; + case "reasoning": + payload = { ...base, type: "reasoning", text: growing, streaming: !completed }; + break; + case "todo_list": + payload = { + ...base, + type: "todo_list", + planId: PlanId.make("plan:turn-item-compaction"), + steps: [{ id: "step", text: growing, status: completed ? "completed" : "running" }], + explanation: growing, + }; + break; + } + return { + id: EventId.make(eventId), + type: "turn-item.updated", + threadId, + occurredAt: now, + payload, + }; +} + +function subagentEvents( + threadId: ThreadId, + now: DateTime.Utc, + revision: number, + eventSuffix = String(revision), +): ReadonlyArray { + const id = NodeId.make("subagent:turn-item-compaction"); + const prompt = revision === 0 ? "Review prompt A. ".repeat(40) : "Review prompt B. ".repeat(40); + const progress = `Review progress ${revision}`; + return [ + { + id: EventId.make(`event:turn-item-compaction:subagent:${eventSuffix}`), + type: "subagent.updated", + threadId, + occurredAt: now, + payload: { + id, + threadId, + runId: null, + parentNodeId: NodeId.make("node:turn-item-compaction-parent"), + origin: "provider_native", + createdBy: "agent", + driver: providerDriver, + providerInstanceId, + providerThreadId: null, + childThreadId: null, + nativeTaskRef: { + driver: providerDriver, + nativeId: "compaction-review", + strength: "strong", + }, + prompt, + progress, + title: "Review", + model: null, + status: "running", + result: null, + startedAt: now, + completedAt: null, + updatedAt: now, + }, + }, + { + id: EventId.make(`event:turn-item-compaction:subagent-item:${eventSuffix}`), + type: "turn-item.updated", + threadId, + occurredAt: now, + payload: { + id: TurnItemId.make("turn-item:compaction:subagent"), + threadId, + runId: null, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 50, + status: "running", + title: "Review", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "subagent", + subagentId: id, + origin: "provider_native", + driver: providerDriver, + providerInstanceId, + childThreadId: null, + prompt, + progress, + result: null, + }, + }, + ] satisfies ReadonlyArray; +} + +const insertRunningRunUpdate = (threadId: ThreadId) => + Effect.flatMap( + SqlClient.SqlClient, + (sql) => sql<{ readonly sequence: number }>` + INSERT INTO orchestration_events ( + event_id, aggregate_kind, stream_id, stream_version, event_type, + occurred_at, actor_kind, payload_json, metadata_json, application_event_version + ) VALUES ( + 'event:prune:run-running', 'thread', ${threadId}, + (SELECT MAX(stream_version) + 1 FROM orchestration_events WHERE stream_id = ${threadId}), + 'run.updated', '2026-11-01T00:00:00.000Z', 'server', '{"status":"running"}', '{}', 2 + ) RETURNING sequence + `, + ); + +const remainingEventIds = (threadId: ThreadId) => + Effect.flatMap( + SqlClient.SqlClient, + (sql) => sql<{ readonly event_id: string }>` + SELECT event_id FROM orchestration_events WHERE stream_id = ${threadId} ORDER BY sequence + `, + ).pipe(Effect.map((rows) => rows.map((row) => row.event_id))); + +it.effect("prunes events the projections cover once the retained day has passed", () => + Effect.gen(function* () { + const sink = yield* EventSink.EventSinkV2; + const eventStore = yield* EventStore.EventStoreV2; + const events = yield* OrchestrationEventStore.OrchestrationEventStore; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:prune"); + yield* sink.write({ events: [threadCreated(makeThread(threadId, now), now)] }); + for (const revision of [0, 1, 2]) { + yield* sink.write({ events: [itemEvent(threadId, now, itemSpecs[0], revision)] }); + } + yield* sink.write({ events: subagentEvents(threadId, now, 0) }); + yield* insertRunningRunUpdate(threadId); + yield* sink.write({ events: [itemEvent(threadId, now, itemSpecs[1], 0)] }); + + // The first pass only marks the head; nothing is a day old yet. + assert.equal((yield* maintenance.pruneEventStore()).deletedEventCount, 0); + const markedThrough = yield* eventStore.latestSequence(); + + // Repeating prompt A borrows it from the anchors written before the mark. + yield* sink.write({ events: subagentEvents(threadId, now, 0, "repeat") }); + yield* sink.write({ events: [itemEvent(threadId, now, itemSpecs[0], 3)] }); + const highWater = yield* eventStore.latestSequence(); + const projection = yield* projections.getThreadProjection(threadId); + + yield* TestClock.adjust("23 hours"); + assert.equal((yield* maintenance.pruneEventStore()).deletedEventCount, 0); + yield* TestClock.adjust("1 hour"); + assert.equal((yield* maintenance.pruneEventStore()).deletedEventCount, 4); + + assert.deepEqual(yield* remainingEventIds(threadId), [ + "event:turn-item-compaction:thread", + "event:turn-item-compaction:subagent:0", + "event:turn-item-compaction:subagent-item:0", + "event:prune:run-running", + "event:turn-item-compaction:subagent:repeat", + "event:turn-item-compaction:subagent-item:repeat", + "event:turn-item-compaction:command:3", + ]); + assert.equal(yield* events.prunedThroughSequence, markedThrough); + assert.equal(yield* eventStore.latestSequence(), highWater); + assert.equal(yield* eventStore.latestSequence({ threadId }), highWater); + assert.deepEqual(yield* projections.getThreadProjection(threadId), projection); + const repeated = yield* eventStore + .read({ threadId, eventType: "subagent.updated", afterSequence: markedThrough }) + .pipe(Stream.runCollect); + assert.deepEqual( + Array.from(repeated, (stored) => + stored.event.type === "subagent.updated" ? stored.event.payload.prompt : null, + ), + ["Review prompt A. ".repeat(40)], + ); + }).pipe(Effect.provide(layerTest)), +); + +it.effect("keeps the anchor a retained stream head borrows from", () => + Effect.gen(function* () { + const sink = yield* EventSink.EventSinkV2; + const eventStore = yield* EventStore.EventStoreV2; + const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:prune-head"); + yield* sink.write({ events: [threadCreated(makeThread(threadId, now), now)] }); + yield* sink.write({ events: subagentEvents(threadId, now, 0) }); + // The stream's newest row borrows prompt A, so it is kept below the boundary. + yield* sink.write({ events: subagentEvents(threadId, now, 0, "repeat") }); + yield* maintenance.pruneEventStore(); + yield* TestClock.adjust("1 day"); + assert.isAbove((yield* maintenance.pruneEventStore()).deletedEventCount, 0); + + assert.include( + yield* remainingEventIds(threadId), + "event:turn-item-compaction:subagent-item:0", + ); + const stored = yield* eventStore.read({ threadId }).pipe(Stream.runCollect); + const head = Array.from(stored).at(-1)?.event; + assert.deepEqual( + head?.type === "turn-item.updated" && head.payload.type === "subagent" + ? head.payload.prompt + : null, + "Review prompt A. ".repeat(40), + ); + }).pipe(Effect.provide(layerTest)), +); + +it.effect("keeps an anchor that a row appended during the pass borrows from", () => + Effect.scoped( + Effect.gen(function* () { + const sink = yield* EventSink.EventSinkV2; + const eventStore = yield* EventStore.EventStoreV2; + const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:prune-race"); + yield* sink.write({ events: [threadCreated(makeThread(threadId, now), now)] }); + yield* sink.write({ events: subagentEvents(threadId, now, 0) }); + yield* maintenance.pruneEventStore(); + yield* sink.write({ events: [itemEvent(threadId, now, itemSpecs[0], 0)] }); + yield* TestClock.adjust("1 day"); + + const scanStarted = yield* Deferred.make(); + const releaseScan = yield* Deferred.make(); + let blocked = false; + const blockFirstScanPage: Statement.Transformer = (statement) => { + const [query] = statement.compile(); + if (!blocked && query.includes("END AS prunable")) { + blocked = true; + return Deferred.succeed(scanStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseScan)), + Effect.as(statement), + ); + } + return Effect.succeed(statement); + }; + const pass = yield* maintenance + .pruneEventStore() + .pipe( + Effect.provideService(Statement.CurrentTransformer, blockFirstScanPage), + Effect.forkScoped, + ); + yield* Deferred.await(scanStarted); + yield* sink.write({ events: subagentEvents(threadId, now, 0, "late") }); + yield* Deferred.succeed(releaseScan, undefined); + assert.equal((yield* Fiber.join(pass)).deletedEventCount, 0); + + const late = yield* eventStore + .read({ threadId, eventType: "subagent.updated" }) + .pipe(Stream.runCollect); + assert.deepEqual( + Array.from(late, (stored) => + stored.event.type === "subagent.updated" ? stored.event.payload.prompt : null, + ), + ["Review prompt A. ".repeat(40), "Review prompt A. ".repeat(40)], + ); + }), + ).pipe(Effect.provide(layerTest)), +); diff --git a/apps/server/src/orchestration-v2/ProjectionMaintenance.ts b/apps/server/src/orchestration-v2/ProjectionMaintenance.ts index 9c76b3c8a3e02..210322aed686f 100644 --- a/apps/server/src/orchestration-v2/ProjectionMaintenance.ts +++ b/apps/server/src/orchestration-v2/ProjectionMaintenance.ts @@ -1,26 +1,18 @@ -import { ThreadId } from "@t3tools/contracts"; import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; -import * as Stream from "effect/Stream"; import * as SqlClient from "effect/sql/SqlClient"; -import * as EventStore from "./EventStore.ts"; +import { + EVENT_LOG_PRUNE_MARK, + EVENT_LOG_PRUNED_THROUGH, +} from "../persistence/OrchestrationEventStore.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; -export interface ProjectionVerificationV2 { - readonly valid: boolean; - readonly schemaVersion: number; - readonly expectedSequence: number; - readonly projectionSequence: number; - readonly unreadableThreadIds: ReadonlyArray; - readonly missingThreadIds: ReadonlyArray; - readonly unexpectedThreadIds: ReadonlyArray; -} - export class ProjectionMaintenanceError extends Schema.TaggedError()( "ProjectionMaintenanceError", { @@ -30,9 +22,8 @@ export class ProjectionMaintenanceError extends Schema.TaggedError; - readonly rebuild: Effect.Effect; - readonly compactEventStore: (betweenPages?: Effect.Effect) => Effect.Effect< + /** Deletes events the projections already cover once they are older than the retained history. */ + readonly pruneEventStore: (betweenPages?: Effect.Effect) => Effect.Effect< { readonly deletedEventCount: number; readonly deletedReceiptCount: number; @@ -47,371 +38,282 @@ export class ProjectionMaintenanceV2 extends Context.Service< ProjectionMaintenanceV2Shape >()("t3/orchestration-v2/ProjectionMaintenance/ProjectionMaintenanceV2") {} -type ProjectionMetadataRow = { - readonly schema_version: number; +// Reconnect replay and command-id retries read this tail; older cursors resync from a snapshot. +const RETAINED_EVENT_HISTORY = Duration.days(1); +// Small pages bound work on the reader worker and synchronous fallback. +const PAGE_SIZE = 50; +const REFERENCE_SCAN_WINDOW = 1000; + +type MetadataRow = { + readonly projection_name: string; readonly last_sequence: number; + readonly updated_at: string; }; -const encodeEntityKey = Schema.encodeSync( - Schema.fromJsonString(Schema.Tuple([Schema.String, Schema.String, Schema.NullOr(Schema.String)])), -); - -export const layer: Layer.Layer< - ProjectionMaintenanceV2, - never, - EventStore.EventStoreV2 | ProjectionStore.ProjectionStoreV2 | SqlClient.SqlClient -> = Layer.effect( +export const layer: Layer.Layer = Layer.effect( ProjectionMaintenanceV2, Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; - const eventStore = yield* EventStore.EventStoreV2; - const projectionStore = yield* ProjectionStore.ProjectionStoreV2; - - const verify = Effect.gen(function* () { - const expectedThreadRows = yield* sql<{ readonly thread_id: string }>` - SELECT DISTINCT stream_id AS thread_id - FROM orchestration_events INDEXED BY orchestration_events_v2_created_threads_idx - WHERE application_event_version = 2 - AND aggregate_kind = 'thread' - AND event_type = 'thread.created' - ORDER BY stream_id ASC - `; - const projectionRows = yield* sql<{ readonly thread_id: string }>` - SELECT thread_id - FROM orchestration_v2_projection_threads - ORDER BY thread_id ASC - `; - const actualIds = projectionRows.map((row) => ThreadId.make(row.thread_id)); - const expectedIds = expectedThreadRows.map((row) => ThreadId.make(row.thread_id)); - const actualSet = new Set(actualIds); - const expectedSet = new Set(expectedIds); - const missingThreadIds = expectedIds.filter((threadId) => !actualSet.has(threadId)); - const unexpectedThreadIds = actualIds.filter((threadId) => !expectedSet.has(threadId)); - const unreadableThreadIds = yield* projectionStore.getUnreadableThreadIds(); - const metadata = yield* sql` - SELECT schema_version, last_sequence - FROM orchestration_v2_projection_metadata - WHERE projection_name = 'thread-projections' - LIMIT 1 - `; - const expectedSequence = yield* eventStore.latestSequence(); - const schemaVersion = metadata[0]?.schema_version ?? 0; - const projectionSequence = metadata[0]?.last_sequence ?? 0; - return { - valid: - schemaVersion === ProjectionStore.ORCHESTRATION_V2_PROJECTION_SCHEMA_VERSION && - projectionSequence === expectedSequence && - missingThreadIds.length === 0 && - unexpectedThreadIds.length === 0 && - unreadableThreadIds.length === 0, - schemaVersion, - expectedSequence, - projectionSequence, - unreadableThreadIds, - missingThreadIds, - unexpectedThreadIds, - } satisfies ProjectionVerificationV2; - }).pipe(sql.withTransaction); - const rebuild = Effect.gen(function* () { - yield* sql.withTransaction( - Effect.gen(function* () { - const throughSequence = yield* eventStore.latestSequence(); - yield* sql`DELETE FROM orchestration_v2_projection_context_transfers`; - yield* sql`DELETE FROM orchestration_v2_projection_context_handoffs`; - yield* sql`DELETE FROM orchestration_v2_projection_checkpoints`; - yield* sql`DELETE FROM orchestration_v2_projection_checkpoint_scopes`; - yield* sql`DELETE FROM orchestration_v2_projection_turn_items`; - yield* sql`DELETE FROM orchestration_v2_projection_plans`; - yield* sql`DELETE FROM orchestration_v2_projection_messages`; - yield* sql`DELETE FROM orchestration_v2_projection_runtime_requests`; - yield* sql`DELETE FROM orchestration_v2_projection_provider_turns`; - yield* sql`DELETE FROM orchestration_v2_projection_provider_threads`; - yield* sql`DELETE FROM orchestration_v2_projection_provider_session_bindings`; - yield* sql`DELETE FROM orchestration_v2_projection_provider_sessions`; - yield* sql`DELETE FROM orchestration_v2_projection_subagents`; - yield* sql`DELETE FROM orchestration_v2_projection_nodes`; - yield* sql`DELETE FROM orchestration_v2_projection_run_attempts`; - yield* sql`DELETE FROM orchestration_v2_projection_runs`; - yield* sql`DELETE FROM orchestration_v2_projection_threads`; - yield* sql`DELETE FROM orchestration_v2_turn_item_positions`; + const writeMetadata = (name: string, sequence: number, at: DateTime.Utc) => sql` + INSERT INTO orchestration_v2_projection_metadata ( + projection_name, schema_version, last_sequence, updated_at + ) + VALUES ( + ${name}, + ${ProjectionStore.ORCHESTRATION_V2_PROJECTION_SCHEMA_VERSION}, + ${sequence}, + ${DateTime.formatIso(at)} + ) + ON CONFLICT(projection_name) DO UPDATE SET + last_sequence = excluded.last_sequence, + updated_at = excluded.updated_at + `; - const pageSize = 500; - let lastSequence = 0; - while (true) { - const page = yield* eventStore - .read({ afterSequence: lastSequence, throughSequence, limit: pageSize }) - .pipe(Stream.runCollect); - for (const stored of page) { - yield* projectionStore.apply(stored.event); - if (stored.event.type === "turn-item.updated") { - yield* sql` - INSERT INTO orchestration_v2_turn_item_positions ( - thread_id, - turn_item_id, - ordinal - ) - VALUES ( - ${stored.event.threadId}, - ${stored.event.payload.id}, - ${stored.event.payload.ordinal} - ) - ON CONFLICT(thread_id, turn_item_id) DO UPDATE SET - ordinal = excluded.ordinal - `; - } - lastSequence = stored.sequence; - } - if (page.length < pageSize) break; - yield* Effect.yieldNow; - } - const now = DateTime.formatIso(yield* DateTime.now); - yield* sql` - INSERT INTO orchestration_v2_projection_metadata ( - projection_name, - schema_version, - last_sequence, - updated_at - ) - VALUES ( - 'thread-projections', - ${ProjectionStore.ORCHESTRATION_V2_PROJECTION_SCHEMA_VERSION}, - ${lastSequence}, - ${now} - ) - ON CONFLICT(projection_name) DO UPDATE SET - schema_version = excluded.schema_version, - last_sequence = excluded.last_sequence, - updated_at = excluded.updated_at - `; - }), - ); - return yield* verify; + // Collects anchors whose prompt later rows borrow through promptSequence. + const readPromptReferences = Effect.fnUntraced(function* ( + afterSequence: number, + throughSequence: number, + betweenPages: Effect.Effect, + ) { + const referenced = new Set(); + for (let cursor = afterSequence; cursor < throughSequence;) { + const windowEnd = Math.min(cursor + REFERENCE_SCAN_WINDOW, throughSequence); + const rows = yield* sql<{ readonly prompt_sequence: number }>` + SELECT json_extract(payload_json, '$.promptSequence') AS prompt_sequence + FROM orchestration_events + WHERE sequence > ${cursor} + AND sequence <= ${windowEnd} + AND event_type IN ('subagent.updated', 'turn-item.updated') + AND json_type(payload_json, '$.promptSequence') = 'integer' + `.pipe(NodeSqliteClient.readOnly); + for (const row of rows) referenced.add(row.prompt_sequence); + cursor = windowEnd; + yield* betweenPages; + } + return referenced; }); - const mapError = - (operation: string) => - (effect: Effect.Effect) => - effect.pipe( - Effect.mapError((cause) => new ProjectionMaintenanceError({ operation, cause })), - ); - - // Keep thread.created as the verification and replay anchor; clients recognize deletion by thread.deleted event type. - const SUPERSEDABLE_THREAD_EVENT_TYPES = [ - "thread.archived", - "thread.unarchived", - "thread.settled", - "thread.unsettled", - "thread.snoozed", - "thread.unsnoozed", - "thread.pinned", - "thread.auto-settle-set", - "thread.hidden-set", - "thread.group-set", - "thread.unpinned", - "thread.pin-reordered", - "thread.active-reordered", - "thread.metadata-updated", - "thread.pull-request-synced", - "thread.runtime-mode-updated", - "thread.interaction-mode-updated", - "thread.model-selection-updated", - "thread.provider-switched", - "thread.guest-share-updated", - ]; - // Read-state events preserve updatedAt and supersede only each other. Activity events supersede both groups. - const READ_STATE_THREAD_EVENT_TYPES = ["thread.visited", "thread.marked-unread"]; - // Older clients skip tag events, so these supersede only each other. - const TAG_THREAD_EVENT_TYPE = "thread.tags-updated"; - - const supersedableThreadEventTypes = new Set(SUPERSEDABLE_THREAD_EVENT_TYPES); - const readStateThreadEventTypes = new Set(READ_STATE_THREAD_EVENT_TYPES); - // Subagent snapshots stay intact because later rows can reference an earlier prompt. - // Commanded turn items stay for command-result replay. - // Small pages bound work on the reader worker and synchronous fallback. - const COMPACTION_PAGE_SIZE = 50; - - // Keep first/latest keyed events for replay order; page every event to bound sparse scans. - const compactEventStore = Effect.fn("ProjectionMaintenance.compactEventStore")(function* ( - betweenPages: Effect.Effect = Effect.void, + const pruneEvents = Effect.fnUntraced(function* ( + throughSequence: number, + head: number, + betweenPages: Effect.Effect, ) { - const bounds = yield* sql<{ - readonly event_sequence: number; - readonly receipt_row_id: number; - }>` - SELECT - COALESCE((SELECT MAX(sequence) FROM orchestration_events), 0) AS event_sequence, - COALESCE((SELECT MAX(rowid) FROM orchestration_command_receipts), 0) AS receipt_row_id + const referenced = yield* readPromptReferences(throughSequence, head, betweenPages); + // Stream heads at or below the boundary are kept, so the anchors they borrow must stay too. + const keptHeadReferences = yield* sql<{ readonly prompt_sequence: number }>` + SELECT json_extract(event.payload_json, '$.promptSequence') AS prompt_sequence + FROM ( + SELECT MAX(sequence) AS sequence FROM orchestration_events + GROUP BY aggregate_kind, stream_id + ) AS heads + JOIN orchestration_events AS event ON event.sequence = heads.sequence + WHERE heads.sequence <= ${throughSequence} + AND event.event_type IN ('subagent.updated', 'turn-item.updated') + AND json_type(event.payload_json, '$.promptSequence') = 'integer' `.pipe(NodeSqliteClient.readOnly); - let throughSequence = bounds[0]?.event_sequence ?? 0; - let throughReceiptRowId = bounds[0]?.receipt_row_id ?? 0; - const retainedThreadIds = new Set(); - const retainedReadStateThreadIds = new Set(); - const retainedTagThreadIds = new Set(); - // Retain each key's oldest candidate, deleting it only when an older event appears. - const oldestCandidateByEntityKey = new Map(); - let deletedEventCount = 0; - let deletedReceiptCount = 0; - - while (throughSequence > 0) { - const rows = yield* sql<{ - readonly sequence: number; - readonly application_event_version: number; - readonly aggregate_kind: string; - readonly stream_id: string; - readonly event_type: string; - readonly entity_id: string | null; - readonly imported_legacy_thread: number; - }>` + for (const row of keptHeadReferences) referenced.add(row.prompt_sequence); + yield* betweenPages; + let referencesThrough = head; + let deleted = 0; + let cursor = 0; + while (cursor < throughSequence) { + const rows = yield* sql<{ readonly sequence: number; readonly prunable: number }>` SELECT event.sequence, - event.application_event_version, - event.aggregate_kind, - event.stream_id, - event.event_type, - CASE - WHEN event.application_event_version = 2 - AND event.event_type IN ('message.updated', 'node.updated') - THEN json_extract(event.payload_json, '$.id') - WHEN event.application_event_version = 2 - AND event.event_type = 'turn-item.updated' - AND event.command_id IS NULL - AND json_extract(event.payload_json, '$.type') <> 'subagent' - THEN json_extract(event.payload_json, '$.id') - WHEN event.application_event_version = 2 - AND event.event_type = 'thread.pull-request-link-updated' - THEN json_array(json_extract(event.payload_json, '$.host'), - json_extract(event.payload_json, '$.repository'), - json_extract(event.payload_json, '$.number')) - ELSE NULL - END AS entity_id, CASE - WHEN event.application_event_version = 1 AND event.aggregate_kind = 'thread' - THEN EXISTS ( - SELECT 1 FROM orchestration_v2_legacy_imports AS legacy_import - WHERE legacy_import.thread_id = event.stream_id - AND legacy_import.transcript_imported_at IS NOT NULL - ) - ELSE 0 - END AS imported_legacy_thread + WHEN event.application_event_version = 1 THEN 0 + -- The legacy importer treats a thread without thread.created as not yet imported. + WHEN event.event_type = 'thread.created' THEN 0 + -- Codex usage history reads the selection a run attempt started with. + WHEN event.event_type = 'run.updated' + AND json_extract(event.payload_json, '$.status') = 'running' THEN 0 + -- The newest row keeps each stream's high water, stream_version, and snapshot sequence. + WHEN event.sequence = ( + SELECT MAX(newer.sequence) FROM orchestration_events AS newer + WHERE newer.aggregate_kind = event.aggregate_kind + AND newer.stream_id = event.stream_id + ) THEN 0 + ELSE 1 + END AS prunable FROM orchestration_events AS event - WHERE event.sequence <= ${throughSequence} - ORDER BY event.sequence DESC - LIMIT ${COMPACTION_PAGE_SIZE + 1} + WHERE event.sequence > ${cursor} + AND event.sequence <= ${throughSequence} + ORDER BY event.sequence ASC + LIMIT ${PAGE_SIZE} `.pipe(NodeSqliteClient.readOnly); - const page = rows.slice(0, COMPACTION_PAGE_SIZE); - const obsolete: number[] = []; - for (const row of page) { - if (row.imported_legacy_thread === 1) { - obsolete.push(row.sequence); - } else if (row.application_event_version === 2) { - if ( - row.aggregate_kind === "thread" && - supersedableThreadEventTypes.has(row.event_type) - ) { - if (retainedThreadIds.has(row.stream_id)) obsolete.push(row.sequence); - else retainedThreadIds.add(row.stream_id); - } else if ( - row.aggregate_kind === "thread" && - readStateThreadEventTypes.has(row.event_type) - ) { - if ( - retainedThreadIds.has(row.stream_id) || - retainedReadStateThreadIds.has(row.stream_id) - ) { - obsolete.push(row.sequence); - } else retainedReadStateThreadIds.add(row.stream_id); - } else if ( - row.aggregate_kind === "thread" && - row.event_type === TAG_THREAD_EVENT_TYPE - ) { - if (retainedThreadIds.has(row.stream_id) || retainedTagThreadIds.has(row.stream_id)) { - obsolete.push(row.sequence); - } else retainedTagThreadIds.add(row.stream_id); - } else if ( - row.event_type === "thread.pull-request-link-updated" && - retainedThreadIds.has(row.stream_id) - ) { - // A newer whole-thread payload already carries this link's state. - obsolete.push(row.sequence); - } else if ( - row.event_type === "thread.pull-request-link-updated" || - row.event_type === "message.updated" || - row.event_type === "node.updated" || - (row.event_type === "turn-item.updated" && row.entity_id !== null) - ) { - const key = encodeEntityKey([row.event_type, row.stream_id, row.entity_id]); - const candidate = oldestCandidateByEntityKey.get(key); - if (candidate === undefined) oldestCandidateByEntityKey.set(key, null); - else { - if (candidate !== null) obsolete.push(candidate); - oldestCandidateByEntityKey.set(key, row.sequence); + const candidates = rows.filter((row) => row.prunable === 1).map((row) => row.sequence); + // Never hold a transaction or the reader lock across a yield or a rest. + yield* Effect.yieldNow; + if (candidates.length > 0) { + deleted += yield* sql.withTransaction( + Effect.gen(function* () { + // Rows appended since the reference scan may borrow a prompt from a candidate. + const appended = yield* sql<{ + readonly sequence: number; + readonly prompt_sequence: number | null; + }>` + SELECT + sequence, + CASE + WHEN event_type IN ('subagent.updated', 'turn-item.updated') + AND json_type(payload_json, '$.promptSequence') = 'integer' + THEN json_extract(payload_json, '$.promptSequence') + END AS prompt_sequence + FROM orchestration_events + WHERE sequence > ${referencesThrough} + `; + for (const row of appended) { + if (row.prompt_sequence !== null) referenced.add(row.prompt_sequence); + referencesThrough = Math.max(referencesThrough, row.sequence); } - } - } + const obsolete = candidates.filter((sequence) => !referenced.has(sequence)); + if (obsolete.length === 0) return 0; + yield* sql`DELETE FROM orchestration_events WHERE sequence IN ${sql.in(obsolete)}`; + return obsolete.length; + }), + ); + yield* Effect.yieldNow; } - // Never hold a transaction or the reader lock across a yield or a rest. + // Rest after every page, including the last, so the next loop starts a fresh slice. + yield* betweenPages; + if (rows.length < PAGE_SIZE) break; + cursor = rows.at(-1)?.sequence ?? throughSequence; + } + return deleted; + }); + + // Imported legacy threads are served from v2 rows; their v1 events and receipts have no reader. + const pruneImportedLegacyHistory = Effect.fnUntraced(function* ( + throughSequence: number, + throughReceiptRowId: number, + betweenPages: Effect.Effect, + ) { + let deletedEventCount = 0; + let deletedReceiptCount = 0; + for (let cursor = 0; ;) { + const rows = yield* sql<{ readonly sequence: number; readonly imported: number }>` + SELECT + event.sequence, + event.aggregate_kind = 'thread' AND EXISTS ( + SELECT 1 FROM orchestration_v2_legacy_imports AS legacy_import + WHERE legacy_import.thread_id = event.stream_id + AND legacy_import.transcript_imported_at IS NOT NULL + ) AS imported + FROM orchestration_events AS event + INDEXED BY idx_orchestration_events_application_sequence + WHERE event.application_event_version = 1 + AND event.sequence > ${cursor} + AND event.sequence <= ${throughSequence} + ORDER BY event.sequence ASC + LIMIT ${PAGE_SIZE} + `.pipe(NodeSqliteClient.readOnly); + const obsolete = rows.filter((row) => row.imported === 1).map((row) => row.sequence); yield* Effect.yieldNow; if (obsolete.length > 0) { yield* sql`DELETE FROM orchestration_events WHERE sequence IN ${sql.in(obsolete)}`; deletedEventCount += obsolete.length; yield* Effect.yieldNow; } - throughSequence = - rows.length <= COMPACTION_PAGE_SIZE ? 0 : (page.at(-1)?.sequence ?? 1) - 1; - if (throughSequence > 0 || throughReceiptRowId > 0) yield* betweenPages; + yield* betweenPages; + if (rows.length < PAGE_SIZE) break; + cursor = rows.at(-1)?.sequence ?? cursor; } - - while (throughReceiptRowId > 0) { + for (let cursor = 0; ;) { const rows = yield* sql<{ readonly row_id: number; readonly command_id: string; - readonly imported_legacy_thread: number; + readonly imported: number; }>` SELECT receipt.rowid AS row_id, receipt.command_id, - CASE - WHEN receipt.command_type = 'legacy' AND receipt.aggregate_kind = 'thread' - THEN EXISTS ( - SELECT 1 FROM orchestration_v2_legacy_imports AS legacy_import - WHERE legacy_import.thread_id = receipt.aggregate_id - AND legacy_import.transcript_imported_at IS NOT NULL - ) - ELSE 0 - END AS imported_legacy_thread + receipt.command_type = 'legacy' AND receipt.aggregate_kind = 'thread' AND EXISTS ( + SELECT 1 FROM orchestration_v2_legacy_imports AS legacy_import + WHERE legacy_import.thread_id = receipt.aggregate_id + AND legacy_import.transcript_imported_at IS NOT NULL + ) AS imported FROM orchestration_command_receipts AS receipt - WHERE receipt.rowid <= ${throughReceiptRowId} - ORDER BY receipt.rowid DESC - LIMIT ${COMPACTION_PAGE_SIZE + 1} + WHERE receipt.rowid > ${cursor} AND receipt.rowid <= ${throughReceiptRowId} + ORDER BY receipt.rowid ASC + LIMIT ${PAGE_SIZE} `.pipe(NodeSqliteClient.readOnly); - const page = rows.slice(0, COMPACTION_PAGE_SIZE); - const obsolete = page - .filter((row) => row.imported_legacy_thread === 1) - .map((row) => row.command_id); + const obsolete = rows.filter((row) => row.imported === 1).map((row) => row.command_id); yield* Effect.yieldNow; if (obsolete.length > 0) { yield* sql`DELETE FROM orchestration_command_receipts WHERE command_id IN ${sql.in(obsolete)}`; deletedReceiptCount += obsolete.length; yield* Effect.yieldNow; } - throughReceiptRowId = (page.at(-1)?.row_id ?? 1) - 1; - if (rows.length <= COMPACTION_PAGE_SIZE) break; yield* betweenPages; + if (rows.length < PAGE_SIZE) break; + cursor = rows.at(-1)?.row_id ?? cursor; } + return { deletedEventCount, deletedReceiptCount }; + }); - const freelistRows = yield* sql<{ readonly freelist_count: number }>`PRAGMA freelist_count`; - const pageSizeRows = yield* sql<{ readonly page_size: number }>`PRAGMA page_size`; - const reclaimableBytes = - (freelistRows[0]?.freelist_count ?? 0) * (pageSizeRows[0]?.page_size ?? 0); + // A pass prunes through the mark the previous pass left once that mark is a day old, so + // at least RETAINED_EVENT_HISTORY of events always remains. Receipts stay for idempotency. + const pruneEventStore = Effect.fn("ProjectionMaintenance.pruneEventStore")( + function* (betweenPages: Effect.Effect = Effect.void) { + const now = yield* DateTime.now; + const metadata = yield* sql` + SELECT projection_name, last_sequence, updated_at + FROM orchestration_v2_projection_metadata + WHERE projection_name IN ( + 'thread-projections', ${EVENT_LOG_PRUNE_MARK}, ${EVENT_LOG_PRUNED_THROUGH} + ) + `.pipe(NodeSqliteClient.readOnly); + const read = (name: string) => metadata.find((row) => row.projection_name === name); + const projected = read("thread-projections")?.last_sequence ?? 0; + const mark = read(EVENT_LOG_PRUNE_MARK); + const prunedThrough = read(EVENT_LOG_PRUNED_THROUGH)?.last_sequence ?? 0; + // Fixed bounds: foreground commits during the pass wait for the next one. + const bounds = yield* sql<{ readonly sequence: number; readonly receipt_row_id: number }>` + SELECT + COALESCE((SELECT MAX(sequence) FROM orchestration_events), 0) AS sequence, + COALESCE((SELECT MAX(rowid) FROM orchestration_command_receipts), 0) AS receipt_row_id + `.pipe(NodeSqliteClient.readOnly); + const head = bounds[0]?.sequence ?? 0; - return { deletedEventCount, deletedReceiptCount, reclaimableBytes }; - }, mapError("compact event store")); + let deletedEventCount = 0; + const markIsDue = + mark !== undefined && + DateTime.toEpochMillis(now) - Date.parse(mark.updated_at) >= + Duration.toMillis(RETAINED_EVENT_HISTORY); + if (mark !== undefined && markIsDue) { + const throughSequence = Math.max(Math.min(mark.last_sequence, projected), prunedThrough); + if (throughSequence > prunedThrough) { + // Publish the boundary first so resuming clients take a snapshot, never a partial replay. + yield* writeMetadata(EVENT_LOG_PRUNED_THROUGH, throughSequence, now); + } + deletedEventCount += yield* pruneEvents(throughSequence, head, betweenPages); + } + if (mark === undefined || markIsDue) yield* writeMetadata(EVENT_LOG_PRUNE_MARK, head, now); - return ProjectionMaintenanceV2.of({ - verify: mapError("verify")(verify), - rebuild: mapError("rebuild")(rebuild), - compactEventStore, - }); + const legacy = yield* pruneImportedLegacyHistory( + head, + bounds[0]?.receipt_row_id ?? 0, + betweenPages, + ); + deletedEventCount += legacy.deletedEventCount; + + const freelistRows = yield* sql<{ readonly freelist_count: number }>`PRAGMA freelist_count`; + const pageSizeRows = yield* sql<{ readonly page_size: number }>`PRAGMA page_size`; + const reclaimableBytes = + (freelistRows[0]?.freelist_count ?? 0) * (pageSizeRows[0]?.page_size ?? 0); + + return { + deletedEventCount, + deletedReceiptCount: legacy.deletedReceiptCount, + reclaimableBytes, + }; + }, + Effect.mapError( + (cause) => new ProjectionMaintenanceError({ operation: "prune event store", cause }), + ), + ); + + return ProjectionMaintenanceV2.of({ pruneEventStore }); }), ); diff --git a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts index f1169f1e910b4..a4914fdee4509 100644 --- a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts @@ -24,6 +24,7 @@ import * as Layer from "effect/Layer"; import * as SqlClient from "effect/sql/SqlClient"; import * as Statement from "effect/sql/Statement"; +import delegatedRecoveryIndexes from "../persistence/Migrations/079_DelegatedRecoveryIndexes.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as ServerSettings from "../serverSettings.ts"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; @@ -357,8 +358,28 @@ it.effect("selects unfinished recovery work without reading settled thread histo new Set(yield* projections.getRecoveryThreadIds("runtime")), new Set([queued, archived, blocked, background, outboxOnly, requestOnly]), ); - assert.deepEqual(yield* projections.getRecoveryThreadIds("delegated-completions"), [delivery]); - assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), []); + for (const [kind, expected, index] of [ + ["delegated-completions", [delivery], "runs_delegated_delivery_idx"], + ["subagent-results", [], "threads_subagent_result_idx"], + ] as const) { + const statements: Array["compile"]>> = []; + const observe: Statement.Transformer = (statement) => { + statements.push(statement.compile()); + return Effect.succeed(statement); + }; + assert.deepEqual( + yield* projections + .getRecoveryThreadIds(kind) + .pipe(Effect.provideService(Statement.CurrentTransformer, observe)), + expected, + ); + const [query, params] = statements[0]!; + const plan = yield* sql.unsafe<{ detail: string }>(`EXPLAIN QUERY PLAN ${query}`, params); + assert.isTrue( + plan.some((row) => row.detail.includes(index)), + JSON.stringify(plan), + ); + } const recoveryState = yield* projections.getRuntimeRecoveryProjection(queued); assert.deepEqual( recoveryState.runs.map((run) => run.id), @@ -371,6 +392,84 @@ it.effect("selects unfinished recovery work without reading settled thread histo }).pipe(Effect.provide(layerTest)), ); +it.effect("maintains delegated candidate indexes across legacy rows and payload transitions", () => + Effect.gen(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const sql = yield* SqlClient.SqlClient; + const parent = yield* createThread("index-parent"); + const child = yield* createThread("index-child"); + const run = yield* createRun(child, "completed"); + const rows = yield* sql<{ payload_json: string }>` + SELECT payload_json FROM orchestration_v2_projection_threads WHERE thread_id = ${child} + `; + const original = JSON.parse(rows[0]!.payload_json); + for (const payload of [ + "{broken", + JSON.stringify({ + ...original, + lineage: { parentThreadId: parent, relationshipToParent: "subagent" }, + forkedFrom: { type: "provider_thread" }, + }), + JSON.stringify({ + ...original, + lineage: { parentThreadId: parent, relationshipToParent: "subagent" }, + forkedFrom: { type: "node", nodeId: "node:index" }, + }), + JSON.stringify({ + ...original, + lineage: { parentThreadId: child, relationshipToParent: "subagent" }, + forkedFrom: { type: "node", nodeId: "node:index" }, + }), + JSON.stringify(original), + ]) { + yield* sql`UPDATE orchestration_v2_projection_threads SET payload_json = ${payload} + WHERE thread_id = ${child}`; + const expected = payload.includes('"type":"node"') ? [child] : []; + assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), expected); + } + for (const entry of [ + ["{broken", false], + ["{}", false], + ['{"delegatedCompletion":{"delivery":null}}', false], + ['{"delegatedCompletion":{"delivery":[]}}', false], + ['{"delegatedCompletion":{"delivery":{}}}', true], + ["{}", false], + ] as const) { + const [payload, included] = entry; + yield* sql`UPDATE orchestration_v2_projection_runs SET payload_json = ${payload} + WHERE run_id = ${run}`; + assert.deepEqual( + yield* projections.getRecoveryThreadIds("delegated-completions"), + included ? [child] : [], + ); + } + yield* sql`UPDATE orchestration_v2_projection_threads SET payload_json = '{broken' + WHERE thread_id = ${parent}`; + yield* sql`UPDATE orchestration_v2_projection_runs SET payload_json = '{broken' + WHERE run_id = ${run}`; + const existing = yield* createThread("index-existing", { + lineage: { parentThreadId: child, relationshipToParent: "subagent", rootThreadId: parent }, + forkedFrom: { type: "node", nodeId: NodeId.make("node:index-existing") }, + }); + yield* createRun(existing, "completed", { + delegatedCompletion: { + disposition: "open", + nextGeneration: 2, + delivery: { + generation: 1, + messageId: MessageId.make("message:index-existing"), + taskIds: [], + }, + }, + }); + yield* sql`DROP INDEX orchestration_v2_projection_threads_subagent_result_idx`; + yield* sql`DROP INDEX orchestration_v2_projection_runs_delegated_delivery_idx`; + yield* delegatedRecoveryIndexes; + assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), [existing]); + assert.deepEqual(yield* projections.getRecoveryThreadIds("delegated-completions"), [existing]); + }).pipe(Effect.provide(layerTest)), +); + it.effect("recovers terminal subagent results until their cross-thread transfer exists", () => Effect.gen(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; @@ -658,6 +757,106 @@ it.effect("includes shared sessions and provider-owned background rosters in rec }).pipe(Effect.provide(layerTest)), ); +it.effect("bounds streaming recovery reads by candidate runs and preserves message ordering", () => + Effect.gen(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const sql = yield* SqlClient.SqlClient; + const now = yield* DateTime.now; + const threadId = yield* createThread("streaming"); + const other = yield* createThread("streaming-other"); + const empty = yield* createThread("streaming-empty"); + const statements: Array["compile"]>> = []; + const observe: Statement.Transformer = (statement) => { + const compiled = statement.compile(); + if (compiled[0].includes("SELECT message.payload_json")) statements.push(compiled); + return Effect.succeed(statement); + }; + const expected: MessageId[] = []; + const statuses = [ + "queued", + "preparing", + "starting", + "running", + "waiting", + "completed", + "interrupted", + "rolled_back", + "failed", + "cancelled", + ] as const; + for (const [index, status] of statuses.entries()) { + const runId = yield* createRun(threadId, status, { ordinal: index + 1 }); + for (const suffix of ["b", "a", "settled", "wrong-thread"]) { + const id = MessageId.make(`message:streaming:${index}:${suffix}`); + const streaming = suffix !== "settled"; + const messageThread = suffix === "wrong-thread" ? other : threadId; + const createdAt = DateTime.add(now, { seconds: -index }); + yield* projections.apply({ + id: EventId.make(`event:${id}`), + type: "message.updated", + threadId: messageThread, + runId, + occurredAt: now, + payload: { + id, + threadId: messageThread, + runId, + nodeId: null, + role: "assistant", + text: id, + attachments: [], + streaming, + createdAt, + updatedAt: now, + createdBy: "agent", + creationSource: "provider", + }, + }); + } + if (index < 5) + expected.unshift( + MessageId.make(`message:streaming:${index}:a`), + MessageId.make(`message:streaming:${index}:b`), + ); + } + const foreignRun = yield* createRun(other, "running"); + yield* sql` + INSERT INTO orchestration_v2_projection_messages + (message_id, thread_id, run_id, role, streaming, created_at, updated_at, payload_json) + VALUES ('foreign-run', ${threadId}, ${foreignRun}, 'assistant', 1, '', '', '{}') + `; + yield* sql` + INSERT INTO orchestration_v2_projection_messages + (message_id, thread_id, run_id, role, streaming, created_at, updated_at, payload_json) + VALUES ('old-history', ${empty}, NULL, 'assistant', 1, '', '', '{}') + `; + const projection = yield* projections + .getRuntimeRecoveryProjection(threadId) + .pipe(Effect.provideService(Statement.CurrentTransformer, observe)); + assert.deepEqual( + projection.messages.map((message) => message.id), + expected, + ); + const emptyProjection = yield* projections + .getRuntimeRecoveryProjection(empty) + .pipe(Effect.provideService(Statement.CurrentTransformer, observe)); + assert.deepEqual(emptyProjection.messages, []); + assert.equal(statements.length, 2); + for (const [query, params] of statements) { + const plan = yield* sql.unsafe<{ detail: string }>(`EXPLAIN QUERY PLAN ${query}`, params); + assert.isTrue(plan[0]?.detail.includes("SEARCH run "), JSON.stringify(plan)); + assert.isTrue( + plan.some((row) => row.detail.includes("messages_run_idx")), + JSON.stringify(plan), + ); + assert.isFalse( + plan.some((row) => row.detail.includes("messages_thread_created_idx")), + JSON.stringify(plan), + ); + } + }).pipe(Effect.provide(layerTest)), +); + it.effect("bounds runtime provider-thread reads by ownership and preserves recovery liveness", () => Effect.gen(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index c0bb3569a3dfb..dd259a4d7f68d 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -31,10 +31,14 @@ import { } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as Deferred from "effect/Deferred"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as Tracer from "effect/Tracer"; import * as SqlClient from "effect/sql/SqlClient"; +import * as Statement from "effect/sql/Statement"; import { projectThreadAwarenessV2 } from "@t3tools/shared/agentAwareness"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; @@ -50,6 +54,8 @@ import { } from "./threadHistoryPaging.ts"; import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts"; +import { layerReaderDatabase, traceSqlRouting } from "../persistence/testkit/SqliteRouting.ts"; + const layerTest = Layer.mergeAll( ProjectionStore.layer.pipe(Layer.provideMerge(SqlitePersistence.layerMemory)), SqlitePersistence.layerMemory, @@ -64,6 +70,7 @@ const encodeUnknownJsonString = Schema.encodeSync(Schema.fromJsonString(Schema.U const traceSqlStatements = () => { const statements: Array = []; + const queryTransactions: Array = []; const tracer = Tracer.make({ span(options) { const span = new Tracer.NativeSpan(options); @@ -71,12 +78,19 @@ const traceSqlStatements = () => { span.end = (endTime, exit) => { end(endTime, exit); const query = span.attributes.get("db.query.text"); - if (typeof query === "string") statements.push(query); + if (typeof query === "string") { + statements.push(query); + let parent = Option.getOrUndefined(span.parent); + while (parent?._tag === "Span" && parent.name !== "sql.transaction") { + parent = Option.getOrUndefined(parent.parent); + } + queryTransactions.push(parent?._tag === "Span" ? parent.spanId : null); + } }; return span; }, }); - return { statements, tracer }; + return { statements, tracer, queryTransactions }; }; /** The shell index flags every live shell and matches its shells on the fields subscriber trims read. */ @@ -229,6 +243,93 @@ const addRolledBackRecoveryCandidate = Effect.fn("addRolledBackRecoveryCandidate return threadId; }); +it.effect("routes full projection reads to readers and retains uncommitted writer state", () => + Effect.gen(function* () { + const store = yield* ProjectionStore.ProjectionStoreV2; + const sql = yield* SqlClient.SqlClient; + const parentId = yield* addRolledBackRecoveryCandidate("reader-routing"); + const parent = yield* store.getThreadProjection(parentId); + const threadId = ThreadId.make("thread:reader-routing:fork"); + yield* store.apply({ + id: EventId.make("event:reader-routing:fork"), + type: "thread.created", + threadId, + occurredAt: parent.thread.createdAt, + payload: { + ...parent.thread, + id: threadId, + forkedFrom: { type: "run", threadId: parentId, runId: parent.runs[0]!.id }, + }, + }); + const { queries, trace } = traceSqlRouting(); + const projection = yield* store.getThreadProjection(threadId).pipe(trace); + assert.equal(projection.thread.id, threadId); + assert.deepEqual( + projection.visibleTurnItems + .filter((row) => row.visibility === "inherited") + .map((row) => row.sourceItemId), + parent.turnItems.map((item) => item.id), + ); + assert.isAbove(queries.length, 1); + assert.deepEqual([...new Set(queries.map((row) => row.connection))], ["reader"]); + const commitRequested = yield* Deferred.make(); + const writer = yield* Deferred.await(commitRequested).pipe( + Effect.andThen( + sql.withTransaction( + Effect.gen(function* () { + yield* sql` + UPDATE orchestration_v2_projection_threads + SET payload_json = json_set(payload_json, '$.title', 'Committed title') + WHERE thread_id = ${threadId} + `; + yield* sql` + UPDATE orchestration_v2_projection_turn_items + SET payload_json = json_set(payload_json, '$.title', 'Committed command') + WHERE thread_id = ${parentId} + `; + }), + ), + ), + Effect.forkChild, + ); + let commitInjected = false; + const duringCommit = yield* store.getThreadProjection(threadId).pipe( + Effect.provideService(Statement.CurrentTransformer, (statement) => { + const [query] = statement.compile(); + if (!commitInjected && query.includes("FROM orchestration_v2_projection_turn_items")) { + commitInjected = true; + return Deferred.succeed(commitRequested, undefined).pipe( + Effect.andThen(Fiber.join(writer).pipe(Effect.orDie)), + Effect.as(statement), + ); + } + return Effect.succeed(statement); + }), + ); + assert.isTrue(commitInjected); + assert.deepEqual(duringCommit, projection); + const afterCommit = yield* store.getThreadProjection(threadId); + assert.equal(afterCommit.thread.title, "Committed title"); + assert.equal( + afterCommit.visibleTurnItems.find((row) => row.visibility === "inherited")?.item.title, + "Committed command", + ); + yield* sql.withTransaction( + Effect.gen(function* () { + yield* sql` + UPDATE orchestration_v2_projection_threads + SET payload_json = json_set(payload_json, '$.title', 'Uncommitted title') + WHERE thread_id = ${threadId} + `; + assert.equal( + (yield* store.getThreadProjection(threadId)).thread.title, + "Uncommitted title", + ); + }), + ); + }).pipe(Effect.provide(ProjectionStore.layer.pipe(Layer.provideMerge(layerReaderDatabase)))), +); + it.effect("record reads restrict provider histories to delivery identities", () => Effect.gen(function* () { const store = yield* ProjectionStore.ProjectionStoreV2; @@ -334,6 +435,8 @@ it.effect("idle compaction filters keep the latest run, live work and the newest [2, "completed"], [3, "completed"], [4, "failed"], + // A removed or steered queued message. + [5, "cancelled"], ] as const) { const runId = RunId.make(`run:idle-compaction:${ordinal}`); yield* store.apply({ @@ -357,7 +460,7 @@ it.effect("idle compaction filters keep the latest run, live work and the newest activeAttemptId: null, status, requestedAt: now, - startedAt: now, + startedAt: status === "cancelled" ? null : now, completedAt: now, checkpointId: null, contextHandoffId: null, @@ -458,7 +561,8 @@ it.effect("idle compaction filters keep the latest run, live work and the newest contextTransferStatuses: ["pending"], }, ); - // The failed latest run stays, so a completed earlier run is not mistaken for the latest. + // The failed latest run stays, so a completed earlier run is not mistaken for the latest, + // and a cancelled queued run never counts as the latest. assert.deepEqual( records.runs.map((run) => run.id), ["run:idle-compaction:rolled-back", "run:idle-compaction:4"], @@ -3802,6 +3906,76 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { .filter((id) => id === readThreadId || id === laterThreadId); yield* createThread(readThreadId); + yield* sql`UPDATE orchestration_v2_projection_threads + SET payload_json = json_set(payload_json, '$.linkedPullRequest', json(${JSON.stringify({ + projectId: "project:shell-read", + repository: "acme/project", + number: 42, + url: "https://github.com/acme/project/pull/42", + })})) WHERE thread_id = ${readThreadId}`; + const trace = traceSqlStatements(); + const discovery = yield* projectionStore + .getPullRequestSnapshot({}) + .pipe(Effect.withTracer(trace.tracer)); + assert.deepEqual( + discovery.threads + .map((thread) => thread.id) + .filter((id) => id === readThreadId || id === laterThreadId), + [readThreadId], + ); + assert.strictEqual(discovery.snapshotSequence, 0); + assert.strictEqual(trace.queryTransactions.length, 2); + assert.isNotNull(trace.queryTransactions[0]); + assert.strictEqual(new Set(trace.queryTransactions).size, 1); + assert.isFalse( + trace.statements.some((query) => + /orchestration_v2_projection_(runs|messages|turn_items|provider_threads)/.test(query), + ), + ); + const fullShell = (yield* projectionStore.getShellSnapshot()).threads.find( + (thread) => thread.id === readThreadId, + )!; + const metadata = discovery.threads.find((thread) => thread.id === readThreadId)!; + for (const key of Object.keys(metadata) as Array) { + assert.deepEqual(metadata[key], fullShell[key]); + } + const narrow = yield* projectionStore.getPullRequestSnapshot({ threadId: readThreadId }); + assert.deepEqual( + narrow.threads, + discovery.threads.filter((thread) => thread.id === readThreadId), + ); + const settledPayload = (settledAt: string | null, archivedAt: string | null) => sql` + UPDATE orchestration_v2_projection_threads + SET payload_json = json_set(payload_json, '$.settledAt', ${settledAt}, '$.archivedAt', ${archivedAt}) + WHERE thread_id = ${readThreadId} + `; + yield* settledPayload(DateTime.formatIso(now), null); + assert.isFalse( + (yield* projectionStore.getPullRequestSnapshot({ unsettledOnly: true })).threads.some( + (thread) => thread.id === readThreadId, + ), + ); + assert.strictEqual( + (yield* projectionStore.getPullRequestSnapshot({ threadId: readThreadId })).threads.length, + 1, + ); + yield* settledPayload(null, DateTime.formatIso(now)); + assert.isFalse( + (yield* projectionStore.getPullRequestSnapshot({})).threads.some( + (thread) => thread.id === readThreadId, + ), + ); + assert.strictEqual( + (yield* projectionStore.getPullRequestSnapshot({ threadId: readThreadId })).threads.length, + 1, + ); + yield* sql`UPDATE orchestration_v2_projection_threads SET deleted_at = '2026-11-03T00:00:00.000Z' WHERE thread_id = ${readThreadId}`; + assert.deepEqual( + (yield* projectionStore.getPullRequestSnapshot({ threadId: readThreadId })).threads, + [], + ); + yield* sql`UPDATE orchestration_v2_projection_threads SET deleted_at = NULL WHERE thread_id = ${readThreadId}`; + yield* settledPayload(null, null); const decode = yield* sql.withTransaction(projectionStore.readShellSnapshot()); yield* createThread(laterThreadId); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index bf8f50fa01d65..47d7c08881656 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -397,6 +397,7 @@ export type ProjectionRecords = Pick< export interface ProjectionTimelinePageOptions { readonly afterPosition?: number; readonly itemId?: TurnItemId; + readonly attachmentId?: string; readonly view?: "messages" | "activity"; readonly limit: number; } @@ -497,6 +498,16 @@ export interface ProjectionStoreV2Shape { readonly apply: ( event: OrchestrationV2DomainEvent, ) => Effect.Effect; + readonly getPullRequestSnapshot: (options: { + readonly threadId?: ThreadId; + readonly unsettledOnly?: boolean; + }) => Effect.Effect< + { + readonly snapshotSequence: number; + readonly threads: ReadonlyArray; + }, + ProjectionStoreV2Error + >; readonly getShellSnapshot: ( options?: ShellSnapshotOptions, ) => Effect.Effect; @@ -1127,6 +1138,37 @@ const encodeContextTransferPayload = Schema.encodeEffect( Schema.fromJsonString(OrchestrationV2ContextTransferJsonSchema), ); +const PullRequestThread = OrchestrationV2AppThreadJsonSchema.mapFields( + ({ + id, + projectId, + lineage, + branch, + worktreePath, + archivedAt, + settledOverride, + settledAt, + linkedPullRequest, + branchPullRequest, + pullRequests, + }) => ({ + id, + projectId, + lineage, + branch, + worktreePath, + archivedAt, + settledOverride, + settledAt, + linkedPullRequest, + branchPullRequest, + pullRequests, + }), +); +const decodePullRequestThread = Schema.decodeUnknownEffect( + Schema.fromJsonString(PullRequestThread), +); + const decodeThreadPayload = Schema.decodeUnknownEffect( Schema.fromJsonString(OrchestrationV2AppThreadJsonSchema), ); @@ -3332,6 +3374,7 @@ export const layer: Layer.Layer = OR (${filter.includeLatestRun === true ? 1 : 0} = 1 AND run_id = ( SELECT run_id FROM orchestration_v2_projection_runs WHERE thread_id = ${threadId} + AND (status <> 'cancelled' OR json_extract(payload_json, '$.startedAt') IS NOT NULL) ORDER BY ordinal DESC LIMIT 1 )))` @@ -4132,6 +4175,7 @@ export const layer: Layer.Layer = json_extract(child.payload_json, '$.lineage.relationshipToParent') = 'subagent' AND json_extract(child.payload_json, '$.lineage.parentThreadId') IS NOT NULL AND json_extract(child.payload_json, '$.forkedFrom.type') = 'node' + ELSE 0 END AND (EXISTS ( SELECT 1 FROM orchestration_v2_projection_runs AS requested JOIN orchestration_v2_projection_subagents AS task @@ -4154,7 +4198,6 @@ export const layer: Layer.Layer = AND target_thread_id = json_extract(child.payload_json, '$.lineage.parentThreadId') AND type = 'subagent_result' ))) - ELSE 0 END `; case "runtime": return sql` @@ -4533,7 +4576,15 @@ export const layer: Layer.Layer = ); const getThreadProjection: ProjectionStoreV2Shape["getThreadProjection"] = (threadId) => - readProjection(threadId, new Set()); + sql + .withTransaction(readProjection(threadId, new Set())) + .pipe( + Effect.mapError((cause) => + isProjectionStoreThreadNotFoundError(cause) || isProjectionStoreReadError(cause) + ? cause + : new ProjectionStoreReadError({ threadId, cause }), + ), + ); const getRuntimeRecoveryProjection: ProjectionStoreV2Shape["getRuntimeRecoveryProjection"] = Effect.fn("ProjectionStore.getRuntimeRecoveryProjection")( @@ -4782,14 +4833,14 @@ export const layer: Layer.Layer = WHERE thread_id = ${threadId} AND status = 'pending' ORDER BY created_at ASC, runtime_request_id ASC `, + // CROSS JOIN keeps empty run cohorts out of message history; the run index bounds each lookup. sql` - SELECT message.payload_json FROM orchestration_v2_projection_messages AS message - WHERE message.thread_id = ${threadId} AND message.streaming = 1 - AND message.run_id IN ( - SELECT run_id FROM orchestration_v2_projection_runs - WHERE thread_id = ${threadId} - AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting') - ) + SELECT message.payload_json FROM orchestration_v2_projection_runs AS run + CROSS JOIN orchestration_v2_projection_messages AS message + INDEXED BY orchestration_v2_projection_messages_run_idx ON message.run_id = run.run_id + WHERE run.thread_id = ${threadId} + AND run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting') + AND message.thread_id = ${threadId} AND message.streaming = 1 ORDER BY message.created_at ASC, message.message_id ASC `, sql` @@ -5519,7 +5570,8 @@ export const layer: Layer.Layer = FROM orchestration_v2_projection_turn_items WHERE thread_id = ${threadId} AND type = 'dynamic_tool' - AND (payload_json LIKE '%htmlRender%' OR payload_json LIKE ${`%${MCP_APP_OUTPUT_KEY}%`}) + AND (payload_json LIKE '%htmlRender%' OR payload_json LIKE ${`%${MCP_APP_OUTPUT_KEY}%`} + OR payload_json LIKE '%preview_recording_stop%') `, sql<{ id: string }>` SELECT DISTINCT json_extract(attachment.value, '$.id') AS id @@ -5900,6 +5952,20 @@ export const layer: Layer.Layer = .withTransaction( Effect.gen(function* () { const index = yield* readTimelineIndex(threadId, new Set()); + const attachmentOwners = + options.attachmentId === undefined + ? undefined + : new Set( + (yield* sql<{ thread_id: string; turn_item_id: string }>` + SELECT item.thread_id, item.turn_item_id + FROM orchestration_v2_projection_turn_items AS item, + json_each(item.payload_json, '$.attachments') AS attachment + WHERE item.thread_id IN ${sql.in([...new Set(index.visible.map((row) => row.sourceThreadId))])} + AND item.type = 'user_message' + AND json_extract(attachment.value, '$.id') = ${options.attachmentId}`).map( + (row) => `${row.thread_id}:${row.turn_item_id}`, + ), + ); const matching = index.visible .map((row, position) => ({ ...row, position })) .filter((row) => @@ -5910,6 +5976,9 @@ export const layer: Layer.Layer = row.item.type, )) : row.sourceItemId === options.itemId, + ) + .filter( + (row) => attachmentOwners?.has(`${row.sourceThreadId}:${row.sourceItemId}`) ?? true, ); const page = matching.slice(0, options.limit); const sources = yield* encodeTimelineSources( @@ -6494,6 +6563,16 @@ export const layer: Layer.Layer = INDEXED BY orchestration_v2_projection_turn_items_waiting_link_idx WHERE link.thread_id = t.thread_id AND link.type = 'link_request' AND link.status = 'waiting' + -- linkCardStillAsks: once the user writes after the card, it stops asking. + AND NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_messages message + WHERE message.thread_id = t.thread_id AND message.role = 'user' + AND json_extract(message.payload_json, '$.createdBy') = 'user' + AND NOT ${isIdleCompactionMessage("message")} + AND message.updated_at > COALESCE( + json_extract(link.payload_json, '$.startedAt'), link.updated_at + ) + ) ) ORDER BY t.updated_at ASC, t.thread_id ASC `; @@ -6944,6 +7023,41 @@ export const layer: Layer.Layer = Effect.map(decodeShellSnapshot), ); + const getPullRequestSnapshot: ProjectionStoreV2Shape["getPullRequestSnapshot"] = Effect.fn( + "ProjectionStore.getPullRequestSnapshot", + )(function* ({ threadId, unsettledOnly = false }) { + const { rows, sequenceRows } = yield* sql.withTransaction( + Effect.gen(function* () { + const rows = yield* sql` + SELECT t.payload_json FROM orchestration_v2_projection_threads t + WHERE t.deleted_at IS NULL${ + threadId === undefined + ? sql` AND json_extract(t.payload_json, '$.archivedAt') IS NULL` + : sql` AND t.thread_id = ${threadId}` + }${ + unsettledOnly + ? sql` AND json_extract(t.payload_json, '$.settledAt') IS NULL + AND json_extract(t.payload_json, '$.settledOverride') IS NOT 'settled'` + : sql`` + } + ORDER BY t.updated_at ASC, t.thread_id ASC + `; + const sequenceRows = yield* sql<{ readonly snapshot_sequence: number | null }>` + SELECT MAX(sequence) AS snapshot_sequence FROM orchestration_events + WHERE application_event_version = 2 AND aggregate_kind = 'thread' + `; + return { rows, sequenceRows }; + }), + ); + // Decode after releasing the reader, keeping rows and sequence from the same transaction. + const threads = yield* Effect.forEach(rows, (row) => + decodePullRequestThread(row.payload_json).pipe( + Effect.map((thread) => ({ ...thread, pullRequests: threadPullRequestsOf(thread) })), + ), + ); + return { snapshotSequence: sequenceRows[0]?.snapshot_sequence ?? 0, threads }; + }, Effect.mapError(shellSnapshotReadError)); + const getShellSnapshot: ProjectionStoreV2Shape["getShellSnapshot"] = (options) => sql .withTransaction(readShellSnapshotRows(options)) @@ -7222,6 +7336,7 @@ export const layer: Layer.Layer = return { apply, getShellSnapshot: readOnReader(getShellSnapshot), + getPullRequestSnapshot: readOnReader(getPullRequestSnapshot), listActiveSubagentThreadIds: readOnReader(listActiveSubagentThreadIds), countThreads: readOnReader(countThreads), listActiveShellIndex: readOnReader(listActiveShellIndex), @@ -7249,7 +7364,7 @@ export const layer: Layer.Layer = getThreadsWithPullRequests: readOnReader(getThreadsWithPullRequests), getThreadWorktreePaths: readOnReader(getThreadWorktreePaths), getOpenRemoteDelegatedTasks: getOpenRemoteDelegatedTasks.pipe(NodeSqliteClient.readOnly), - getThreadProjection, + getThreadProjection: readOnReader(getThreadProjection), getTurnStartContext: readOnReader(getTurnStartContext), getTurnStartHistory: readOnReader(getTurnStartHistory), getRuntimeRecoveryProjection: readOnReader(getRuntimeRecoveryProjection), diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts index 01e3e88b0f499..aa862389773cc 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts @@ -650,6 +650,14 @@ it.effect( nodeId: rootNodeId, status: "running", }, + ...(["assistant_message", "reasoning", "proposed_plan"] as const).map((type) => ({ + id: TurnItemId.make(`turn_item_recovery_cancel_${type}`), + runId, + nodeId: rootNodeId, + type, + status: "running", + streaming: true, + })), ], } as unknown as OrchestrationV2ThreadProjection; const layer = ProviderRuntimeRecovery.layer.pipe( @@ -703,12 +711,27 @@ it.effect( ["provider-turn.updated", "cancelled"], ["message.updated", null], ["turn-item.updated", "cancelled"], + ["turn-item.updated", "cancelled"], + ["turn-item.updated", "cancelled"], + ["turn-item.updated", "cancelled"], ["provider-thread.updated", "idle"], ["provider-session.updated", "stopped"], ], ); const messageEvent = events.find((event) => event.type === "message.updated"); assert.isFalse(messageEvent?.type === "message.updated" && messageEvent.payload.streaming); + assert.deepEqual( + events.flatMap((event) => + event.type === "turn-item.updated" && "streaming" in event.payload + ? [[event.payload.type, event.payload.streaming]] + : [], + ), + [ + ["assistant_message", false], + ["reasoning", false], + ["proposed_plan", false], + ], + ); }).pipe(Effect.provide(layer)); }, ); diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index 6135b107ceaa4..0d5ae73406762 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -155,7 +155,13 @@ const cancelledItem = ( ): OrchestrationV2ThreadProjection["turnItems"][number] => item.type === "secret_request" && item.secretStatus === "pending" ? { ...item, status: "cancelled", secretStatus: "cancelled", completedAt: now, updatedAt: now } - : { ...item, status: "cancelled", completedAt: now, updatedAt: now }; + : { + ...item, + ...("streaming" in item ? { streaming: false } : {}), + status: "cancelled", + completedAt: now, + updatedAt: now, + }; function latestStartedRun( projection: ProjectionStore.ProjectionRuntimeRecoveryState, diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts index 99007d402f3e5..af75b5c95f466 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts @@ -273,6 +273,7 @@ it.effect( getUnreadableThreadIds: () => Effect.die("unused getUnreadableThreadIds"), getThreadSnapshot: () => Effect.die("unused getThreadSnapshot"), getThreadSnapshotWindow: () => Effect.die("unused getThreadSnapshotWindow"), + getPullRequestSnapshot: () => Effect.die("unused getPullRequestSnapshot"), }), ); const layerSessionManager = Layer.succeed( diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index 2ad835fce92f6..2bbab37e0c62d 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -280,6 +280,7 @@ function makeLocalCommandHarness(input: { readonly openFailure?: unknown; readonly ensureThreadFailure?: unknown; readonly historyReadFailureAfterFallback?: unknown; + readonly resumeFailure?: ProviderAdapter.ProviderAdapterV2Error; readonly interruptOpen?: boolean; readonly interruptRunBeforeOpenFailure?: boolean; readonly writeFailure?: unknown; @@ -512,6 +513,35 @@ function makeLocalCommandHarness(input: { strength: "strong" as const, }, }; + const parentProviderTurn = (ordinal: number, attemptRunId: RunId) => ({ + attempt: { + ...projection.attempts[0]!, + id: RunAttemptId.make(`side-chat-parent-attempt-${ordinal}`), + runId: attemptRunId, + providerThreadId: parentProviderThread.id, + status: "completed" as const, + }, + turn: { + id: ProviderTurnId.make(`side-chat-parent-turn-${ordinal}`), + providerThreadId: parentProviderThread.id, + nodeId: rootNodeId, + runAttemptId: RunAttemptId.make(`side-chat-parent-attempt-${ordinal}`), + nativeTurnRef: { + driver: providerThread.driver, + nativeId: `side-chat-parent-native-turn-${ordinal}`, + strength: "strong" as const, + }, + ordinal, + status: "completed" as const, + startedAt: now, + completedAt: now, + }, + }); + const parentRolledBackRunId = RunId.make("side-chat-parent-rolled-back-run"); + const parentTurns = [ + parentProviderTurn(1, parentRunId), + parentProviderTurn(2, parentRolledBackRunId), + ]; const parentRecords = { ...projection, thread: { ...projection.thread, id: parentThreadId }, @@ -522,17 +552,27 @@ function makeLocalCommandHarness(input: { threadId: parentThreadId, ordinal: 1, providerThreadId: parentProviderThread.id, - activeAttemptId: null, + activeAttemptId: parentTurns[0]!.attempt.id, status: "completed" as const, completedAt: now, }, + { + ...run, + id: parentRolledBackRunId, + threadId: parentThreadId, + ordinal: 2, + providerThreadId: parentProviderThread.id, + activeAttemptId: parentTurns[1]!.attempt.id, + status: "rolled_back" as const, + completedAt: now, + }, ...(input.sideChat?.nativeForkFrom === "busy" ? [ { ...run, id: RunId.make("side-chat-parent-busy-run"), threadId: parentThreadId, - ordinal: 2, + ordinal: 3, providerThreadId: parentProviderThread.id, status: "running" as const, }, @@ -540,8 +580,8 @@ function makeLocalCommandHarness(input: { : []), ], providerThreads: [parentProviderThread], - attempts: [], - providerTurns: [], + attempts: parentTurns.map(({ attempt }) => attempt), + providerTurns: parentTurns.map(({ turn }) => turn), turnItems: [], }; if (input.sideChat !== undefined) { @@ -633,7 +673,7 @@ function makeLocalCommandHarness(input: { ], }; } - if ("historyReadFailureAfterFallback" in input) { + if ("historyReadFailureAfterFallback" in input || "resumeFailure" in input) { const nativeThreadRef = { driver: providerThread.driver, nativeId: "native-resume-thread", @@ -740,17 +780,19 @@ function makeLocalCommandHarness(input: { ), ), ); + const fallbackEnsureThread = vi.fn(() => Effect.succeed(providerThread)); const resumeFallbackSession = { driver: providerThread.driver, resumeThread: () => Effect.fail( - new ProviderAdapter.ProviderAdapterEventStreamError({ - driver: providerThread.driver, - providerSessionId, - cause: "native thread is gone", - }), + input.resumeFailure ?? + new ProviderAdapter.ProviderAdapterEventStreamError({ + driver: providerThread.driver, + providerSessionId, + cause: "native thread is gone", + }), ), - ensureThread: () => Effect.succeed(providerThread), + ensureThread: fallbackEnsureThread, }; const readyProviderSession = (driver: ProviderDriverKind) => ({ id: providerSessionId, @@ -773,7 +815,9 @@ function makeLocalCommandHarness(input: { strength: "strong" as const, }, }; - const forkThread = vi.fn(() => Effect.succeed(sideChatThread)); + const forkThread = vi.fn((_fork: ProviderAdapter.ProviderAdapterV2ForkThreadInput) => + Effect.succeed(sideChatThread), + ); const sideChatSession = { driver: providerThread.driver, providerSession: readyProviderSession(providerThread.driver), @@ -787,7 +831,7 @@ function makeLocalCommandHarness(input: { ? Effect.succeed(sideChatSession as never) : input.interruptOpen === true ? Effect.interrupt - : "historyReadFailureAfterFallback" in input + : "historyReadFailureAfterFallback" in input || "resumeFailure" in input ? Effect.succeed(resumeFallbackSession as never) : "ensureThreadFailure" in input ? Effect.succeed({ driver: providerThread.driver, ensureThread } as never) @@ -1008,6 +1052,7 @@ function makeLocalCommandHarness(input: { resolveRuntimePolicy, createWorktree, open, + fallbackEnsureThread, writeIfRunCurrent, startRootRun, startTurn, @@ -1310,6 +1355,10 @@ effectIt.effect.each([ yield* harness.start; expect(harness.forkThread).toHaveBeenCalledTimes(forks ? 1 : 0); + // The parent's rolled-back turn is gone from its native conversation. + expect( + harness.forkThread.mock.calls[0]?.[0].sourceProviderTurns?.map((turn) => turn.id), + ).toEqual(forks ? ["side-chat-parent-turn-1"] : undefined); expect(harness.projection().contextTransfers).toMatchObject([ { id: "side-chat-fork", status: transferStatus }, ]); @@ -1722,6 +1771,40 @@ effectIt.effect("does not overwrite a run interrupted while its provider session }), ); +effectIt.effect("fails the run instead of replacing a native thread another client holds", () => + Effect.gen(function* () { + const detail = "This conversation is open in another Codex client."; + const harness = makeLocalCommandHarness({ + text: "Continue", + resumeFailure: new ProviderAdapter.ProviderAdapterThreadInUseError({ + driver: ProviderDriverKind.make("codex"), + providerThreadId: ProviderThreadId.make("new-provider-thread"), + detail, + cause: new Error("thread native-resume-thread already has an active writer"), + }), + }); + + const exit = yield* Effect.exit(harness.start); + + expect(harness.fallbackEnsureThread).not.toHaveBeenCalled(); + expect(exit._tag).toBe("Success"); + expect(harness.startRootRun).not.toHaveBeenCalled(); + const projection = harness.projection(); + expect(projection.providerThreads.at(-1)?.nativeThreadRef?.nativeId).toBe( + "native-resume-thread", + ); + expect(projection.contextHandoffs).toEqual([]); + expect(projection.runs.at(-1)).toMatchObject({ status: "failed", startedAt: null }); + expect(projection.turnItems).toMatchObject([ + { + type: "error", + title: "Provider turn failed to start", + failure: { class: "provider_error", message: detail }, + }, + ]); + }), +); + effectIt.effect("does not mistake a failed state read for a superseded run", () => Effect.gen(function* () { const harness = makeLocalCommandHarness({ text: "Continue", failReadsAfterRunning: true }); diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 3f454e5956522..46687c69db5a3 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -1177,10 +1177,24 @@ export const layer: Layer.Layer< (candidate) => candidate.ordinal > sourceRun.ordinal && isActiveRun(candidate), ); if (!sourceBusy) { + // Rolled-back turns are no longer in the source conversation, so they cannot bound a fork. + const rolledBackAttemptIds = new Set( + sourceProjection.attempts + .filter((attempt) => + sourceProjection.runs.some( + (candidate) => + candidate.id === attempt.runId && candidate.status === "rolled_back", + ), + ) + .map((attempt) => attempt.id), + ); return yield* loadFromProvider( session.forkThread({ sourceProviderThread, - sourceProviderTurns: sourceProjection.providerTurns, + sourceProviderTurns: sourceProjection.providerTurns.filter( + (turn) => + turn.runAttemptId === null || !rolledBackAttemptIds.has(turn.runAttemptId), + ), targetThreadId: projection.thread.id, modelSelection: run.modelSelection, runtimePolicy: resolvedRuntimePolicy, @@ -1349,6 +1363,14 @@ export const layer: Layer.Layer< resumedHandoffContext = undefined; resumedRetryHandoff = []; if (!(yield* isCurrentAttemptInStatus("starting"))) return undefined; + // Another client is still writing to the native thread. A fresh + // session here would silently split the conversation from it. + if ( + resumed._tag === "Failure" && + resumed.failure._tag === "ProviderAdapterThreadInUseError" + ) { + return yield* loadFromProvider(Effect.fail(resumed.failure)); + } yield* Effect.logWarning("Recovering provider context in a fresh native session", { driver: session.driver, diff --git a/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts b/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts index a1bacda0806ed..0161054781567 100644 --- a/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts +++ b/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts @@ -979,6 +979,49 @@ describe("PullRequestSyncReactor", () => { ), ); + it.effect("stops reading a pull request a day after it closed, until a refresh asks", () => + Effect.scoped( + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse(NOW)); + const closed = (closedAt: string | null) => ({ state: "closed" as const, closedAt }); + const fixture = yield* makeHarness({ + snapshot: makeSnapshot([ + makeThread("closed", { + pullRequests: [ + makeLink(1, closed("2026-08-20T00:00:00.000Z")), + makeLink(2, closed("2026-08-26T00:00:00.000Z")), + makeLink(3, closed("2026-08-28T11:00:00.000Z")), + makeLink(4, closed(null)), + ], + }), + ]), + summary: (input) => + Effect.succeed(makeSummary(input, { state: "closed", closedAt: NOW })), + }); + const reads = Ref.get(fixture.summaryCalls).pipe( + Effect.map((calls) => calls.map((call) => call.number).toSorted()), + ); + + yield* Effect.gen(function* () { + const reactor = yield* startAndSweep(fixture); + // Closed within the day, or with no close time, keeps the fifteen-minute cadence. + assert.deepStrictEqual(yield* reads, [3, 4]); + for (let index = 0; index < 15; index += 1) yield* sweepAgain(fixture, reactor); + assert.deepStrictEqual(yield* reads, [3, 3, 4, 4]); + + yield* reactor.requestSync({ + host: "github.com", + repository: "owner/repository", + number: 1, + }); + yield* Queue.take(fixture.snapshotReads); + yield* reactor.drain; + assert.deepStrictEqual(yield* reads, [1, 3, 3, 4, 4]); + }).pipe(Effect.provide(fixture.layer)); + }), + ), + ); + it.effect("re-reads a merged pull request until the host supplies its merge time", () => Effect.scoped( Effect.gen(function* () { @@ -1579,46 +1622,55 @@ describe("PullRequestSyncReactor", () => { ), ); - it.effect("reads open links fresh as soon as a merge command ends, before its run ends", () => - Effect.scoped( - Effect.gen(function* () { - yield* TestClock.setTime(Date.parse(NOW)); - const state = yield* Ref.make("open"); - const fixture = yield* makeHarness({ - snapshot: makeSnapshot([ - makeThread("agent", { pullRequests: [makeLink(7, { state: "open" })] }), - ]), - summary: (input) => - Ref.get(state).pipe( - Effect.map((current) => - makeSummary(input, current === "merged" ? { state: current, mergedAt: NOW } : {}), + it.effect.each(["open", "closed"] as const)( + "reads %s links fresh as soon as a merge command ends, before its run ends", + (initialState) => + Effect.scoped( + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse(NOW)); + const state = yield* Ref.make(initialState); + const initialSnapshot = { + state: initialState, + closedAt: initialState === "closed" ? "2026-08-20T00:00:00.000Z" : null, + }; + const fixture = yield* makeHarness({ + snapshot: makeSnapshot([ + makeThread("agent", { pullRequests: [makeLink(7, initialSnapshot)] }), + ]), + summary: (input) => + Ref.get(state).pipe( + Effect.map((current) => + makeSummary( + input, + current === "merged" ? { state: current, mergedAt: NOW } : initialSnapshot, + ), + ), ), - ), - }); + }); - yield* Effect.gen(function* () { - const reactor = yield* startAndSweep(fixture); - yield* Ref.set(state, "merged"); - yield* Ref.set(fixture.summaryCalls, []); + yield* Effect.gen(function* () { + const reactor = yield* startAndSweep(fixture); + yield* Ref.set(state, "merged"); + yield* Ref.set(fixture.summaryCalls, []); - yield* Queue.offer( - fixture.domainEvents, - commandRan(ThreadId.make("agent"), "gh pr merge 7"), - ); - yield* Queue.take(fixture.snapshotReads); - yield* Queue.take(fixture.snapshotReads); - yield* reactor.drain; + yield* Queue.offer( + fixture.domainEvents, + commandRan(ThreadId.make("agent"), "gh pr merge 7"), + ); + yield* Queue.take(fixture.snapshotReads); + yield* Queue.take(fixture.snapshotReads); + yield* reactor.drain; - assert.deepStrictEqual( - (yield* Ref.get(fixture.syncCommands)).map((command) => [ - command.number, - command.snapshot.state, - ]), - [[7, "merged"]], - ); - }).pipe(Effect.provide(fixture.layer)); - }), - ), + assert.deepStrictEqual( + (yield* Ref.get(fixture.syncCommands)).map((command) => [ + command.number, + command.snapshot.state, + ]), + [[7, "merged"]], + ); + }).pipe(Effect.provide(fixture.layer)); + }), + ), ); it.effect.each([ diff --git a/apps/server/src/orchestration-v2/PullRequestSyncReactor.ts b/apps/server/src/orchestration-v2/PullRequestSyncReactor.ts index 25c062301e35f..9dffe4babb424 100644 --- a/apps/server/src/orchestration-v2/PullRequestSyncReactor.ts +++ b/apps/server/src/orchestration-v2/PullRequestSyncReactor.ts @@ -44,6 +44,9 @@ const SLOW_SYNC_INTERVAL_MS = 15 * 60 * 1_000; // requested refreshes, watches and a host CLI's merge command still read it at once. const QUIET_AFTER_MS = 30 * 60 * 1_000; const QUIET_SYNC_INTERVAL_MS = 5 * 60 * 1_000; +// A pull request closed this long ago is rarely reopened, so the sweep stops reading it, as it +// does merged ones. A reader that sees it open, or a requested refresh, still reads it at once. +const CLOSED_FINAL_AFTER_MS = 24 * 60 * 60 * 1_000; const isChangeRequestCloseCommand = (command: string) => sourceControlDefinitions.definitions.some( @@ -351,13 +354,12 @@ export const make = Effect.gen(function* () { if (entries.some((entry) => entry.link.snapshot === null)) return true; const active = entries.filter((entry) => isUnsettled(entry.thread)); // Merged snapshots without mergedAt cannot settle threads; reread slowly until supplied. - if ( - active.every( - (entry) => entry.link.snapshot?.state === "merged" && entry.link.snapshot.mergedAt != null, - ) - ) { - return false; - } + const isFinal = ({ link: { snapshot } }: LinkEntry) => + snapshot?.state === "merged" + ? snapshot.mergedAt != null + : snapshot?.state === "closed" && + nowMs - Date.parse(snapshot.closedAt ?? "") >= CLOSED_FINAL_AFTER_MS; + if (active.every(isFinal)) return false; const last = lastSyncedAt.get(key); const sinceLastMs = last === undefined ? Number.POSITIVE_INFINITY : nowMs - last; const open = active.filter((entry) => entry.link.snapshot?.state === "open"); @@ -627,13 +629,13 @@ export const make = Effect.gen(function* () { // Threads that ran a close command this run, with the commands already read for. const closeCommands = new Map>(); - const refreshOpenLinks = (threadId: ThreadId) => + const refreshUnmergedLinks = (threadId: ThreadId) => projections.getThreadsWithPullRequests(threadId).pipe( Effect.flatMap((threads) => Effect.forEach( threads.flatMap((thread) => visibleThreadPullRequests(thread.pullRequests ?? []).filter( - (link) => link.snapshot?.state === "open", + (link) => link.snapshot?.state === "open" || link.snapshot?.state === "closed", ), ), requestSync, @@ -698,7 +700,7 @@ export const make = Effect.gen(function* () { { discard: true }, ); // An agent can merge or close its pull request from a shell with its host's CLI, which - // sends no merge notification. Read the thread's open links fresh once that command + // sends no merge notification. Read the thread's unmerged links fresh once that command // ends, while the agent usually still replies, so settlement can land before the run // ends. Read again when the run ends, since `merge --auto` returns before it merges. case "turn-item.updated": { @@ -710,11 +712,11 @@ export const make = Effect.gen(function* () { closeCommands.set(event.threadId, ended); if (isOrchestrationV2WorkActive(item.status) || ended.has(item.id)) return linked; ended.add(item.id); - return Effect.andThen(linked, refreshOpenLinks(event.threadId)); + return Effect.andThen(linked, refreshUnmergedLinks(event.threadId)); } case "run.updated": return isTerminalRunStatus(event.payload.status) && closeCommands.delete(event.threadId) - ? refreshOpenLinks(event.threadId) + ? refreshUnmergedLinks(event.threadId) : Effect.void; default: return Effect.void; diff --git a/apps/server/src/orchestration-v2/PullRequestWatchReactor.ts b/apps/server/src/orchestration-v2/PullRequestWatchReactor.ts index 11fcf2067ca74..c572d1f86c303 100644 --- a/apps/server/src/orchestration-v2/PullRequestWatchReactor.ts +++ b/apps/server/src/orchestration-v2/PullRequestWatchReactor.ts @@ -516,9 +516,9 @@ export const make = Effect.gen(function* () { } readFailures.delete(group.key); const [detail, activity] = read.value; - const headSha = detail.headSha ?? null; for (const target of group.targets) { const life = lifeOf(target); + const headSha = detail.headSha ?? life.headSha; const pushed = life.headSha !== null && headSha !== life.headSha; lives.set(lifeKey(target), { ...life, diff --git a/apps/server/src/orchestration-v2/RestartBackgroundNote.test.ts b/apps/server/src/orchestration-v2/RestartBackgroundNote.test.ts index 24269dfeaff24..954ba2f31a3eb 100644 --- a/apps/server/src/orchestration-v2/RestartBackgroundNote.test.ts +++ b/apps/server/src/orchestration-v2/RestartBackgroundNote.test.ts @@ -113,6 +113,13 @@ it("names a lost subagent by the id the agent resumes it with", () => { parentTaskId: "a".repeat(400), }).label; assert.lengthOf(longId, 160); + assert.isTrue( + cancelledRosterTaskWork({ + kind: "command", + taskId: "shell-1", + description: `${"d".repeat(158)}🚀 deploy`, + }).label.isWellFormed(), + ); assert.isFalse(isAwaitedWork(cancelledRosterTaskWork({ kind: "command", taskId: "shell-2" }))); assert.isTrue( isAwaitedWork(cancelledRosterTaskWork({ kind: "command", taskId: "tests", awaited: true })), diff --git a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts index 24afd61d6186a..c495290ad0813 100644 --- a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts +++ b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts @@ -17,7 +17,11 @@ function compactLabel(value: unknown, maxLength = MAX_LABEL_LENGTH): string | un if (typeof value !== "string") return undefined; const text = value.replaceAll(/\s+/g, " ").trim(); if (text.length === 0) return undefined; - return text.length > maxLength ? `${text.slice(0, maxLength - 1)}…` : text; + if (text.length <= maxLength) return text; + // Half an emoji serializes to a lone surrogate escape, which provider JSON parsers reject. + const last = text.charCodeAt(maxLength - 2); + const end = last >= 0xd800 && last <= 0xdbff ? maxLength - 2 : maxLength - 1; + return `${text.slice(0, end)}…`; } /** A bounded label that keeps its id suffix whole whenever it fits. */ diff --git a/apps/server/src/orchestration-v2/RuntimePolicy.test.ts b/apps/server/src/orchestration-v2/RuntimePolicy.test.ts index 21647ef49bf28..ec610b093f7d3 100644 --- a/apps/server/src/orchestration-v2/RuntimePolicy.test.ts +++ b/apps/server/src/orchestration-v2/RuntimePolicy.test.ts @@ -15,6 +15,7 @@ import * as Layer from "effect/Layer"; import * as Stream from "effect/Stream"; import type { ProviderInstance } from "@t3tools/provider-core/server/driver"; +import { acpPermissionDisposition } from "@t3tools/provider-acp/server/clientPolicy"; import * as ServerSettings from "../serverSettings.ts"; import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts"; import * as ProjectStore from "./ProjectStore.ts"; @@ -63,8 +64,10 @@ function makeThread(input: { } const grokInstanceId = ProviderInstanceId.make("grok"); +const antigravityInstanceId = ProviderInstanceId.make("antigravity"); const supportedRuntimeModesByInstance = new Map>([ [grokInstanceId, ["approval-required", "auto", "full-access"]], + [antigravityInstanceId, ["approval-required", "auto-accept-edits", "full-access"]], ]); const providerInstanceFor = (instanceId: ProviderInstanceId) => ({ @@ -113,6 +116,28 @@ const layerWithPolicy = (policy?: ManagedPolicySummary) => const layerTest = layerWithPolicy(); it.layer(layerTest)("RuntimePolicyV2", (it) => { + it.effect("asks for Antigravity command approval when a stored Auto thread falls back", () => + Effect.gen(function* () { + const policy = yield* RuntimePolicy.RuntimePolicyV2; + const now = yield* DateTime.now; + const resolve = (runtimeMode: RuntimeMode) => + policy.resolve({ + thread: makeThread({ now, worktreePath: null, runtimeMode }), + modelSelection: { instanceId: antigravityInstanceId, model: "test-model" }, + }); + const request = { + sessionId: "session-1", + toolCall: { toolCallId: "command-1", title: "Run command", kind: "execute" as const }, + options: [{ optionId: "allow", name: "Allow", kind: "allow_once" as const }], + }; + const auto = yield* resolve("auto"); + assert.equal(auto.runtimeMode, "approval-required"); + assert.equal(acpPermissionDisposition(auto, request), "ask"); + assert.equal(acpPermissionDisposition(yield* resolve("auto-accept-edits"), request), "ask"); + assert.equal(acpPermissionDisposition(yield* resolve("full-access"), request), "allow"); + }), + ); + it.effect("runs a mode the provider does not offer in Supervised", () => Effect.gen(function* () { const policy = yield* RuntimePolicy.RuntimePolicyV2; diff --git a/apps/server/src/orchestration-v2/ThreadFork.execution.test.ts b/apps/server/src/orchestration-v2/ThreadFork.execution.test.ts index 96c7263173f32..d955755e92122 100644 --- a/apps/server/src/orchestration-v2/ThreadFork.execution.test.ts +++ b/apps/server/src/orchestration-v2/ThreadFork.execution.test.ts @@ -852,7 +852,11 @@ const forkSideChat = (commandId: string, sourceThreadId: ThreadId, targetThreadI }); }); -const askSideChat = (threadId: ThreadId, commandId: string) => +const askSideChat = ( + threadId: ThreadId, + commandId: string, + modelSelection?: (typeof forkCases)[number]["modelSelection"], +) => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; yield* orchestrator.dispatch({ @@ -862,6 +866,7 @@ const askSideChat = (threadId: ThreadId, commandId: string) => messageId: MessageId.make(commandId), text: "What is the main thread doing?", attachments: [], + ...(modelSelection === undefined ? {} : { modelSelection }), dispatchMode: { type: "start_immediately" }, createdBy: "user", creationSource: "web", @@ -954,10 +959,17 @@ it.effect("a side chat of an idle parent still starts with the framing note", () query.includes("FROM orchestration_v2_projection_context_transfers"), ), ); - yield* askSideChat(sideChatId, "side-chat-idle-first-message"); + yield* askSideChat(sideChatId, "side-chat-idle-first-message", { + ...forkCases[0]!.modelSelection, + model: "side-chat-model", + }); const started = yield* orchestrator.getThreadProjection(sideChatId); assert.isNotNull(started.providerThreads[0]?.forkedFrom); + // The side chat does not show the parent's run, so it is no baseline. + assert.isFalse( + started.turnItems.some((item) => item.type === "user_message" && item.modelSwitch), + ); assert.lengthOf(started.contextHandoffs, 1); const [handoff] = started.contextHandoffs; assert.isTrue( diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts index d0fe470482366..f41a138d75b43 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts @@ -173,6 +173,10 @@ function makeHarness(options: HarnessOptions = {}) { (_input: Parameters[0]) => Effect.void, ); + const deleteLocalBranch = vi.fn( + (_input: Parameters[0]) => + Effect.void, + ); const runSetup = vi.fn( options.runSetup ?? (() => Effect.succeed({ status: "no-script" as const })), ); @@ -220,6 +224,7 @@ function makeHarness(options: HarnessOptions = {}) { remoteExists: options.remoteExists ?? (() => Effect.succeed(true)), remoteBranchExists: () => Effect.succeed(true), removeWorktree, + deleteLocalBranch, ...options.git, }), Layer.succeed(ProjectSetupScriptRunner.ProjectSetupScriptRunner, { @@ -275,7 +280,12 @@ function makeHarness(options: HarnessOptions = {}) { }); const layerTitleRegeneration = ThreadTitleRegeneration.layer.pipe( Layer.provide( - Layer.mergeAll(layerThreadManagement, layerProjectedProjects, layerExternalServices), + Layer.mergeAll( + layerThreadManagement, + layerProjectedProjects, + layerExternalServices, + NodeServices.layer, + ), ), ); return { @@ -291,6 +301,7 @@ function makeHarness(options: HarnessOptions = {}) { ), createWorktree, removeWorktree, + deleteLocalBranch, renameBranch, generateBranchName, generateThreadTitle, @@ -2493,8 +2504,15 @@ it.effect("shared intake preserves durable attachment bytes after a lost launch it.effect("cancels tracked setup before provider work is released", () => Effect.gen(function* () { const entered = yield* Deferred.make(); + const renamed = yield* Deferred.make(); const harness = makeHarness({ - runSetup: () => Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never)), + renameBranch: (input) => + Deferred.succeed(renamed, undefined).pipe(Effect.as({ branch: input.newBranch })), + runSetup: () => + Deferred.await(renamed).pipe( + Effect.andThen(Deferred.succeed(entered, undefined)), + Effect.andThen(Effect.never), + ), }); yield* Effect.gen(function* () { const launches = yield* ThreadLaunch.ThreadLaunchService; @@ -2515,11 +2533,50 @@ it.effect("cancels tracked setup before provider work is released", () => const projection = yield* threads.getThreadProjection(launched.threadId); assert.equal(projection.runs[0]?.status, "failed"); assert.isNull(projection.thread.worktreePath); + assert.deepEqual(harness.deleteLocalBranch.mock.calls[0]?.[0], { + cwd: "/repo", + refName: "generated-branch", + force: true, + }); assert.isEmpty(yield* outbox.listByCommandId(CommandId.make(`${input.commandId}:release`))); }).pipe(Effect.provide(harness.layer)); }), ); +it.effect( + "deletes a claimed worktree's branch when setup is cancelled before checkout returns", + () => + Effect.gen(function* () { + const claimed = yield* Deferred.make(); + const harness = makeHarness({ + createWorktree: (_input, options) => + (options?.progress?.onWorktreeClaimed?.("/repo-worktrees/feature") ?? Effect.void).pipe( + Effect.andThen(Deferred.succeed(claimed, undefined)), + Effect.andThen(Effect.never), + ), + }); + yield* Effect.gen(function* () { + const launches = yield* ThreadLaunch.ThreadLaunchService; + const tracker = yield* WorktreeSetupTracker.WorktreeSetupTracker; + const launched = yield* launches.launch( + launchInput({ + command: "launch:cancel-claimed", + thread: "thread:cancel-claimed", + message: "Start", + workspace: { type: "worktree", baseRef: "main" }, + }), + ); + yield* Deferred.await(claimed); + assert.isTrue(yield* tracker.cancel(launched.threadId)); + assert.equal(harness.removeWorktree.mock.calls[0]?.[0].path, "/repo-worktrees/feature"); + assert.equal( + harness.deleteLocalBranch.mock.calls[0]?.[0].refName, + harness.createWorktree.mock.calls[0]?.[0].newRefName, + ); + }).pipe(Effect.provide(harness.layer)); + }), +); + it.effect("stops an in-flight branch rename when tracked setup is cancelled", () => Effect.gen(function* () { const setupEntered = yield* Deferred.make(); @@ -2566,6 +2623,10 @@ it.effect("stops an in-flight branch rename when tracked setup is cancelled", () ); assert.equal(harness.renameBranch.mock.calls.length, 0); assert.isNull((yield* threads.getThreadProjection(launched.threadId)).thread.worktreePath); + assert.equal( + harness.deleteLocalBranch.mock.calls[0]?.[0].refName, + harness.createWorktree.mock.calls[0]?.[0].newRefName, + ); }).pipe(Effect.provide(harness.layer)); }), ); diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.ts index bbb68464ba7cc..b91f61cca6738 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.ts @@ -406,16 +406,22 @@ const make = Effect.gen(function* () { readonly threadId: ThreadId; readonly worktreePath: string; readonly oldBranch: string; + readonly onRenamed?: (branch: string) => void; readonly generated: Effect.Effect<{ readonly branch: string; readonly exactName: boolean }, E>; }) => input.generated.pipe( Effect.flatMap(({ branch: newBranch, exactName }) => - git.renameBranch({ - cwd: input.worktreePath, - oldBranch: input.oldBranch, - newBranch, - ...(exactName ? { exactName: true } : {}), - }), + git + .renameBranch({ + cwd: input.worktreePath, + oldBranch: input.oldBranch, + newBranch, + ...(exactName ? { exactName: true } : {}), + }) + .pipe( + Effect.tap((renamed) => Effect.sync(() => input.onRenamed?.(renamed.branch))), + Effect.uninterruptible, + ), ), Effect.flatMap((renamed) => Effect.gen(function* () { @@ -600,6 +606,7 @@ const make = Effect.gen(function* () { Option.Option<{ readonly branch: string; readonly exactName: boolean }> > | null = null; let createdWorktreePath: string | null = null; + let createdBranch: string | null = null; let setupTerminalId: string | null = null; let workspaceRecorded = false; let runReleased = false; @@ -730,6 +737,7 @@ const make = Effect.gen(function* () { onWorktreeClaimed: (path) => Effect.sync(() => { createdWorktreePath = path; + createdBranch = branch; }), onCheckoutProgress: (progress) => setupTracker.stage(threadId, "checkout", { percent: progress.percent }), @@ -740,6 +748,7 @@ const make = Effect.gen(function* () { worktreePath = worktree.worktree.path; branch = worktree.worktree.refName; createdWorktreePath = worktreePath; + createdBranch = branch; yield* setupTracker.update(threadId, (snapshot) => ({ ...snapshot, worktreePath, branch })); yield* setupTracker.stageStatus(threadId, "checkout", "done"); } @@ -765,6 +774,12 @@ const make = Effect.gen(function* () { ...(generated.exactName ? { exactName: true } : {}), }) .pipe( + Effect.tap((renamed) => + Effect.sync(() => { + createdBranch = renamed.branch; + }), + ), + Effect.uninterruptible, Effect.map((renamed) => renamed.branch), Effect.catchCause((cause) => Effect.logWarning("Thread worktree branch rename failed", { @@ -807,6 +822,9 @@ const make = Effect.gen(function* () { threadId, worktreePath, oldBranch: branch, + onRenamed: (renamed) => { + createdBranch = renamed; + }, generated: Fiber.join(lateBranchName).pipe( Effect.flatMap( Option.match({ @@ -963,10 +981,22 @@ const make = Effect.gen(function* () { if (branchRename !== null) yield* Fiber.interrupt(branchRename); if (branchNameFiber !== null) yield* Fiber.interrupt(branchNameFiber); const removedPath = createdWorktreePath; + const removedBranch = createdBranch; // Forget worktrees only after removal succeeds; failed removals must not reuse partially written checkouts. yield* git .removeWorktree({ cwd: project.workspaceRoot, path: removedPath, force: true }) .pipe( + Effect.andThen( + removedBranch === null + ? Effect.void + : git + .deleteLocalBranch({ + cwd: project.workspaceRoot, + refName: removedBranch, + force: true, + }) + .pipe(Effect.ignoreCause({ log: true })), + ), Effect.andThen( threads .dispatch({ diff --git a/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.ts b/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.ts index b3b8af0e64d96..00adbb0cb5445 100644 --- a/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.ts +++ b/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.ts @@ -95,7 +95,8 @@ export const makeThreadLiveEventCoalescer = Effect.fn( let windowFiber: Fiber.Fiber | null = null; let closed = false; - const cancelWindow = Effect.fn("ThreadLiveEventCoalescer.cancelWindow")(function* () { + // cancelWindow/flushPending/offerAll run per live event or batch; sub-ms spans here would flood the rotating trace. + const cancelWindow = Effect.fnUntraced(function* () { const fiber = windowFiber; if (!fiber) { return; @@ -104,7 +105,7 @@ export const makeThreadLiveEventCoalescer = Effect.fn( yield* Fiber.interrupt(fiber); }); - const flushPending = Effect.fn("ThreadLiveEventCoalescer.flushPending")(function* () { + const flushPending = Effect.fnUntraced(function* () { if (pendingUpdates.length === 0) { return; } @@ -135,9 +136,7 @@ export const makeThreadLiveEventCoalescer = Effect.fn( // Keep each source batch together so a synchronization marker cannot pass // events already pulled from PubSub but still being coalesced. - const offerAll = Effect.fn("ThreadLiveEventCoalescer.offerAll")(function* ( - inputs: ReadonlyArray, - ) { + const offerAll = Effect.fnUntraced(function* (inputs: ReadonlyArray) { yield* mutex.withPermits(1)( Effect.forEach( inputs, diff --git a/apps/server/src/orchestration-v2/ThreadPullRequestService.test.ts b/apps/server/src/orchestration-v2/ThreadPullRequestService.test.ts index 9c2ff9ef18f7a..423d1cddd943c 100644 --- a/apps/server/src/orchestration-v2/ThreadPullRequestService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadPullRequestService.test.ts @@ -28,6 +28,10 @@ import * as ProjectStore from "./ProjectStore.ts"; import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts"; import * as PullRequestService from "../pullRequest/PullRequestService.ts"; import * as ServerActivation from "../serverActivation.ts"; +import * as ProjectionStore from "./ProjectionStore.ts"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import { EventId } from "@t3tools/contracts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ThreadPullRequestService from "./ThreadPullRequestService.ts"; @@ -105,15 +109,13 @@ describe("ThreadPullRequestServiceV2 reads", () => { const dependencies = Layer.mergeAll( Layer.mock(Orchestrator.OrchestratorV2)({ streamDomainEvents: Stream.fromQueue(domainEvents), - getShellSnapshot: () => + getPullRequestSnapshot: () => Effect.succeed({ schemaVersion: 2, snapshotSequence: 1, threads: [thread], archivedThreads: [], }), - getThreadEventSequence: () => Effect.succeed(1), - getThreadShell: () => Effect.succeed(thread), }), Layer.mock(ProjectStore.ProjectStoreV2)({ listShells: () => Effect.succeed([project]), @@ -212,6 +214,8 @@ describe("ThreadPullRequestServiceV2 reads", () => { Effect.gen(function* () { const thread = { ...threadShell("worktree-thread"), + settledOverride: "settled" as const, + settledAt: NOW, branch: "feature/worktree", worktreePath: "/workspace/worktree", }; @@ -230,7 +234,7 @@ describe("ThreadPullRequestServiceV2 reads", () => { const dependencies = Layer.mergeAll( Layer.mock(Orchestrator.OrchestratorV2)({ streamDomainEvents: Stream.never, - getShellSnapshot: () => + getPullRequestSnapshot: () => Deferred.succeed(sweepStarted, undefined).pipe( Effect.as({ schemaVersion: 2, @@ -335,18 +339,17 @@ describe("ThreadPullRequestServiceV2 reads", () => { const dependencies = Layer.mergeAll( Layer.mock(Orchestrator.OrchestratorV2)({ streamDomainEvents: Stream.fromQueue(domainEvents), - getShellSnapshot: () => - Queue.offer(sweeps, undefined).pipe( - Effect.as({ - schemaVersion: 2, - snapshotSequence: 1, - threads: [parent, child], - archivedThreads: [], - }), - ), - getThreadEventSequence: () => Effect.succeed(1), - getThreadShell: (threadId) => - Effect.succeed([parent, child].find((thread) => thread.id === threadId) ?? null), + getPullRequestSnapshot: ({ threadId }) => + threadId !== undefined + ? Effect.succeed({ snapshotSequence: 1, threads: [child] }) + : Queue.offer(sweeps, undefined).pipe( + Effect.as({ + schemaVersion: 2, + snapshotSequence: 1, + threads: [parent, child], + archivedThreads: [], + }), + ), }), Layer.mock(ProjectStore.ProjectStoreV2)({ listShells: () => Effect.succeed([project]), @@ -457,14 +460,37 @@ describe("ThreadPullRequestServiceV2 reads", () => { const dependencies = Layer.mergeAll( Layer.mock(Orchestrator.OrchestratorV2)({ streamDomainEvents: Stream.never, - getShellSnapshot: () => - Deferred.succeed(sweepStarted, undefined).pipe( - Effect.as({ - schemaVersion: 2, - snapshotSequence: 1, - threads: [thread], - archivedThreads: [], - }), + getPullRequestSnapshot: (options) => + Effect.gen(function* () { + const store = yield* ProjectionStore.ProjectionStoreV2; + yield* store.apply({ + id: EventId.make(`event:${thread.id}:created`), + type: "thread.created", + threadId: thread.id, + occurredAt: NOW, + payload: { ...thread, lastVisitedAt: thread.lastVisitedAt ?? null }, + }); + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO orchestration_events ( + sequence, event_id, aggregate_kind, stream_id, stream_version, event_type, + occurred_at, actor_kind, payload_json, metadata_json, application_event_version + ) VALUES (7, 'pr-discovery', 'thread', ${thread.id}, 1, 'thread.created', + '2026-09-20T00:00:00.000Z', 'system', '{}', '{}', 2)`; + for (const table of [ + "orchestration_v2_projection_turn_items", + "orchestration_v2_projection_messages", + "orchestration_v2_projection_provider_threads", + "orchestration_v2_projection_runs", + ]) + yield* sql.unsafe(`DROP TABLE ${table}`); + const snapshot = yield* store.getPullRequestSnapshot(options); + yield* Deferred.succeed(sweepStarted, undefined); + return snapshot; + }).pipe( + Effect.provide( + ProjectionStore.layer.pipe(Layer.provideMerge(SqlitePersistence.layerMemory)), + ), + Effect.orDie, ), dispatch: (command) => Effect.sync(() => { @@ -529,6 +555,14 @@ describe("ThreadPullRequestServiceV2 reads", () => { expect(commands[0]).toMatchObject({ type: "thread.pull-request.sync", threadId: thread.id, + snapshotSequence: 7, + expected: { + workspaceRoot: project.workspaceRoot, + branch: thread.branch, + worktreePath: thread.worktreePath, + linkedPullRequest: savedReference, + branchPullRequest: savedReference, + }, branchPullRequest: null, }); expect(commands[0]).not.toHaveProperty("linkedPullRequest"); diff --git a/apps/server/src/orchestration-v2/ThreadPullRequestService.ts b/apps/server/src/orchestration-v2/ThreadPullRequestService.ts index e221388a5b8c0..1cb01acd4ed9c 100644 --- a/apps/server/src/orchestration-v2/ThreadPullRequestService.ts +++ b/apps/server/src/orchestration-v2/ThreadPullRequestService.ts @@ -7,7 +7,6 @@ import { type ThreadLinkedPullRequest, } from "@t3tools/contracts"; import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; -import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; @@ -120,19 +119,9 @@ export const make = Effect.gen(function* () { }; const readThreadSnapshot = ({ threadId, backfill }: RefreshRequest) => - threadId === null - ? orchestrator - .getShellSnapshot({ - location: "active", - unsettledOnly: !(backfill || pendingBackfill.size > 0), - }) - .pipe(NodeSqliteClient.readOnly) - : Effect.gen(function* () { - // Read sequence first so guarded sync rejects intervening changes. - const snapshotSequence = yield* orchestrator.getThreadEventSequence(threadId); - const thread = yield* orchestrator.getThreadShell(threadId); - return { snapshotSequence, threads: thread === null ? [] : [thread] }; - }); + orchestrator.getPullRequestSnapshot( + threadId === null ? { unsettledOnly: !(backfill || pendingBackfill.size > 0) } : { threadId }, + ); const synchronize = Effect.fn("ThreadPullRequestServiceV2.synchronize")(function* ( request: RefreshRequest, diff --git a/apps/server/src/orchestration-v2/ThreadTitleRegenerationService.test.ts b/apps/server/src/orchestration-v2/ThreadTitleRegenerationService.test.ts index 8f7de819f5718..c54e6e1f4815d 100644 --- a/apps/server/src/orchestration-v2/ThreadTitleRegenerationService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadTitleRegenerationService.test.ts @@ -1,4 +1,5 @@ import { assert, describe, it, vi } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; import { CommandId, ChatAttachmentId, @@ -14,6 +15,7 @@ import { } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences"; @@ -98,6 +100,7 @@ function makeHarness( layerProjectedProjects, Layer.mock(TextGeneration.TextGeneration)({ generateThreadTitle }), layerServerSettings, + NodeServices.layer, ), ), ); @@ -113,7 +116,11 @@ function makeHarness( }; } -function createThread(input: { readonly command: string; readonly thread: string }) { +function createThread(input: { + readonly command: string; + readonly thread: string; + readonly worktreePath?: string; +}) { return Effect.gen(function* () { const threads = yield* ThreadManagement.ThreadManagementService; const threadId = ThreadId.make(input.thread); @@ -127,7 +134,7 @@ function createThread(input: { readonly command: string; readonly thread: string runtimeMode: "full-access", interactionMode: "default", branch: null, - worktreePath: null, + worktreePath: input.worktreePath ?? null, createdBy: "user", creationSource: "web", }); @@ -391,31 +398,43 @@ describe("ThreadTitleRegenerationService", () => { const harness = makeHarness({ generateTitle: () => Effect.succeed({ title: "Fresh title" }), }); + const fs = yield* FileSystem.FileSystem; + const existingWorktree = yield* fs.makeTempDirectoryScoped({ prefix: "title-worktree-" }); yield* Effect.gen(function* () { const threads = yield* ThreadManagement.ThreadManagementService; const titleRegeneration = yield* ThreadTitleRegeneration.ThreadTitleRegenerationService; - const threadId = yield* createThread({ - command: "command:title:landing:create", - thread: "thread:title:landing", - }); - yield* dispatchUserMessage({ - command: "command:title:landing:message", - threadId, - text: "Investigate the flaky login test", - }); - const requestId = yield* armRegeneration({ command: "command:title:landing:1", threadId }); + const cases = [ + { name: "removed", worktreePath: "/repo-worktrees/removed-by-cleanup", cwd: "/repo" }, + { name: "existing", worktreePath: existingWorktree, cwd: existingWorktree }, + ]; + for (const [index, testCase] of cases.entries()) { + const threadId = yield* createThread({ + command: `command:title:landing:${testCase.name}:create`, + thread: `thread:title:landing:${testCase.name}`, + worktreePath: testCase.worktreePath, + }); + yield* dispatchUserMessage({ + command: `command:title:landing:${testCase.name}:message`, + threadId, + text: "Investigate the flaky login test", + }); + const requestId = yield* armRegeneration({ + command: `command:title:landing:${testCase.name}:1`, + threadId, + }); - yield* titleRegeneration.execute({ threadId, requestId, kind: { type: "regenerate" } }); + yield* titleRegeneration.execute({ threadId, requestId, kind: { type: "regenerate" } }); - const call = harness.generateThreadTitle.mock.calls[0]?.[0]; - assert.equal(call?.previousTitle, "Seed title"); - assert.equal(call?.cwd, "/repo"); - assert.include(call?.message, "USER:\nInvestigate the flaky login test"); - const projection = yield* threads.getThreadProjection(threadId); - assert.equal(projection.thread.title, "Fresh title"); - assert.isNotOk(projection.thread.titleRegeneration); + const call = harness.generateThreadTitle.mock.calls[index]?.[0]; + assert.equal(call?.previousTitle, "Seed title"); + assert.equal(call?.cwd, testCase.cwd); + assert.include(call?.message, "USER:\nInvestigate the flaky login test"); + const projection = yield* threads.getThreadProjection(threadId); + assert.equal(projection.thread.title, "Fresh title"); + assert.isNotOk(projection.thread.titleRegeneration); + } }).pipe(Effect.provide(harness.layer)); - }), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("titles a fork's first message with the source history up to the fork point", () => diff --git a/apps/server/src/orchestration-v2/ThreadTitleRegenerationService.ts b/apps/server/src/orchestration-v2/ThreadTitleRegenerationService.ts index 601b783675d0e..6c80def39059f 100644 --- a/apps/server/src/orchestration-v2/ThreadTitleRegenerationService.ts +++ b/apps/server/src/orchestration-v2/ThreadTitleRegenerationService.ts @@ -13,6 +13,7 @@ import { import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Schedule from "effect/Schedule"; @@ -50,6 +51,7 @@ const make = Effect.gen(function* () { const projects = yield* ProjectStore.ProjectStoreV2; const serverSettings = yield* ServerSettings.ServerSettingsService; const textGeneration = yield* TextGeneration.TextGeneration; + const fileSystem = yield* FileSystem.FileSystem; const complete = (input: { readonly threadId: ThreadId; @@ -160,8 +162,13 @@ const make = Effect.gen(function* () { yield* serverSettings.getSettings, projection.thread.projectId, ).settings; + // Storage cleanup removes worktrees but keeps the path; CLIs cannot spawn in a missing cwd. + const worktreePath = projection.thread.worktreePath; + const worktreeExists = + worktreePath !== null && + (yield* fileSystem.exists(worktreePath).pipe(Effect.orElseSucceed(() => false))); const result = yield* textGeneration.generateThreadTitle({ - cwd: projection.thread.worktreePath ?? project.value.workspaceRoot, + cwd: worktreeExists ? worktreePath : project.value.workspaceRoot, message: context.message, attachments: context.attachments, ...(input.kind.type === "regenerate" ? { previousTitle: projection.thread.title } : {}), diff --git a/apps/server/src/orchestration-v2/TurnItemCompaction.test.ts b/apps/server/src/orchestration-v2/TurnItemCompaction.test.ts deleted file mode 100644 index 5799c5fbdcd31..0000000000000 --- a/apps/server/src/orchestration-v2/TurnItemCompaction.test.ts +++ /dev/null @@ -1,488 +0,0 @@ -import { applyOrchestrationV2ProjectionEvent } from "../../../../packages/client-runtime/src/state/orchestrationV2Projection.ts"; -import { assert, it } from "@effect/vitest"; -import { - CommandId, - EventId, - NodeId, - PlanId, - type OrchestrationV2AppThread, - type OrchestrationV2DomainEvent, - type OrchestrationV2TurnItem, - ProjectId, - ProviderDriverKind, - ProviderInstanceId, - ThreadId, - TurnItemId, -} from "@t3tools/contracts"; -import * as DateTime from "effect/DateTime"; -import * as Deferred from "effect/Deferred"; -import * as Effect from "effect/Effect"; -import * as Fiber from "effect/Fiber"; -import * as Layer from "effect/Layer"; -import * as Stream from "effect/Stream"; -import * as Tracer from "effect/Tracer"; -import * as SqlClient from "effect/sql/SqlClient"; -import * as Statement from "effect/sql/Statement"; - -import * as SqlitePersistence from "../persistence/Sqlite.ts"; -import * as EventSink from "./EventSink.ts"; -import * as EventStore from "./EventStore.ts"; -import * as ProjectionMaintenance from "./ProjectionMaintenance.ts"; -import * as ProjectionStore from "./ProjectionStore.ts"; - -const database = SqlitePersistence.layerMemory; -const eventStoreProvided = EventStore.layer.pipe(Layer.provideMerge(database)); -const projectionStoreProvided = ProjectionStore.layer.pipe(Layer.provideMerge(database)); -const stores = Layer.mergeAll(database, eventStoreProvided, projectionStoreProvided); -const layerTest = Layer.mergeAll( - stores, - EventSink.layer.pipe(Layer.provide(stores)), - ProjectionMaintenance.layer.pipe(Layer.provide(stores)), -); - -const providerInstanceId = ProviderInstanceId.make("codex"); -const providerDriver = ProviderDriverKind.make("codex"); -const modelSelection = { instanceId: providerInstanceId, model: "gpt-5.4" }; - -function makeThread(threadId: ThreadId, now: DateTime.Utc): OrchestrationV2AppThread { - return { - id: threadId, - projectId: ProjectId.make(`project:${threadId}`), - title: "Turn item compaction", - providerInstanceId, - modelSelection, - runtimeMode: "full-access", - interactionMode: "default", - branch: null, - worktreePath: null, - activeProviderThreadId: null, - lineage: { rootThreadId: threadId, parentThreadId: null, relationshipToParent: null }, - forkedFrom: null, - createdBy: "user", - creationSource: "web", - createdAt: now, - updatedAt: now, - archivedAt: null, - settledOverride: null, - settledAt: null, - lastVisitedAt: null, - deletedAt: null, - }; -} - -function threadCreated(thread: OrchestrationV2AppThread, now: DateTime.Utc) { - return { - id: EventId.make("event:turn-item-compaction:thread"), - type: "thread.created", - threadId: thread.id, - providerInstanceId, - occurredAt: now, - payload: thread, - } satisfies OrchestrationV2DomainEvent; -} - -const itemSpecs = [ - { key: "command", type: "command_execution", ordinal: 10 }, - { key: "reasoning", type: "reasoning", ordinal: 20 }, - { key: "todo", type: "todo_list", ordinal: 30 }, - { key: "commanded", type: "command_execution", ordinal: 40 }, -] as const; - -function itemEvent( - threadId: ThreadId, - now: DateTime.Utc, - spec: (typeof itemSpecs)[number], - revision: number, - eventId = `event:turn-item-compaction:${spec.key}:${revision}`, -): OrchestrationV2DomainEvent { - const id = TurnItemId.make(`turn-item:compaction:${spec.key}`); - const growing = `${spec.key}:${revision}:` + "x".repeat((revision + 1) * 192); - const completed = revision === 69; - const base = { - id, - threadId, - runId: null, - nodeId: null, - providerThreadId: null, - providerTurnId: null, - nativeItemRef: null, - parentItemId: null, - ordinal: spec.ordinal, - status: completed ? ("completed" as const) : ("running" as const), - title: spec.key, - startedAt: now, - completedAt: completed ? now : null, - updatedAt: now, - }; - let payload: OrchestrationV2TurnItem; - switch (spec.type) { - case "command_execution": - payload = { - ...base, - type: "command_execution", - input: `run ${spec.key}`, - output: growing, - }; - break; - case "reasoning": - payload = { ...base, type: "reasoning", text: growing, streaming: !completed }; - break; - case "todo_list": - payload = { - ...base, - type: "todo_list", - planId: PlanId.make("plan:turn-item-compaction"), - steps: [{ id: "step", text: growing, status: completed ? "completed" : "running" }], - explanation: growing, - }; - break; - } - return { - id: EventId.make(eventId), - type: "turn-item.updated", - threadId, - occurredAt: now, - payload, - }; -} - -function subagentEvents( - threadId: ThreadId, - now: DateTime.Utc, - revision: number, -): ReadonlyArray { - const id = NodeId.make("subagent:turn-item-compaction"); - const prompt = revision === 0 ? "Review prompt A. ".repeat(40) : "Review prompt B. ".repeat(40); - const progress = `Review progress ${revision}`; - return [ - { - id: EventId.make(`event:turn-item-compaction:subagent:${revision}`), - type: "subagent.updated", - threadId, - occurredAt: now, - payload: { - id, - threadId, - runId: null, - parentNodeId: NodeId.make("node:turn-item-compaction-parent"), - origin: "provider_native", - createdBy: "agent", - driver: providerDriver, - providerInstanceId, - providerThreadId: null, - childThreadId: null, - nativeTaskRef: { - driver: providerDriver, - nativeId: "compaction-review", - strength: "strong", - }, - prompt, - progress, - title: "Review", - model: null, - status: "running", - result: null, - startedAt: now, - completedAt: null, - updatedAt: now, - }, - }, - { - id: EventId.make(`event:turn-item-compaction:subagent-item:${revision}`), - type: "turn-item.updated", - threadId, - occurredAt: now, - payload: { - id: TurnItemId.make("turn-item:compaction:subagent"), - threadId, - runId: null, - nodeId: null, - providerThreadId: null, - providerTurnId: null, - nativeItemRef: null, - parentItemId: null, - ordinal: 50, - status: "running", - title: "Review", - startedAt: now, - completedAt: null, - updatedAt: now, - type: "subagent", - subagentId: id, - origin: "provider_native", - driver: providerDriver, - providerInstanceId, - childThreadId: null, - prompt, - progress, - result: null, - }, - }, - ] satisfies ReadonlyArray; -} - -it.effect( - "compacts ordinary snapshots while preserving replay, prompts, command lookup, and positions", - () => - Effect.gen(function* () { - const sink = yield* EventSink.EventSinkV2; - const eventStore = yield* EventStore.EventStoreV2; - const projections = yield* ProjectionStore.ProjectionStoreV2; - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; - const sql = yield* SqlClient.SqlClient; - const now = yield* DateTime.now; - const threadId = ThreadId.make("thread:turn-item-compaction"); - const thread = makeThread(threadId, now); - const writes = [...(yield* sink.write({ events: [threadCreated(thread, now)] }))]; - const commandId = CommandId.make("command:turn-item-compaction:middle"); - const snapshotsPerItem = 70; - for (let revision = 0; revision < snapshotsPerItem; revision += 1) { - const order = - revision === 0 - ? [itemSpecs[0], itemSpecs[1], itemSpecs[2], itemSpecs[3]] - : [itemSpecs[1], itemSpecs[0], itemSpecs[2], itemSpecs[3]]; - const events = order - .filter((spec) => spec.key !== "commanded" || revision !== 35) - .map((spec) => itemEvent(threadId, now, spec, revision)); - if (revision < 3) events.push(...subagentEvents(threadId, now, revision)); - const stored = yield* sink.write({ events }); - writes.push(...stored); - if (revision === 35) { - const middle = itemEvent( - threadId, - now, - itemSpecs[3], - revision, - "event:turn-item-compaction:commanded:middle", - ); - writes.push(...(yield* sink.write({ commandId, events: [middle] }))); - } - } - const beforeProjection = yield* projections.getThreadProjection(threadId); - const highWater = yield* eventStore.latestSequence(); - const bytes = yield* sql<{ readonly rows: number; readonly payload_bytes: number }>` - SELECT COUNT(*) AS rows, COALESCE(SUM(octet_length(payload_json)), 0) AS payload_bytes - FROM orchestration_events WHERE stream_id = ${threadId} - `; - const summary = yield* maintenance.compactEventStore(); - const replay = yield* eventStore.read({ threadId }).pipe(Stream.runCollect); - const expectedIds = new Set([ - "event:turn-item-compaction:thread", - ...itemSpecs.flatMap((spec) => [ - `event:turn-item-compaction:${spec.key}:0`, - `event:turn-item-compaction:${spec.key}:${snapshotsPerItem - 1}`, - ]), - "event:turn-item-compaction:commanded:middle", - ...Array.from({ length: 3 }, (_, revision) => [ - `event:turn-item-compaction:subagent:${revision}`, - `event:turn-item-compaction:subagent-item:${revision}`, - ]).flat(), - ]); - assert.deepEqual( - replay, - writes.filter((stored) => expectedIds.has(stored.event.id)), - ); - assert.lengthOf(replay, 16); - assert.equal(summary.deletedEventCount, bytes[0]!.rows - replay.length); - assert.isBelow( - (yield* sql<{ readonly payload_bytes: number }>` - SELECT COALESCE(SUM(octet_length(payload_json)), 0) AS payload_bytes - FROM orchestration_events WHERE stream_id = ${threadId} - `)[0]!.payload_bytes, - bytes[0]!.payload_bytes, - ); - assert.equal(yield* eventStore.latestSequence(), highWater); - - const commanded = yield* sink.readByCommandId({ commandId }).pipe(Stream.runCollect); - assert.deepEqual( - commanded.map((stored) => stored.event.id), - [EventId.make("event:turn-item-compaction:commanded:middle")], - ); - const storedPrompts = yield* sql<{ - readonly event_id: string; - readonly sequence: number; - readonly prompt: string; - readonly prompt_sequence: number | null; - }>` - SELECT event_id, sequence, - json_extract(payload_json, '$.prompt') AS prompt, - json_extract(payload_json, '$.promptSequence') AS prompt_sequence - FROM orchestration_events - WHERE stream_id = ${threadId} AND event_type = 'subagent.updated' - ORDER BY sequence ASC - `; - assert.lengthOf(storedPrompts, 3); - assert.equal(storedPrompts[0]!.prompt_sequence, null); - assert.equal(storedPrompts[1]!.prompt_sequence, null); - assert.equal(storedPrompts[2]!.prompt, ""); - assert.equal(storedPrompts[2]!.prompt_sequence, storedPrompts[1]!.sequence); - - const deletedCursor = writes.find( - (stored) => stored.event.id === "event:turn-item-compaction:command:35", - )!.sequence; - assert.isFalse(replay.some((stored) => stored.sequence === deletedCursor)); - const cursors = [0, deletedCursor, highWater]; - for (const cursor of cursors) { - const cursorReplay = yield* eventStore - .read({ threadId, afterSequence: cursor, throughSequence: highWater }) - .pipe(Stream.runCollect); - const expected = replay.filter((stored) => stored.sequence > cursor); - assert.deepEqual( - cursorReplay.map((stored) => stored.sequence), - expected.map((stored) => stored.sequence), - ); - - let clientProjection: typeof beforeProjection | null = { - ...beforeProjection, - subagents: [], - turnItems: [], - visibleTurnItems: [], - }; - let latestLocalTurnOrdinal: number | null = null; - for (const stored of [ - ...writes.filter((entry) => entry.sequence <= cursor), - ...cursorReplay, - ]) { - clientProjection = applyOrchestrationV2ProjectionEvent(clientProjection, stored.event, { - partialTimeline: true, - latestLocalTurnOrdinal, - }); - if (stored.event.type === "turn-item.updated") { - latestLocalTurnOrdinal = Math.max( - latestLocalTurnOrdinal ?? 0, - stored.event.payload.ordinal, - ); - } - } - assert.deepEqual(clientProjection?.turnItems, beforeProjection.turnItems); - assert.deepEqual(clientProjection?.visibleTurnItems, beforeProjection.visibleTurnItems); - } - assert.isTrue((yield* maintenance.verify).valid); - assert.isTrue((yield* maintenance.rebuild).valid); - assert.deepEqual(yield* projections.getThreadProjection(threadId), beforeProjection); - const positions = yield* sql<{ - readonly turn_item_id: string; - readonly ordinal: number; - }>` - SELECT turn_item_id, ordinal - FROM orchestration_v2_turn_item_positions - WHERE thread_id = ${threadId} - ORDER BY ordinal ASC - `; - assert.deepEqual( - positions.map((row) => [row.turn_item_id, row.ordinal]), - [ - ...itemSpecs.map((spec, index) => [`turn-item:compaction:${spec.key}`, index + 1]), - ["turn-item:compaction:subagent", 5], - ], - ); - }).pipe(Effect.provide(layerTest)), -); - -it.effect("keeps post-bound appends visible during compaction", () => - Effect.scoped( - Effect.gen(function* () { - const sink = yield* EventSink.EventSinkV2; - const eventStore = yield* EventStore.EventStoreV2; - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; - const now = yield* DateTime.now; - const threadId = ThreadId.make("thread:turn-item-compaction-race"); - const thread = makeThread(threadId, now); - const initial = yield* sink.write({ - events: [ - threadCreated(thread, now), - ...[0, 1, 2].map((revision) => itemEvent(threadId, now, itemSpecs[0], revision)), - ], - }); - const beforeSequence = initial.at(-1)!.sequence; - - const streamReady = yield* Deferred.make(); - const streamTracer = Tracer.make({ - span(options) { - const span = new Tracer.NativeSpan(options); - const end = span.end.bind(span); - span.end = (endTime, exit) => { - end(endTime, exit); - const query = span.attributes.get("db.query.text"); - if (typeof query === "string" && query.includes("MAX(sequence)")) { - Deferred.doneUnsafe(streamReady, Effect.void); - } - }; - return span; - }, - }); - const liveEvents = yield* sink - .stream({ threadId, afterSequence: beforeSequence }) - .pipe( - Stream.take(3), - Stream.runCollect, - Effect.withTracer(streamTracer), - Effect.forkScoped, - ); - yield* Deferred.await(streamReady); - - const scanStarted = yield* Deferred.make(); - const releaseScan = yield* Deferred.make(); - let blocked = false; - const blockFirstScanPage: Statement.Transformer = (statement) => { - const [query] = statement.compile(); - if ( - !blocked && - query.includes("FROM orchestration_events AS event") && - query.includes("ORDER BY event.sequence DESC") - ) { - blocked = true; - return Deferred.succeed(scanStarted, undefined).pipe( - Effect.andThen(Deferred.await(releaseScan)), - Effect.as(statement), - ); - } - return Effect.succeed(statement); - }; - const compaction = yield* maintenance - .compactEventStore() - .pipe( - Effect.provideService(Statement.CurrentTransformer, blockFirstScanPage), - Effect.forkScoped, - ); - yield* Deferred.await(scanStarted); - const snapshots = [3, 4, 5].map((revision) => - itemEvent( - threadId, - now, - itemSpecs[0], - revision, - `event:turn-item-compaction-race:${revision}`, - ), - ); - const appended = yield* sink.write({ events: snapshots }); - yield* Deferred.succeed(releaseScan, undefined); - assert.equal((yield* Fiber.join(compaction)).deletedEventCount, 1); - const streamed = yield* Fiber.join(liveEvents); - assert.deepEqual( - streamed.map((stored) => stored.sequence), - appended.map((stored) => stored.sequence), - ); - assert.deepEqual( - streamed.map((stored) => stored.event.id), - snapshots.map((event) => event.id), - ); - assert.equal(yield* eventStore.latestSequence(), appended.at(-1)!.sequence); - const firstPassRows = yield* eventStore - .read({ threadId, eventType: "turn-item.updated" }) - .pipe(Stream.runCollect); - assert.deepEqual( - firstPassRows.map((stored) => stored.sequence), - [initial[1]!.sequence, beforeSequence, ...appended.map((stored) => stored.sequence)], - ); - yield* maintenance.compactEventStore(); - const secondPassRows = yield* eventStore - .read({ threadId, eventType: "turn-item.updated" }) - .pipe(Stream.runCollect); - assert.deepEqual( - secondPassRows.map((stored) => stored.sequence), - [initial[1]!.sequence, appended.at(-1)!.sequence], - ); - }), - ).pipe(Effect.provide(layerTest)), -); diff --git a/apps/server/src/orchestration-v2/TurnStartReads.test.ts b/apps/server/src/orchestration-v2/TurnStartReads.test.ts index 6db6b4c308827..cfc362c9db080 100644 --- a/apps/server/src/orchestration-v2/TurnStartReads.test.ts +++ b/apps/server/src/orchestration-v2/TurnStartReads.test.ts @@ -406,11 +406,30 @@ it.effect.each(["sqlite"] as const)( }, }, }); + const recordingId = "thread-completion-reads-0b9f6c1e-3c1d-4f7a-9d2e-5a6b7c8d9e0f-webm"; + yield* store.apply({ + id: EventId.make("attachment:recording-event"), + type: "turn-item.updated", + threadId, + runId: oldRunId, + occurredAt: now, + payload: { + ...item, + id: TurnItemId.make("attachment:recording"), + type: "dynamic_tool", + toolName: "mcp__t3-code__preview_recording_stop", + input: {}, + output: [ + { type: "text", text: JSON.stringify({ id: recordingId, mimeType: "video/webm" }) }, + ], + }, + }); assert.deepEqual((yield* store.getThreadAttachmentIds(threadId)).toSorted(), [ "answer-document", "answer-image", "attachment:old", "message-file", + recordingId, ]); }).pipe(Effect.provide(layerDatabase)), ); diff --git a/apps/server/src/orchestration-v2/legacy/LegacyV1Cutover.integration.test.ts b/apps/server/src/orchestration-v2/legacy/LegacyV1Cutover.integration.test.ts index 5d2343f6a1ea6..83c34c7edab17 100644 --- a/apps/server/src/orchestration-v2/legacy/LegacyV1Cutover.integration.test.ts +++ b/apps/server/src/orchestration-v2/legacy/LegacyV1Cutover.integration.test.ts @@ -42,7 +42,7 @@ import * as EventSink from "../EventSink.ts"; import * as EventStore from "../EventStore.ts"; import * as LegacyV1ThreadImporter from "./LegacyV1ThreadImporter.ts"; import * as Orchestrator from "../Orchestrator.ts"; -import * as ProjectionMaintenance from "../ProjectionMaintenance.ts"; +import * as ProjectionReplay from "../testkit/ProjectionReplay.ts"; import * as ProjectionStore from "../ProjectionStore.ts"; import * as ProviderAdapterRegistry from "../ProviderAdapterRegistry.ts"; import * as ProviderReplayHarness from "../testkit/ProviderReplayHarness.ts"; @@ -600,9 +600,7 @@ const layerBoot = (input: { const layerImporterProvided = LegacyV1ThreadImporter.layer.pipe( Layer.provide(Layer.mergeAll(layerStoresProvided, layerEventSinkProvided)), ); - const layerMaintenanceProvided = ProjectionMaintenance.layer.pipe( - Layer.provide(layerStoresProvided), - ); + const layerMaintenanceProvided = ProjectionReplay.layer.pipe(Layer.provide(layerStoresProvided)); const layerOrchestratorProvided = ProviderReplayHarness.layerWithRegistry( { name: input.name, @@ -676,7 +674,7 @@ describe("orchestration v2 legacy v1 cutover", () => { Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const importer = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter; - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const maintenance = yield* ProjectionReplay.ProjectionReplay; const projections = yield* ProjectionStore.ProjectionStoreV2; const orchestrator = yield* Orchestrator.OrchestratorV2; diff --git a/apps/server/src/orchestration-v2/legacy/LegacyV1ThreadImporter.test.ts b/apps/server/src/orchestration-v2/legacy/LegacyV1ThreadImporter.test.ts index 87059b7c1d0ae..a50b3ed3c1910 100644 --- a/apps/server/src/orchestration-v2/legacy/LegacyV1ThreadImporter.test.ts +++ b/apps/server/src/orchestration-v2/legacy/LegacyV1ThreadImporter.test.ts @@ -14,15 +14,16 @@ import * as Layer from "effect/Layer"; import * as SqlClient from "effect/sql/SqlClient"; import * as Statement from "effect/sql/Statement"; -import * as SqlitePersistence from "../../persistence/Sqlite.ts"; import { listLinkedPullRequestThreads } from "../../pullRequest/linkedThreads.ts"; import * as EventSink from "../EventSink.ts"; import * as EventStore from "../EventStore.ts"; import * as LegacyV1ThreadImporter from "./LegacyV1ThreadImporter.ts"; -import * as ProjectionMaintenance from "../ProjectionMaintenance.ts"; +import * as ProjectionReplay from "../testkit/ProjectionReplay.ts"; import * as ProjectionStore from "../ProjectionStore.ts"; -const layerDatabase = SqlitePersistence.layerMemory; +import { layerReaderDatabase, traceSqlRouting } from "../../persistence/testkit/SqliteRouting.ts"; + +const layerDatabase = layerReaderDatabase; const layerEventStoreProvided = EventStore.layer.pipe(Layer.provideMerge(layerDatabase)); const layerProjectionStoreProvided = ProjectionStore.layer.pipe(Layer.provideMerge(layerDatabase)); const layerStoresProvided = Layer.mergeAll( @@ -34,14 +35,14 @@ const layerEventSinkProvided = EventSink.layer.pipe(Layer.provide(layerStoresPro const layerImporterProvided = LegacyV1ThreadImporter.layer.pipe( Layer.provide(Layer.mergeAll(layerStoresProvided, layerEventSinkProvided)), ); -const layerProjectionMaintenanceProvided = ProjectionMaintenance.layer.pipe( +const layerProjectionReplayProvided = ProjectionReplay.layer.pipe( Layer.provide(layerStoresProvided), ); const layerTest = Layer.mergeAll( layerStoresProvided, layerEventSinkProvided, layerImporterProvided, - layerProjectionMaintenanceProvided, + layerProjectionReplayProvided, ); const seedPendingTranscripts = Effect.gen(function* () { @@ -80,6 +81,72 @@ const seedPendingTranscripts = Effect.gen(function* () { yield* importer.reconcileShells; }); +it.effect("routes shell selectors to readers while imports stay writable", () => + Effect.gen(function* () { + const importer = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter; + const sql = yield* SqlClient.SqlClient; + const { queries, trace } = traceSqlRouting(); + yield* seedPendingTranscripts.pipe(trace); + const selectors = queries.filter( + ({ query }) => + query.includes("FROM projection_threads AS thread") || + query.includes("FROM orchestration_v2_legacy_imports AS legacy_import") || + query.includes("FROM projection_thread_messages AS message"), + ); + assert.lengthOf(selectors, 6); + assert.deepEqual([...new Set(selectors.map((row) => row.connection))], ["reader"]); + assert.deepEqual(yield* importer.reconcileShells, { + importedThreadCount: 0, + importedMessageCount: 0, + }); + yield* sql.withTransaction( + Effect.gen(function* () { + yield* sql` + INSERT INTO projection_threads ( + thread_id, project_id, title, model_selection_json, runtime_mode, + interaction_mode, created_at, updated_at + ) VALUES ( + 'thread:uncommitted', 'project:background', 'Uncommitted legacy title', + '{"instanceId":"codex","model":"gpt-5.4"}', 'full-access', 'default', + '2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z' + ) + `; + yield* sql` + INSERT INTO projection_thread_messages ( + message_id, thread_id, role, text, is_streaming, created_at, updated_at + ) VALUES ( + 'message:uncommitted', 'thread:uncommitted', 'user', 'Uncommitted preview', 0, + '2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z' + ) + `; + yield* sql` + UPDATE orchestration_v2_projection_threads + SET payload_json = json_remove(payload_json, '$.pinnedAt') + WHERE thread_id = 'thread:first' + `; + yield* sql` + UPDATE projection_threads SET pinned_at = '2026-01-01T00:00:00.000Z' + WHERE thread_id = 'thread:first' + `; + const result = yield* importer.reconcileShells; + assert.deepEqual(result, { importedThreadCount: 2, importedMessageCount: 1 }); + const projections = yield* ProjectionStore.ProjectionStoreV2; + const repaired = yield* projections.getThreadProjection(ThreadId.make("thread:first")); + assert.equal(DateTime.formatIso(repaired.thread.pinnedAt!), "2026-01-01T00:00:00.000Z"); + const imported = yield* projections.getThreadProjection( + ThreadId.make("thread:uncommitted"), + ); + assert.equal(imported.messages[0]?.text, "Uncommitted preview"); + assert.equal( + (yield* projections.getThreadProjection(ThreadId.make("thread:uncommitted"))).thread + .title, + "Uncommitted legacy title", + ); + }), + ); + }).pipe(Effect.provide(layerTest)), +); + it.effect( "keeps hydrating later transcripts after a defect and leaves the failed thread pending", () => @@ -220,7 +287,7 @@ it.layer(layerTest)("LegacyV1ThreadImporter", (it) => { Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const importer = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter; - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const maintenance = yield* ProjectionReplay.ProjectionReplay; const projections = yield* ProjectionStore.ProjectionStoreV2; const eventSink = yield* EventSink.EventSinkV2; const threadId = ThreadId.make("thread:legacy-import"); @@ -605,7 +672,7 @@ it.layer(layerTest)("LegacyV1ThreadImporter", (it) => { Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const importer = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter; - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const maintenance = yield* ProjectionReplay.ProjectionReplay; const projections = yield* ProjectionStore.ProjectionStoreV2; const eventSink = yield* EventSink.EventSinkV2; const threadId = ThreadId.make("thread:legacy-metadata-upgrade"); diff --git a/apps/server/src/orchestration-v2/legacy/LegacyV1ThreadImporter.ts b/apps/server/src/orchestration-v2/legacy/LegacyV1ThreadImporter.ts index d82a9e80a23b5..8c134d3b57d71 100644 --- a/apps/server/src/orchestration-v2/legacy/LegacyV1ThreadImporter.ts +++ b/apps/server/src/orchestration-v2/legacy/LegacyV1ThreadImporter.ts @@ -394,7 +394,7 @@ const make = Effect.gen(function* () { AND message.role IN ('user', 'assistant') ORDER BY message.created_at DESC, message.message_id DESC LIMIT 1 - `; + `.pipe(NodeSqliteClient.readOnly); const latestUser = yield* sql` SELECT message.message_id, @@ -424,7 +424,7 @@ const make = Effect.gen(function* () { AND message.role = 'user' ORDER BY message.created_at DESC, message.message_id DESC LIMIT 1 - `; + `.pipe(NodeSqliteClient.readOnly); return [latestUser[0], latest[0]].filter( (message, index, selected): message is LegacyMessageRow => message !== undefined && @@ -476,7 +476,7 @@ const make = Effect.gen(function* () { OR json_type(projection.payload_json, '$.branchPullRequest') IS NULL OR json_type(projection.payload_json, '$.activeOrderKey') IS NULL ORDER BY thread.created_at ASC, thread.thread_id ASC - `; + `.pipe(NodeSqliteClient.readOnly); let repairedThreadCount = 0; for (const row of repairRows) { const decoded = decodeStoredThread(row.payload_json); @@ -574,7 +574,7 @@ const make = Effect.gen(function* () { AND event.event_type = 'thread.created' ) ORDER BY thread.created_at ASC, thread.thread_id ASC - `; + `.pipe(NodeSqliteClient.readOnly); let importedThreadCount = repairedThreadCount; let importedMessageCount = 0; for (const row of rows) { diff --git a/apps/server/src/orchestration-v2/pullRequestWatch.test.ts b/apps/server/src/orchestration-v2/pullRequestWatch.test.ts index 37c05895602d4..bdd64c3a77747 100644 --- a/apps/server/src/orchestration-v2/pullRequestWatch.test.ts +++ b/apps/server/src/orchestration-v2/pullRequestWatch.test.ts @@ -7,9 +7,9 @@ import { assert, describe, it } from "@effect/vitest"; import { PULL_REQUEST_WATCH_FOLLOW_UP_LIMIT } from "@t3tools/shared/pullRequestWatch"; import { - pullRequestWatchMessage, PULL_REQUEST_WATCH_WAKE_LIMIT, evaluatePullRequestWatch as evaluate, + pullRequestWatchMessage, } from "./pullRequestWatch.ts"; // These cases read the head once or at one moment, so the no-checks notice never comes due. @@ -337,10 +337,31 @@ describe("evaluatePullRequestWatch", () => { assert.deepEqual(evaluatePullRequestWatch(report.next, detail(), comments).changes, []); }); + it("cuts a long remark in the wake message without splitting an emoji", () => { + const body = `${"x".repeat(196)}🚀 and more`; + const report = evaluatePullRequestWatch(watch(), detail(), [ + remark("reviewer", "2026-10-02T12:06:00Z", body), + ]); + const { text } = pullRequestWatchMessage({ + host: "github.com", + number: 1, + url: "https://github.com/o/r/pull/1", + baseBranch: "main", + headSha: null, + report, + }); + assert.isTrue(text.isWellFormed()); + assert.include(text, `"${"x".repeat(196)}..."`); + }); + it("reports a conflict once, until the branch is clean again", () => { const conflicting = detail({ mergeability: "conflicting" }); const first = evaluatePullRequestWatch(watch(), conflicting, noRemarks); assert.deepEqual(first.changes, [{ kind: "conflicting" }]); + const { headSha: _unreadHead, ...headUnread } = conflicting; + const unread = evaluatePullRequestWatch(first.next, headUnread, noRemarks); + assert.deepEqual(unread.changes, []); + assert.deepEqual(evaluatePullRequestWatch(unread.next, conflicting, noRemarks).changes, []); const recomputing = evaluatePullRequestWatch( first.next, detail({ mergeability: "unknown" }), @@ -354,6 +375,20 @@ describe("evaluatePullRequestWatch", () => { assert.deepEqual(evaluatePullRequestWatch(clean.next, conflicting, noRemarks).changes, [ { kind: "conflicting" }, ]); + const pushed = { headSha: "bbbbbbbbbb" }; + const recomputed = evaluatePullRequestWatch( + first.next, + detail({ ...pushed, mergeability: "unknown" }), + noRemarks, + ); + assert.deepEqual( + evaluatePullRequestWatch( + recomputed.next, + detail({ ...pushed, mergeability: "conflicting" }), + noRemarks, + ).changes, + [{ kind: "conflicting" }], + ); }); it("reports requested changes once, until the review decision clears", () => { diff --git a/apps/server/src/orchestration-v2/pullRequestWatch.ts b/apps/server/src/orchestration-v2/pullRequestWatch.ts index 27f07312a90ab..8c5a186add2b7 100644 --- a/apps/server/src/orchestration-v2/pullRequestWatch.ts +++ b/apps/server/src/orchestration-v2/pullRequestWatch.ts @@ -60,7 +60,8 @@ export function evaluatePullRequestWatch( now: number, ): PullRequestWatchReport { const changes: Array = []; - const headSha = detail.headSha ?? null; + // Azure DevOps omits the head when its iteration read fails; that is not a push. + const headSha = detail.headSha ?? watch.headSha; const headMoved = headSha !== watch.headSha; // Hosts can return no checks on read failure; preserve prior state. @@ -145,12 +146,14 @@ export function evaluatePullRequestWatch( ...atLatest.map((remark) => remark.id), ]; - if (detail.mergeability === "conflicting" && !watch.conflicting) { + // Like a failed check, a conflict a push did not resolve is news for the new head. + const toldConflict = !headMoved && watch.conflicting; + if (detail.mergeability === "conflicting" && !toldConflict) { changes.push({ kind: "conflicting" }); } // "unknown" is a host still computing after a push; only a clean answer clears a conflict. const conflicting = - detail.mergeability === "unknown" ? watch.conflicting : detail.mergeability === "conflicting"; + detail.mergeability === "unknown" ? toldConflict : detail.mergeability === "conflicting"; // Like a conflict, requested changes are reported once and re-arm when the decision clears. const changesRequested = detail.reviewDecision === "changes-requested"; @@ -195,7 +198,11 @@ function snippet(body: string): string { .replaceAll(//g, " ") .replaceAll(/\s+/g, " ") .trim(); - return text.length <= SNIPPET_LENGTH ? text : `${text.slice(0, SNIPPET_LENGTH - 3)}...`; + if (text.length <= SNIPPET_LENGTH) return text; + // Half an emoji serializes to a lone surrogate escape, which provider JSON parsers reject. + const last = text.charCodeAt(SNIPPET_LENGTH - 4); + const end = last >= 0xd800 && last <= 0xdbff ? SNIPPET_LENGTH - 4 : SNIPPET_LENGTH - 3; + return `${text.slice(0, end)}...`; } function listed(items: ReadonlyArray, line: (item: T) => string): Array { diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index abde59e10b362..0bcd2153b0243 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -69,7 +69,8 @@ import * as EffectWorker from "./EffectWorker.ts"; import * as EffectOutbox from "./EffectOutbox.ts"; import * as EventSink from "./EventSink.ts"; import * as ProviderRuntimeRecoveryService from "./ProviderRuntimeRecoveryService.ts"; -import * as ProjectionMaintenance from "./ProjectionMaintenance.ts"; +import * as EventStore from "./EventStore.ts"; +import * as ProjectionReplay from "./testkit/ProjectionReplay.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as PullRequestWatchReactor from "./PullRequestWatchReactor.ts"; import { importForkPullRequestWatches } from "./forkPullRequestWatchImport.ts"; @@ -279,8 +280,21 @@ const moveProject = (projectId: ProjectId, workspaceRoot: string, updatedAt: str }), ); +// Read-only oracle: projections still equal a replay of these unpruned test logs. +const layerProjectionReplayTest = ProjectionReplay.layer.pipe( + Layer.provide( + Layer.merge( + EventStore.layerFromOrchestrationEventStore.pipe( + Layer.provide(RuntimeLayer.layerEventInfrastructure), + ), + ProjectionStore.layer, + ), + ), +); + const layerTest = Layer.mergeAll( RuntimeLayer.layer, + layerProjectionReplayTest, RuntimeLayer.layerEventSink, ProjectStore.layer, ProjectionStore.layer, @@ -299,7 +313,7 @@ const layerTest = Layer.mergeAll( Layer.provide(layerPlatformTest), ); -const layerLegacyImportTest = RuntimeLayer.layer.pipe( +const layerLegacyImportTest = Layer.merge(RuntimeLayer.layer, layerProjectionReplayTest).pipe( Layer.provide(McpSessionRegistryTestkit.layer), Layer.provide(McpProviderSessions.layer), Layer.provideMerge(SqlitePersistence.layerMemory), @@ -821,6 +835,7 @@ it.layer(layerProjectDeletionTest)("project deletion during thread commands", (i }); const layerSharedApplicationDataPlaneTest = Layer.mergeAll( + layerProjectionReplayTest, RuntimeLayer.layer.pipe(Layer.provide(RuntimeLayer.layerProjectService)), RuntimeLayer.layerProjectService, RuntimeLayer.layerEventSink, @@ -2266,7 +2281,7 @@ it.layer(layerLegacyImportTest)("OrchestrationV2 legacy import", (it) => { Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const importer = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter; - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const maintenance = yield* ProjectionReplay.ProjectionReplay; const orchestrator = yield* Orchestrator.OrchestratorV2; const threadManagement = yield* ThreadManagementService.ThreadManagementService; const metadataThreadId = ThreadId.make("runtime-layer-legacy-metadata-thread"); @@ -2700,6 +2715,20 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { ); assert.deepEqual(activeShell?.unsettledAt, activeProjection.thread.unsettledAt); + yield* orchestrator.dispatch({ + type: "thread.settle", + commandId: CommandId.make("runtime-layer-lifecycle-resettle"), + threadId, + }); + yield* orchestrator.dispatch({ + type: "thread.pin", + commandId: CommandId.make("runtime-layer-lifecycle-pin-settled"), + threadId, + }); + const pinnedProjection = yield* orchestrator.getThreadProjection(threadId); + assert.equal(pinnedProjection.thread.settledOverride, "active"); + assert.isNotNull(pinnedProjection.thread.unsettledAt); + const archive = yield* orchestrator.dispatch({ type: "thread.archive", commandId: CommandId.make("runtime-layer-lifecycle-archive"), @@ -2781,7 +2810,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { it.effect("keeps the branch pull request when linking another pull request", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const maintenance = yield* ProjectionReplay.ProjectionReplay; const threadId = ThreadId.make("branch-pr-link"); const projectId = ProjectId.make("branch-pr-project"); yield* seedProject({ @@ -2890,7 +2919,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { it.effect("retains multiple pull requests and dismissed stack members through rebuilds", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const maintenance = yield* ProjectionReplay.ProjectionReplay; const threadId = ThreadId.make("runtime-multiple-pull-requests"); yield* orchestrator.dispatch({ type: "thread.create", @@ -2981,7 +3010,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { it.effect("starts, records, and stops a pull request watch", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const maintenance = yield* ProjectionReplay.ProjectionReplay; const threadId = ThreadId.make("runtime-pull-request-watch"); yield* orchestrator.dispatch({ type: "thread.create", @@ -3074,7 +3103,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { it.effect.each(["manual", "automatic"])("settling keeps pull request watches: %s", (mode) => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const maintenance = yield* ProjectionReplay.ProjectionReplay; const threadId = ThreadId.make(`pr-watch-settle-${mode}`); yield* orchestrator.dispatch({ type: "thread.create", @@ -4993,7 +5022,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { it.effect("cancels preparing work on archive and keeps it cancelled through replay", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const maintenance = yield* ProjectionReplay.ProjectionReplay; const threadId = ThreadId.make("runtime-layer-archive-preparing-thread"); yield* orchestrator.dispatch({ type: "thread.create", @@ -6561,7 +6590,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { assert.equal(queued.length, 2); yield* recovery.reconcile(trigger); yield* recovery.reconcile("startup"); - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const maintenance = yield* ProjectionReplay.ProjectionReplay; assert.isTrue((yield* maintenance.rebuild).valid); assert.equal(yield* orchestrator.resumeQueuedRuns, 0); const held = yield* orchestrator.getThreadProjection(threadId); @@ -7787,11 +7816,11 @@ it.layer(Layer.fresh(layerSharedApplicationDataPlaneTest))("thread opens", (it) ); }); -it.layer(Layer.fresh(layerSharedApplicationDataPlaneTest))("compact PR events", (it) => { +it.layer(Layer.fresh(layerSharedApplicationDataPlaneTest))("PR link events", (it) => { it.effect("persists only the changed PR link and replays it without losing other links", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const maintenance = yield* ProjectionReplay.ProjectionReplay; const sql = yield* SqlClient.SqlClient; const threadId = ThreadId.make("compact-pr-sync"); yield* orchestrator.dispatch({ @@ -7890,85 +7919,6 @@ it.layer(Layer.fresh(layerSharedApplicationDataPlaneTest))("compact PR events", (yield* orchestrator.getThreadShell(threadId))?.pullRequests, after?.pullRequests, ); - for (const [commandId, title] of [ - ["compact-pr-sync-again", "Still open PR"], - ["compact-pr-sync-merged", "Merged PR"], - ] as const) { - yield* orchestrator.dispatch({ - type: "thread.pull-request-link.sync", - commandId: CommandId.make(commandId), - threadId, - ...key, - number: 2, - snapshot: { - state: title === "Merged PR" ? "merged" : "open", - title, - headBranch: "feature/pr", - baseBranch: "main", - isDraft: false, - updatedAt: null, - syncedAt: "2026-10-13T13:00:00.000Z", - }, - stack: null, - }); - } - const latest = yield* orchestrator.getThreadShell(threadId); - yield* maintenance.compactEventStore(); - const linkEvents = yield* sql<{ command_id: string }>` - SELECT command_id FROM orchestration_events - WHERE stream_id = ${threadId} AND event_type = 'thread.pull-request-link-updated' - ORDER BY sequence - `; - assert.deepEqual( - linkEvents.map((row) => row.command_id), - ["compact-pr-sync", "compact-pr-sync-merged"], - ); - assert.isTrue((yield* maintenance.rebuild).valid); - assert.deepEqual( - (yield* orchestrator.getThreadShell(threadId))?.pullRequests, - latest?.pullRequests, - ); - - // Visits preserve updatedAt, so compaction must retain earlier renames; later renames supersede visits. - const threadEvents = () => sql<{ event_type: string }>` - SELECT event_type FROM orchestration_events - WHERE stream_id = ${threadId} - AND event_type IN ('thread.metadata-updated', 'thread.visited') - ORDER BY sequence - `; - yield* orchestrator.dispatch({ - type: "thread.metadata.update", - commandId: CommandId.make("compact-pr-rename"), - threadId, - title: "Renamed before visit", - }); - for (const [commandId, visitedAt] of [ - ["compact-pr-visit", "2026-10-13T14:00:00.000Z"], - ["compact-pr-visit-again", "2026-10-13T15:00:00.000Z"], - ] as const) { - yield* orchestrator.dispatch({ - type: "thread.visit", - commandId: CommandId.make(commandId), - threadId, - visitedAt, - }); - } - yield* maintenance.compactEventStore(); - assert.deepEqual( - (yield* threadEvents()).map((row) => row.event_type), - ["thread.metadata-updated", "thread.visited"], - ); - yield* orchestrator.dispatch({ - type: "thread.metadata.update", - commandId: CommandId.make("compact-pr-rename-again"), - threadId, - title: "Renamed after visit", - }); - yield* maintenance.compactEventStore(); - assert.deepEqual( - (yield* threadEvents()).map((row) => row.event_type), - ["thread.metadata-updated"], - ); }), ); }); diff --git a/apps/server/src/orchestration-v2/testkit/ProjectionReplay.ts b/apps/server/src/orchestration-v2/testkit/ProjectionReplay.ts new file mode 100644 index 0000000000000..437db15c375f8 --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/ProjectionReplay.ts @@ -0,0 +1,182 @@ +import { ThreadId } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import * as SqlClient from "effect/sql/SqlClient"; + +import * as EventStore from "../EventStore.ts"; +import * as ProjectionStore from "../ProjectionStore.ts"; + +export interface ProjectionReplayVerification { + readonly valid: boolean; + readonly schemaVersion: number; + readonly expectedSequence: number; + readonly projectionSequence: number; + readonly unreadableThreadIds: ReadonlyArray; + readonly missingThreadIds: ReadonlyArray; + readonly unexpectedThreadIds: ReadonlyArray; +} + +export class ProjectionReplayError extends Schema.TaggedError()( + "ProjectionReplayError", + { cause: Schema.Defect() }, +) {} + +/** + * Test oracle: projections must equal a replay of an unpruned event log. Production prunes + * events the projections cover, so it never replays (see ProjectionMaintenance). + */ +export class ProjectionReplay extends Context.Service< + ProjectionReplay, + { + readonly verify: Effect.Effect; + readonly rebuild: Effect.Effect; + } +>()("t3/orchestration-v2/testkit/ProjectionReplay") {} + +type ProjectionMetadataRow = { + readonly schema_version: number; + readonly last_sequence: number; +}; + +export const layer: Layer.Layer< + ProjectionReplay, + never, + EventStore.EventStoreV2 | ProjectionStore.ProjectionStoreV2 | SqlClient.SqlClient +> = Layer.effect( + ProjectionReplay, + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const eventStore = yield* EventStore.EventStoreV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + + const verify = Effect.gen(function* () { + const expectedThreadRows = yield* sql<{ readonly thread_id: string }>` + SELECT DISTINCT stream_id AS thread_id + FROM orchestration_events INDEXED BY orchestration_events_v2_created_threads_idx + WHERE application_event_version = 2 + AND aggregate_kind = 'thread' + AND event_type = 'thread.created' + ORDER BY stream_id ASC + `; + const projectionRows = yield* sql<{ readonly thread_id: string }>` + SELECT thread_id + FROM orchestration_v2_projection_threads + ORDER BY thread_id ASC + `; + const actualIds = projectionRows.map((row) => ThreadId.make(row.thread_id)); + const expectedIds = expectedThreadRows.map((row) => ThreadId.make(row.thread_id)); + const actualSet = new Set(actualIds); + const expectedSet = new Set(expectedIds); + const missingThreadIds = expectedIds.filter((threadId) => !actualSet.has(threadId)); + const unexpectedThreadIds = actualIds.filter((threadId) => !expectedSet.has(threadId)); + const unreadableThreadIds = yield* projectionStore.getUnreadableThreadIds(); + const metadata = yield* sql` + SELECT schema_version, last_sequence + FROM orchestration_v2_projection_metadata + WHERE projection_name = 'thread-projections' + LIMIT 1 + `; + const expectedSequence = yield* eventStore.latestSequence(); + const schemaVersion = metadata[0]?.schema_version ?? 0; + const projectionSequence = metadata[0]?.last_sequence ?? 0; + return { + valid: + schemaVersion === ProjectionStore.ORCHESTRATION_V2_PROJECTION_SCHEMA_VERSION && + projectionSequence === expectedSequence && + missingThreadIds.length === 0 && + unexpectedThreadIds.length === 0 && + unreadableThreadIds.length === 0, + schemaVersion, + expectedSequence, + projectionSequence, + unreadableThreadIds, + missingThreadIds, + unexpectedThreadIds, + } satisfies ProjectionReplayVerification; + }).pipe(sql.withTransaction); + + const rebuild = Effect.gen(function* () { + yield* sql.withTransaction( + Effect.gen(function* () { + const throughSequence = yield* eventStore.latestSequence(); + yield* sql`DELETE FROM orchestration_v2_projection_context_transfers`; + yield* sql`DELETE FROM orchestration_v2_projection_context_handoffs`; + yield* sql`DELETE FROM orchestration_v2_projection_checkpoints`; + yield* sql`DELETE FROM orchestration_v2_projection_checkpoint_scopes`; + yield* sql`DELETE FROM orchestration_v2_projection_turn_items`; + yield* sql`DELETE FROM orchestration_v2_projection_plans`; + yield* sql`DELETE FROM orchestration_v2_projection_messages`; + yield* sql`DELETE FROM orchestration_v2_projection_runtime_requests`; + yield* sql`DELETE FROM orchestration_v2_projection_provider_turns`; + yield* sql`DELETE FROM orchestration_v2_projection_provider_threads`; + yield* sql`DELETE FROM orchestration_v2_projection_provider_session_bindings`; + yield* sql`DELETE FROM orchestration_v2_projection_provider_sessions`; + yield* sql`DELETE FROM orchestration_v2_projection_subagents`; + yield* sql`DELETE FROM orchestration_v2_projection_nodes`; + yield* sql`DELETE FROM orchestration_v2_projection_run_attempts`; + yield* sql`DELETE FROM orchestration_v2_projection_runs`; + yield* sql`DELETE FROM orchestration_v2_projection_threads`; + yield* sql`DELETE FROM orchestration_v2_turn_item_positions`; + + const pageSize = 500; + let lastSequence = 0; + while (true) { + const page = yield* eventStore + .read({ afterSequence: lastSequence, throughSequence, limit: pageSize }) + .pipe(Stream.runCollect); + for (const stored of page) { + yield* projectionStore.apply(stored.event); + if (stored.event.type === "turn-item.updated") { + yield* sql` + INSERT INTO orchestration_v2_turn_item_positions ( + thread_id, + turn_item_id, + ordinal + ) + VALUES ( + ${stored.event.threadId}, + ${stored.event.payload.id}, + ${stored.event.payload.ordinal} + ) + ON CONFLICT(thread_id, turn_item_id) DO UPDATE SET + ordinal = excluded.ordinal + `; + } + lastSequence = stored.sequence; + } + if (page.length < pageSize) break; + yield* Effect.yieldNow; + } + const now = DateTime.formatIso(yield* DateTime.now); + yield* sql` + INSERT INTO orchestration_v2_projection_metadata ( + projection_name, + schema_version, + last_sequence, + updated_at + ) + VALUES ( + 'thread-projections', + ${ProjectionStore.ORCHESTRATION_V2_PROJECTION_SCHEMA_VERSION}, + ${lastSequence}, + ${now} + ) + ON CONFLICT(projection_name) DO UPDATE SET + schema_version = excluded.schema_version, + last_sequence = excluded.last_sequence, + updated_at = excluded.updated_at + `; + }), + ); + return yield* verify; + }); + + const mapError = (effect: Effect.Effect) => + effect.pipe(Effect.mapError((cause) => new ProjectionReplayError({ cause }))); + return ProjectionReplay.of({ verify: mapError(verify), rebuild: mapError(rebuild) }); + }), +); diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index f36e463079f99..629b3902e763d 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -52,7 +52,7 @@ import * as LegacyV1ThreadImporter from "../legacy/LegacyV1ThreadImporter.ts"; import * as Orchestrator from "../Orchestrator.ts"; import * as EffectWorker from "../EffectWorker.ts"; import * as EffectOutbox from "../EffectOutbox.ts"; -import * as ProjectionMaintenance from "../ProjectionMaintenance.ts"; +import * as ProjectionReplay from "./ProjectionReplay.ts"; import * as ProjectionStore from "../ProjectionStore.ts"; import * as ProviderAdapterRegistry from "../ProviderAdapterRegistry.ts"; import { makeProviderFailure } from "@t3tools/provider-core/server/failure"; @@ -2529,21 +2529,36 @@ describe("orchestration v2 provider switching", () => { // regression: #20343 sending to an instance with separate history must release the old session it.live.each([ - ["detaches the previous instance's session when a send hands off to another instance", false], - ["refuses the hand-off while the previous instance still runs background work", true], - ] as const)("%s", ([, backgroundRunning]) => + [ + "detaches the previous instance's session when a send hands off to another instance", + { background: false, queued: false, sharedHistory: false }, + ], + [ + "refuses the hand-off while the previous instance still runs background work", + { background: true, queued: false, sharedHistory: false }, + ], + [ + "keeps a busy previous session running when a queued hand-off starts", + { background: true, queued: true, sharedHistory: false }, + ], + [ + "holds a queued account switch while the replaced session still runs background work", + { background: true, queued: true, sharedHistory: true }, + ], + ] as const)("%s", ([, { background, queued, sharedHistory }]) => Effect.scoped( Effect.gen(function* () { const cwd = yield* checkpointWorkspace("handoff-releases-session"); const capturedTurns = yield* Ref.make>([]); - const backgroundWork = yield* Ref.make(backgroundRunning); + const backgroundWork = yield* Ref.make(background); const closedSessions = yield* Ref.make>([]); + const firstStarted = yield* Deferred.make(); + const releaseFirst = yield* Deferred.make(); const secondSelection: ModelSelection = { ...CLAUDE_MODEL_SELECTION, instanceId: ProviderInstanceId.make("claude-second-account"), }; - // No shared continuation key: each Claude instance owns its own native history. - const layerRegistry = ProviderAdapterRegistry.layerFromAdapters([ + const adapters = [ makeTestAdapter({ instanceId: CLAUDE_MODEL_SELECTION.instanceId, driver: CLAUDE_DRIVER, @@ -2553,6 +2568,7 @@ describe("orchestration v2 provider switching", () => { capturedTurns, backgroundWork, closedSessions, + ...(queued ? { holdFirstTurn: firstStarted, releaseFirstTurn: releaseFirst } : {}), }), makeTestAdapter({ instanceId: secondSelection.instanceId, @@ -2562,7 +2578,33 @@ describe("orchestration v2 provider switching", () => { responseByRunOrdinal: { 2: "Second account complete" }, capturedTurns, }), - ]); + ]; + // Without a shared continuation key, each Claude instance owns its own native history. + const layerRegistry = sharedHistory + ? Layer.succeed( + ProviderAdapterRegistry.ProviderAdapterRegistryV2, + ProviderAdapterRegistry.ProviderAdapterRegistryV2.of({ + get: (instanceId) => { + const adapter = adapters.find((candidate) => candidate.instanceId === instanceId); + return adapter === undefined + ? Effect.fail( + new ProviderAdapterRegistry.ProviderAdapterRegistryLookupError({ + instanceId, + }), + ) + : Effect.succeed(adapter); + }, + list: () => Effect.succeed(adapters.map((adapter) => adapter.instanceId)), + getMetadata: () => + Effect.succeed({ + driver: CLAUDE_DRIVER, + continuationKey: "claude:shared-native-account-history", + enabled: true, + capabilities: ClaudeProviderCapabilitiesV2, + }), + }), + ) + : ProviderAdapterRegistry.layerFromAdapters(adapters); const handoffThreadId = ThreadId.make("thread:handoff-releases-session"); yield* Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -2578,7 +2620,7 @@ describe("orchestration v2 provider switching", () => { text: `Prompt ${ordinal}`, attachments: [], modelSelection, - dispatchMode: { type: "start_immediately" }, + dispatchMode: { type: queued ? "queue_after_active" : "start_immediately" }, }); yield* orchestrator.dispatch({ type: "thread.create", @@ -2595,6 +2637,69 @@ describe("orchestration v2 provider switching", () => { worktreePath: cwd, }); yield* send(1, CLAUDE_MODEL_SELECTION); + + if (queued) { + yield* Deferred.await(firstStarted); + yield* send(2, secondSelection); + const queuedRunId = (yield* orchestrator.getThreadProjection(handoffThreadId)).runs[1] + ?.id; + assert.isDefined(queuedRunId); + yield* Deferred.succeed(releaseFirst, undefined); + yield* orchestrator.streamStoredEvents.pipe( + Stream.filter( + ({ event }) => + event.type === "run.updated" && + event.runId === queuedRunId && + (event.payload.queueHeld === true || event.payload.status === "completed"), + ), + Stream.runHead, + ); + yield* worker.drain(); + const afterQueue = yield* orchestrator.getThreadProjection(handoffThreadId); + const firstSession = afterQueue.providerSessions.find( + (session) => session.providerInstanceId === CLAUDE_MODEL_SELECTION.instanceId, + ); + // Detaching would end the background agents, so the busy session must stay up. + assert.deepEqual(yield* Ref.get(closedSessions), []); + assert.equal(firstSession?.status, "ready"); + if (!sharedHistory) { + assert.deepEqual( + afterQueue.runs.map((run) => [run.providerInstanceId, run.status]), + [ + [CLAUDE_MODEL_SELECTION.instanceId, "completed"], + [secondSelection.instanceId, "completed"], + ], + ); + return; + } + // Resuming the native thread beside the busy session would fork its history. + assert.deepEqual( + afterQueue.runs.map((run) => [run.status, run.queueHeld === true]), + [ + ["completed", false], + ["queued", true], + ], + ); + assert.lengthOf(yield* Ref.get(capturedTurns), 1); + yield* Ref.set(backgroundWork, false); + yield* orchestrator.dispatch({ + type: "queue.resume", + commandId: CommandId.make("command:handoff-releases-session:resume"), + threadId: handoffThreadId, + }); + yield* worker.drain(); + const resumed = yield* waitForIdle(handoffThreadId); + assert.deepEqual( + resumed.runs.map((run) => [run.providerInstanceId, run.status]), + [ + [CLAUDE_MODEL_SELECTION.instanceId, "completed"], + [secondSelection.instanceId, "completed"], + ], + ); + assert.deepEqual(yield* Ref.get(closedSessions), [firstSession!.id]); + return; + } + yield* worker.drain(); const afterFirst = yield* waitForIdle(handoffThreadId); const firstSession = afterFirst.providerSessions.find( @@ -2602,7 +2707,7 @@ describe("orchestration v2 provider switching", () => { ); assert.equal(firstSession?.status, "ready"); - if (backgroundRunning) { + if (background) { // Native subagents live only in the adapter's roster, so the projection's // pendingBackgroundTasks stays empty while the probe still reports work. assert.deepEqual(afterFirst.providerThreads[0]?.pendingBackgroundTasks ?? [], []); @@ -3148,7 +3253,7 @@ describe("orchestration v2 provider switching", () => { const layerImporterProvided = LegacyV1ThreadImporter.layer.pipe( Layer.provide(Layer.mergeAll(layerStoresProvided, layerEventSinkProvided)), ); - const layerMaintenanceProvided = ProjectionMaintenance.layer.pipe( + const layerMaintenanceProvided = ProjectionReplay.layer.pipe( Layer.provide(layerStoresProvided), ); const layerOrchestratorProvided = ProviderReplayHarness.layerWithRegistry( @@ -3177,7 +3282,7 @@ describe("orchestration v2 provider switching", () => { const projection = yield* Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const importer = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter; - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const maintenance = yield* ProjectionReplay.ProjectionReplay; const orchestrator = yield* Orchestrator.OrchestratorV2; const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; diff --git a/apps/server/src/peer/PeerLinkRequests.test.ts b/apps/server/src/peer/PeerLinkRequests.test.ts index 0d69bd65f5698..a973ba3a63632 100644 --- a/apps/server/src/peer/PeerLinkRequests.test.ts +++ b/apps/server/src/peer/PeerLinkRequests.test.ts @@ -537,6 +537,9 @@ it.live("the agent asks without waiting, and the user links the box after its tu yield* a.orchestrator.getThreadProjection(threadId), ).pendingRuntimeRequest, ).toBeNull(); + expect( + (yield* a.projections.getSettlementCandidates(threadId)).map((thread) => thread.id), + ).toEqual([threadId]); expect(yield* cardOf(a)).toMatchObject({ status: "waiting", linkStatus: "pending" }); // The user's client mints a code on the box and picks more than the agent suggested. diff --git a/apps/server/src/peer/PeerLinks.test.ts b/apps/server/src/peer/PeerLinks.test.ts index 3c510c35e6904..36c9b09fb4459 100644 --- a/apps/server/src/peer/PeerLinks.test.ts +++ b/apps/server/src/peer/PeerLinks.test.ts @@ -36,6 +36,8 @@ import { } from "./PeerLinks.testkit.ts"; import * as PeerMcpClient from "./PeerMcpClient.ts"; +import { layerReaderDatabase, traceSqlRouting } from "../persistence/testkit/SqliteRouting.ts"; + /** What B's probe saw: the calling client and the modes it may start work with. */ const Probe = Schema.Struct({ caller: Schema.String, modes: Schema.String }); @@ -115,6 +117,47 @@ const serveApart = (descriptor: ExecutionEnvironmentDescriptor) => Effect.flatMap((server) => serveB(descriptor).pipe(Effect.provideContext(server))), ); +it.effect("routes peer reads to readers and sees uncommitted link updates", () => + Effect.scoped( + Effect.gen(function* () { + const context = yield* layerLinkingEnvironment( + laptop, + ServerSecretStore.layer, + layerReaderDatabase, + ).pipe(Layer.build); + const links = Context.get(context, PeerLinks.PeerLinks); + const sql = Context.get(context, SqlClient.SqlClient); + yield* sql` + INSERT INTO peer_environment_links ( + environment_id, secret_name, label, urls_json, access, linked_at, expires_at, fenced + ) VALUES ( + ${box.environmentId}, 'test-secret', 'Box', '[]', 'auto', + '2026-01-01T00:00:00.000Z', '2026-01-02T00:00:00.000Z', 1 + ) + `; + const { queries, trace } = traceSqlRouting(); + yield* links.list.pipe(trace); + expect(Option.getOrThrow(yield* links.get(box.environmentId).pipe(trace)).label).toBe("Box"); + const reads = queries.filter(({ query }) => + query.includes("SELECT * FROM peer_environment_links"), + ); + expect(reads).toHaveLength(2); + expect(reads.map((row) => row.connection)).toEqual(["reader", "reader"]); + yield* sql.withTransaction( + Effect.gen(function* () { + yield* sql`UPDATE peer_environment_links SET label = 'Uncommitted box'`; + expect(Option.getOrThrow(yield* links.get(box.environmentId)).label).toBe( + "Uncommitted box", + ); + expect((yield* links.list)[0]?.label).toBe("Uncommitted box"); + }), + ); + expect(yield* links.unlink(box.environmentId)).toBe(true); + expect(yield* links.list).toEqual([]); + }), + ), +); + it.effect("links with a pairing code, and the peer holds its agents to the limits they carry", () => Effect.scoped( Effect.gen(function* () { @@ -410,6 +453,27 @@ it.effect("reads one stored link without asking the peer", () => ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); +it.effect("a linked peer that is renamed shows its new name", () => + Effect.scoped( + Effect.gen(function* () { + const b = yield* serveB(box); + const a = yield* makeA(laptop); + const pairing = yield* b.auth.issuePairingCredential(); + yield* a.links.link({ url: b.url, pairingCode: pairing.credential, access: "auto" }); + + yield* Ref.set(b.descriptor, descriptorOf(box.environmentId, "Box Mini")); + expect((yield* a.links.resolve(box.environmentId)).link.label).toBe("Box Mini"); + const stored = yield* a.links.get(box.environmentId); + expect(Option.map(stored, (link) => link.label)).toEqual(Option.some("Box Mini")); + + yield* Ref.set(b.descriptor, descriptorOf(box.environmentId, "Box Studio")); + expect((yield* a.links.list).map((link) => link.label)).toEqual(["Box Studio"]); + const storedAfterList = yield* a.links.get(box.environmentId); + expect(Option.map(storedAfterList, (link) => link.label)).toEqual(Option.some("Box Studio")); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + it.effect("tells the agent to link again once the peer revokes the link or it expires", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/peer/PeerLinks.testkit.ts b/apps/server/src/peer/PeerLinks.testkit.ts index c72c5ba8c81bc..1ff940af9b3e1 100644 --- a/apps/server/src/peer/PeerLinks.testkit.ts +++ b/apps/server/src/peer/PeerLinks.testkit.ts @@ -21,6 +21,7 @@ import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; import * as AuthHttp from "../auth/http.ts"; import * as McpOAuth from "../auth/McpOAuth.ts"; import * as McpOAuthHttp from "../auth/mcpOAuthHttp.ts"; +import type * as SqlClient from "effect/sql/SqlClient"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as ServerConfig from "../config.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; @@ -154,14 +155,18 @@ export type ServedPeer = Effect.Success>; * The environment that links: its own database and secret store, reaching * peers over plain HTTP. Provide it to anything built on its links. */ -export const layerLinkingEnvironment = ( +export const layerLinkingEnvironment = ( descriptor: ExecutionEnvironmentDescriptor, secrets: typeof ServerSecretStore.layer = ServerSecretStore.layer, + database: Layer.Layer< + SqlClient.SqlClient, + E | Layer.Error + > = Sqlite.layerMemory, ) => PeerMcpClient.layer.pipe( Layer.provideMerge(PeerLinks.layer), Layer.provideMerge(ServerSettings.layerTest({ linkedEnvironmentsEnabled: true })), - Layer.provideMerge(Sqlite.layerMemory), + Layer.provideMerge(database), Layer.provide(Layer.mergeAll(secrets, FetchHttpClient.layer)), // Services that act for this environment, such as delegation, name it too. Layer.provideMerge( diff --git a/apps/server/src/peer/PeerLinks.ts b/apps/server/src/peer/PeerLinks.ts index adbc8cbd1edea..dfcab9f11f273 100644 --- a/apps/server/src/peer/PeerLinks.ts +++ b/apps/server/src/peer/PeerLinks.ts @@ -9,9 +9,9 @@ import { type PeerLink, type PeerLinkCreateInput, PeerLinkError, - type PeerLinkStatus, type PeerLinkSummary, } from "@t3tools/contracts"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; import * as Context from "effect/Context"; import * as Digest from "effect/crypto/Digest"; import * as SecureRandom from "effect/crypto/SecureRandom"; @@ -222,12 +222,13 @@ const make = Effect.gen(function* () { const readRows = sql` SELECT * FROM peer_environment_links ORDER BY label, environment_id - `.pipe(Effect.catch(failStorage)); + `.pipe(NodeSqliteClient.readOnly, Effect.catch(failStorage)); const readRow = (environmentId: EnvironmentId) => sql` SELECT * FROM peer_environment_links WHERE environment_id = ${environmentId} `.pipe( + NodeSqliteClient.readOnly, Effect.catch(failStorage), Effect.map((rows) => Option.fromNullishOr(rows[0])), ); @@ -494,29 +495,42 @@ const make = Effect.gen(function* () { Effect.orElseSucceed(() => Option.none()), ); - const statusOf = (link: PeerLink, now: DateTime.Utc) => - DateTime.isLessThanOrEqualTo(link.expiresAt, now) - ? Effect.succeed("expired") - : Effect.findFirst(link.urls, (url) => - describe(url).pipe( - Effect.map( - (found) => Option.isSome(found) && found.value.environmentId === link.environmentId, - ), - ), - ).pipe( - Effect.map((answered): PeerLinkStatus => - Option.isSome(answered) ? "reachable" : "unreachable", - ), - ); + /** A peer's label follows its machine name, so the link takes the one its own address answers with. */ + const withLabel = (link: PeerLink, label: string) => + label === link.label + ? Effect.succeed(link) + : sql` + UPDATE peer_environment_links SET label = ${label} + WHERE environment_id = ${link.environmentId} + `.pipe(Effect.ignoreCause({ log: true }), Effect.as({ ...link, label })); + + const statusOf = Effect.fn("PeerLinks.statusOf")(function* ( + link: PeerLink, + now: DateTime.Utc, + ): Effect.fn.Return { + if (DateTime.isLessThanOrEqualTo(link.expiresAt, now)) return { ...link, status: "expired" }; + let label = link.label; + const answered = yield* Effect.findFirst(link.urls, (url) => + describe(url).pipe( + Effect.map((found) => { + if (Option.isNone(found) || found.value.environmentId !== link.environmentId) { + return false; + } + label = found.value.label; + return true; + }), + ), + ); + if (Option.isNone(answered)) return { ...link, status: "unreachable" }; + return { ...(yield* withLabel(link, label)), status: "reachable" }; + }); const list: PeerLinks["Service"]["list"] = Effect.gen(function* () { const links = yield* Effect.forEach(yield* readRows, toLink); const now = yield* DateTime.now; - return yield* Effect.forEach( - links, - (link) => statusOf(link, now).pipe(Effect.map((status) => ({ ...link, status }))), - { concurrency: "unbounded" }, - ); + return yield* Effect.forEach(links, (link) => statusOf(link, now), { + concurrency: "unbounded", + }); }); const get: PeerLinks["Service"]["get"] = (environmentId) => @@ -568,11 +582,13 @@ const make = Effect.gen(function* () { // The token goes only to an address that proves it is this peer, so a // reused LAN address or a moved hostname never receives it. let fenced = true; + let label = link.label; const url = yield* Effect.findFirst(link.urls, (candidate) => describe(candidate).pipe( Effect.map((found) => { if (Option.isNone(found) || found.value.environmentId !== environmentId) return false; fenced = linkLimitsHold(found.value); + label = found.value.label; return true; }), ), @@ -591,7 +607,11 @@ const make = Effect.gen(function* () { message: `${link.label} runs a T3 Code version that cannot keep a linked agent's limits. Update it, then link again.`, }); } - return { link, url: url.value, token: new TextDecoder().decode(stored.value) }; + return { + link: yield* withLabel(link, label), + url: url.value, + token: new TextDecoder().decode(stored.value), + }; }, ); diff --git a/apps/server/src/peer/PeerMcpClient.test.ts b/apps/server/src/peer/PeerMcpClient.test.ts index 7d19f4e856bf7..e504b14f4dd2e 100644 --- a/apps/server/src/peer/PeerMcpClient.test.ts +++ b/apps/server/src/peer/PeerMcpClient.test.ts @@ -1,5 +1,6 @@ import { NodeHttpServer } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; +import { OrchestratorMcpFailure } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; @@ -32,7 +33,12 @@ const toolkit = Toolkit.make( }), ); -it.effect.each([false, true])( +const refusal = new OrchestratorMcpFailure({ + code: "thread_not_found", + message: "Thread thread-gone was not found.", +}); + +it.live.each([false, true])( "reuses an initialized session before its first call finishes (cancelled: %s)", (cancelFirst) => Effect.scoped( @@ -94,7 +100,8 @@ it.effect.each([false, true])( it.effect("renews a missing peer session once and reuses its replacement", () => Effect.gen(function* () { const sessionHeaders: Array = []; - const statuses = [200, 202, 404, 200, 202, 200, 200]; + const statuses = [200, 202, 404, 200, 202, 200, 200, 200, 200]; + const outcomes: Array = []; const http = HttpClient.make((request) => Effect.sync(() => { const index = sessionHeaders.length; @@ -104,13 +111,27 @@ it.effect("renews a missing peer session once and reuses its replacement", () => return HttpClientResponse.fromWeb( request, new Response( - index >= 5 + index === 8 ? JSON.stringify({ result: { - content: [{ type: "text", text: '{"modes":"approval-required/plan"}' }], + content: [ + { + type: "text", + text: JSON.stringify({ code: refusal.code, message: refusal.message }), + }, + ], + isError: true, }, }) - : "", + : index === 7 + ? "{}" + : index >= 5 + ? JSON.stringify({ + result: { + content: [{ type: "text", text: '{"modes":"approval-required/plan"}' }], + }, + }) + : "", { status, headers: @@ -146,7 +167,7 @@ it.effect("renews a missing peer session once and reuses its replacement", () => url: "https://peer.example", token: "fixture", }), - recordOutcome: () => Effect.void, + recordOutcome: (_, outcome) => Effect.sync(() => outcomes.push(outcome.error)), }), ), ), @@ -164,6 +185,27 @@ it.effect("renews a missing peer session once and reuses its replacement", () => }), ).toEqual({ modes: "approval-required/plan" }); } + // An answer with neither a result nor an error is the link's problem, not the call's. + const empty = yield* client + .call({ + environmentId: box.environmentId, + tool: "probe", + arguments: {}, + limits: { runtimeMode: "approval-required", interactionMode: "plan" }, + success: Result, + }) + .pipe(Effect.flip); + const refused = yield* client + .call({ + environmentId: box.environmentId, + tool: "refuse", + arguments: {}, + limits: { runtimeMode: "full-access", interactionMode: "default" }, + success: Result, + }) + .pipe(Effect.flip); + expect(refused).toMatchObject({ code: refusal.code, message: refusal.message }); + expect(outcomes).toEqual([null, null, empty.message, null]); expect(sessionHeaders).toEqual([ undefined, "old", @@ -172,6 +214,8 @@ it.effect("renews a missing peer session once and reuses its replacement", () => "renewed", "renewed", "renewed", + "renewed", + "renewed", ]); }).pipe(Effect.scoped), ); diff --git a/apps/server/src/peer/PeerMcpClient.ts b/apps/server/src/peer/PeerMcpClient.ts index 27397d54ba7de..885c91dce8665 100644 --- a/apps/server/src/peer/PeerMcpClient.ts +++ b/apps/server/src/peer/PeerMcpClient.ts @@ -9,6 +9,7 @@ import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Ref from "effect/Ref"; +import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import * as HttpBody from "effect/http/HttpBody"; import * as HttpClient from "effect/http/HttpClient"; @@ -210,25 +211,36 @@ const make = Effect.gen(function* () { }), ), ); - if (body.error !== undefined || body.result === undefined) { + if (body.error !== undefined) { + return Result.fail( + new OrchestratorMcpFailure({ code: "invalid_request", message: body.error.message }), + ); + } + if (body.result === undefined) { return yield* new OrchestratorMcpFailure({ code: "invalid_request", - message: body.error?.message ?? "The linked environment returned no result.", + message: "The linked environment returned no result.", }); } const text = body.result.content.map((part) => part.text ?? "").join(""); if (body.result.isError === true) { // A refusal keeps the peer's own code, so the agent sees why it was refused. const refused = decodeFailure(text); - return yield* refused._tag === "Some" - ? new OrchestratorMcpFailure({ code: refused.value.code, message: refused.value.message }) - : new OrchestratorMcpFailure({ - code: "orchestration_error", - message: text || "The linked environment refused the call.", - }); + return Result.fail( + refused._tag === "Some" + ? new OrchestratorMcpFailure({ + code: refused.value.code, + message: refused.value.message, + }) + : new OrchestratorMcpFailure({ + code: "orchestration_error", + message: text || "The linked environment refused the call.", + }), + ); } const success = Schema.fromJsonString(input.success); return yield* Schema.decodeEffect(success)(text).pipe( + Effect.map(Result.succeed), Effect.mapError( () => new OrchestratorMcpFailure({ @@ -244,6 +256,7 @@ const make = Effect.gen(function* () { Effect.tapError((error) => links.recordOutcome(input.environmentId, { error: error.message }), ), + // A refusal is still an answer: the link works, so only this call fails. Effect.tap(() => links.recordOutcome(input.environmentId, { error: null })), // Recorded above with the transport's own message; the agent hears it plainly. Effect.catchTags({ @@ -255,6 +268,7 @@ const make = Effect.gen(function* () { }), ), }), + Effect.flatMap(Effect.fromResult), ), ); diff --git a/apps/server/src/persistence/DatabaseMaintenance.test.ts b/apps/server/src/persistence/DatabaseMaintenance.test.ts index 381b143923adf..c767cebd0f886 100644 --- a/apps/server/src/persistence/DatabaseMaintenance.test.ts +++ b/apps/server/src/persistence/DatabaseMaintenance.test.ts @@ -12,9 +12,6 @@ import * as SqlClient from "effect/sql/SqlClient"; import * as DatabaseMaintenance from "./DatabaseMaintenance.ts"; import * as Sqlite from "./Sqlite.ts"; -import * as EventStore from "../orchestration-v2/EventStore.ts"; -import * as ProjectionMaintenance from "../orchestration-v2/ProjectionMaintenance.ts"; -import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; it.layer(DatabaseMaintenance.layer.pipe(Layer.provideMerge(NodeServices.layer)))( "DatabaseMaintenance", @@ -111,17 +108,8 @@ it.layer(DatabaseMaintenance.layer.pipe(Layer.provideMerge(NodeServices.layer))) const maintenance = yield* DatabaseMaintenance.DatabaseMaintenance; const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-compact-" }); const dbPath = path.join(baseDir, "userdata", "statev2.sqlite"); - yield* Effect.flatMap( - ProjectionMaintenance.ProjectionMaintenanceV2, - (projections) => projections.rebuild, - ).pipe( - Effect.provide( - ProjectionMaintenance.layer.pipe( - Layer.provide(Layer.merge(EventStore.layer, ProjectionStore.layer)), - Layer.provideMerge(Sqlite.layerFromPath(dbPath)), - ), - ), - ); + // Opening the database migrates it. + yield* Effect.void.pipe(Effect.provide(Sqlite.layerFromPath(dbPath))); yield* Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; yield* sql`PRAGMA auto_vacuum = NONE`; diff --git a/apps/server/src/persistence/DatabaseMaintenance.ts b/apps/server/src/persistence/DatabaseMaintenance.ts index 4a97e6dcace98..d7876785c898d 100644 --- a/apps/server/src/persistence/DatabaseMaintenance.ts +++ b/apps/server/src/persistence/DatabaseMaintenance.ts @@ -9,9 +9,7 @@ import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import { isSqlError } from "effect/sql/SqlError"; -import * as EventStore from "../orchestration-v2/EventStore.ts"; import * as ProjectionMaintenance from "../orchestration-v2/ProjectionMaintenance.ts"; -import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import { rebuildMessageSearch } from "../orchestration-v2/messageSearchIndex.ts"; export class DatabaseMissingError extends Schema.TaggedError()( @@ -32,15 +30,6 @@ export class DatabaseInUseError extends Schema.TaggedError() } } -export class DatabaseProjectionsInvalidError extends Schema.TaggedError()( - "DatabaseProjectionsInvalidError", - { path: Schema.String }, -) { - override get message() { - return "Database projections need repair before compaction. Start T3 Code once so it rebuilds them, then run doctor again. If this CLI is older than the one that wrote the database, update T3 Code first."; - } -} - export class DatabaseInspectError extends Schema.TaggedError()( "DatabaseInspectError", { path: Schema.String, cause: Schema.Defect() }, @@ -89,10 +78,7 @@ export class DatabaseMaintenance extends Context.Service< readonly deletedEventCount: number; readonly deletedReceiptCount: number; }, - | DatabaseMissingError - | DatabaseInUseError - | DatabaseProjectionsInvalidError - | DatabaseCompactError + DatabaseMissingError | DatabaseInUseError | DatabaseCompactError >; /** Deletes only the pre-0.1.0 database; earlier versions still use it. */ readonly removeLegacyDatabase: ( @@ -101,9 +87,7 @@ export class DatabaseMaintenance extends Context.Service< } >()("t3/persistence/DatabaseMaintenance") {} -const isDomainError = Schema.is( - Schema.Union([DatabaseMissingError, DatabaseInUseError, DatabaseProjectionsInvalidError]), -); +const isDomainError = Schema.is(Schema.Union([DatabaseMissingError, DatabaseInUseError])); const readStorage = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; @@ -180,13 +164,7 @@ const make = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; yield* lockExclusively(databasePath); const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; - if (!(yield* maintenance.verify).valid) { - return yield* new DatabaseProjectionsInvalidError({ path: databasePath }); - } - const summary = yield* maintenance.compactEventStore(); - if (!(yield* maintenance.verify).valid) { - return yield* new DatabaseProjectionsInvalidError({ path: databasePath }); - } + const summary = yield* maintenance.pruneEventStore(); // VACUUM is required to enable incremental auto_vacuum on existing databases. yield* sql`PRAGMA auto_vacuum = INCREMENTAL`; yield* sql`VACUUM`; @@ -196,7 +174,6 @@ const make = Effect.gen(function* () { }).pipe( Effect.provide( ProjectionMaintenance.layer.pipe( - Layer.provide(Layer.merge(EventStore.layer, ProjectionStore.layer)), Layer.provideMerge(NodeSqliteClient.layer({ filename: databasePath })), ), ), diff --git a/apps/server/src/persistence/DatabaseStorage.test.ts b/apps/server/src/persistence/DatabaseStorage.test.ts index 878f596600888..73a0d4cac7cd8 100644 --- a/apps/server/src/persistence/DatabaseStorage.test.ts +++ b/apps/server/src/persistence/DatabaseStorage.test.ts @@ -9,9 +9,7 @@ import * as TestClock from "effect/testing/TestClock"; import * as SqlClient from "effect/sql/SqlClient"; import * as ServerConfig from "../config.ts"; -import * as EventStore from "../orchestration-v2/EventStore.ts"; import * as ProjectionMaintenance from "../orchestration-v2/ProjectionMaintenance.ts"; -import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import * as DatabaseStorage from "./DatabaseStorage.ts"; import * as Sqlite from "./Sqlite.ts"; @@ -28,7 +26,6 @@ const storageLayer = ( ) => DatabaseStorage.layer.pipe( Layer.provide(ProjectionMaintenance.layer), - Layer.provide(Layer.merge(EventStore.layer, ProjectionStore.layer)), Layer.provideMerge(Sqlite.layerMemory), Layer.provideMerge(config), Layer.provide(NodeServices.layer), @@ -54,14 +51,12 @@ it.effect("a new database returns freed pages online without losing data", () => ); const compactionLayer = ( - compactEventStore: ProjectionMaintenance.ProjectionMaintenanceV2["Service"]["compactEventStore"], + pruneEventStore: ProjectionMaintenance.ProjectionMaintenanceV2["Service"]["pruneEventStore"], ) => DatabaseStorage.layer.pipe( Layer.provide( Layer.succeed(ProjectionMaintenance.ProjectionMaintenanceV2, { - compactEventStore, - verify: Effect.die("unexpected verification"), - rebuild: Effect.die("unexpected rebuild"), + pruneEventStore, }), ), Layer.provide(Sqlite.layerMemory), diff --git a/apps/server/src/persistence/DatabaseStorage.ts b/apps/server/src/persistence/DatabaseStorage.ts index bbb4dd1399efd..a0dd307237d46 100644 --- a/apps/server/src/persistence/DatabaseStorage.ts +++ b/apps/server/src/persistence/DatabaseStorage.ts @@ -31,7 +31,6 @@ export class DatabaseStorage extends Context.Service< } >()("t3/persistence/DatabaseStorage") {} -// Deleted events have newer retained replacements; old cursors replay to the same state. const make = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; @@ -108,7 +107,7 @@ const make = Effect.gen(function* () { yield* Effect.sleep((Number(elapsed) / 1_000_000) * 9); sliceStarted = yield* Clock.monotonicTimeNanos; }); - const result = yield* maintenance.compactEventStore(betweenPages); + const result = yield* maintenance.pruneEventStore(betweenPages); const completedAt = yield* Clock.currentTimeMillis; yield* fs.writeFileString(compactedAtPath, String(completedAt)).pipe(Effect.ignore); if (result.deletedEventCount > 0 || result.deletedReceiptCount > 0) { diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 30daafee5c9e8..93d2d5e5feaf0 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -82,6 +82,8 @@ import Migration0076 from "./Migrations/076_ProjectionProjectsPinArchive.ts"; import Migration0077 from "./Migrations/077_RepairBooleanSettledOverride.ts"; import Migration0078 from "./Migrations/078_DropOrchestrationEventCorrelationIndex.ts"; +import Migration0079 from "./Migrations/079_DelegatedRecoveryIndexes.ts"; + const migrationEntries = [ [1, "OrchestrationEvents", Migration0001], [2, "OrchestrationCommandReceipts", Migration0002], @@ -162,6 +164,7 @@ const migrationEntries = [ [76, "ProjectionProjectsPinArchive", Migration0076], [77, "RepairBooleanSettledOverride", Migration0077], [78, "DropOrchestrationEventCorrelationIndex", Migration0078], + [79, "DelegatedRecoveryIndexes", Migration0079], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/064_CanonicalizeThreadPullRequestLinks.test.ts b/apps/server/src/persistence/Migrations/064_CanonicalizeThreadPullRequestLinks.test.ts index 69eb8eff57027..b53309c187ed3 100644 --- a/apps/server/src/persistence/Migrations/064_CanonicalizeThreadPullRequestLinks.test.ts +++ b/apps/server/src/persistence/Migrations/064_CanonicalizeThreadPullRequestLinks.test.ts @@ -9,7 +9,7 @@ import * as SqlClient from "effect/sql/SqlClient"; import * as EventStore from "../../orchestration-v2/EventStore.ts"; import * as ProjectionStore from "../../orchestration-v2/ProjectionStore.ts"; -import * as ProjectionMaintenance from "../../orchestration-v2/ProjectionMaintenance.ts"; +import * as ProjectionReplay from "../../orchestration-v2/testkit/ProjectionReplay.ts"; import { listLinkedPullRequestThreads } from "../../pullRequest/linkedThreads.ts"; import { runMigrations } from "../Migrations.ts"; @@ -53,7 +53,7 @@ const thread = (id: string) => ({ }); const layer = it.layer( - ProjectionMaintenance.layer.pipe( + ProjectionReplay.layer.pipe( Layer.provideMerge( Layer.mergeAll(EventStore.layer, ProjectionStore.layer).pipe( Layer.provideMerge(NodeSqliteClient.layer({ filename: ":memory:" })), @@ -156,7 +156,7 @@ layer("064_CanonicalizeThreadPullRequestLinks", (it) => { `; assert.equal(v1?.payload_json, v1Payload); - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const maintenance = yield* ProjectionReplay.ProjectionReplay; assert.isTrue((yield* maintenance.rebuild).valid); for (const [host, repository, id] of [ ["github.com", "acme/web", "github"], diff --git a/apps/server/src/persistence/Migrations/066_RepairClientSettlementTimestamps.test.ts b/apps/server/src/persistence/Migrations/066_RepairClientSettlementTimestamps.test.ts index f30289cb92de9..d62654b37ece6 100644 --- a/apps/server/src/persistence/Migrations/066_RepairClientSettlementTimestamps.test.ts +++ b/apps/server/src/persistence/Migrations/066_RepairClientSettlementTimestamps.test.ts @@ -6,13 +6,13 @@ import * as Layer from "effect/Layer"; import * as SqlClient from "effect/sql/SqlClient"; import * as EventStore from "../../orchestration-v2/EventStore.ts"; -import * as ProjectionMaintenance from "../../orchestration-v2/ProjectionMaintenance.ts"; +import * as ProjectionReplay from "../../orchestration-v2/testkit/ProjectionReplay.ts"; import * as ProjectionStore from "../../orchestration-v2/ProjectionStore.ts"; import { runMigrations } from "../Migrations.ts"; import Migration0066 from "./066_RepairClientSettlementTimestamps.ts"; const layer = it.layer( - ProjectionMaintenance.layer.pipe( + ProjectionReplay.layer.pipe( Layer.provideMerge( Layer.mergeAll(EventStore.layer, ProjectionStore.layer).pipe( Layer.provideMerge(NodeSqliteClient.layer({ filename: ":memory:" })), @@ -363,7 +363,7 @@ layer("066_RepairClientSettlementTimestamps", (it) => { yield* Migration0066; assert.deepEqual(yield* snapshot, repaired); - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const maintenance = yield* ProjectionReplay.ProjectionReplay; assert.isTrue((yield* maintenance.rebuild).valid); assert.deepEqual(yield* imported, expectedImported); }), diff --git a/apps/server/src/persistence/Migrations/079_DelegatedRecoveryIndexes.ts b/apps/server/src/persistence/Migrations/079_DelegatedRecoveryIndexes.ts new file mode 100644 index 0000000000000..3ba72feaf1788 --- /dev/null +++ b/apps/server/src/persistence/Migrations/079_DelegatedRecoveryIndexes.ts @@ -0,0 +1,26 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + // Include the parent ID so transfer checks do not reread each child payload. + yield* sql` + CREATE INDEX orchestration_v2_projection_threads_subagent_result_idx + ON orchestration_v2_projection_threads( + thread_id, json_extract(payload_json, '$.lineage.parentThreadId') + ) + WHERE CASE WHEN json_valid(payload_json) THEN + json_extract(payload_json, '$.lineage.relationshipToParent') = 'subagent' + AND json_extract(payload_json, '$.lineage.parentThreadId') IS NOT NULL + AND json_extract(payload_json, '$.forkedFrom.type') = 'node' + ELSE 0 END + `; + yield* sql` + CREATE INDEX orchestration_v2_projection_runs_delegated_delivery_idx + ON orchestration_v2_projection_runs(thread_id) + WHERE CASE WHEN json_valid(payload_json) + THEN json_type(payload_json, '$.delegatedCompletion.delivery') = 'object' + ELSE 0 END + `; +}); diff --git a/apps/server/src/persistence/OrchestrationEventStore.ts b/apps/server/src/persistence/OrchestrationEventStore.ts index dbea09d666988..e55d82713871b 100644 --- a/apps/server/src/persistence/OrchestrationEventStore.ts +++ b/apps/server/src/persistence/OrchestrationEventStore.ts @@ -33,6 +33,10 @@ import { type OrchestrationEventStoreError, } from "./Errors.ts"; +/** Projection metadata rows: events at or below the pruned-through sequence may be gone. */ +export const EVENT_LOG_PRUNED_THROUGH = "event-log-pruned-through"; +export const EVENT_LOG_PRUNE_MARK = "event-log-prune-mark"; + export type UnsequencedProjectEvent = ApplicationProjectEvent extends infer Event ? Event extends ApplicationProjectEvent ? Omit @@ -93,6 +97,9 @@ export class OrchestrationEventStore extends Context.Service< readonly latestApplicationSequence: Effect.Effect; + /** Replay from a cursor below this misses pruned events; resync from a snapshot instead. */ + readonly prunedThroughSequence: Effect.Effect; + /** Unknown event types fail unless skipUnknownEventTypes logs and drops them; only client replay may opt in. */ readonly readApplicationEvents: (input: { readonly afterSequence: number; @@ -450,7 +457,7 @@ const makeEventStore = Effect.gen(function* () { const promptKey = subagent === undefined ? undefined : `${event.type}:${subagent.id}`; const hint = promptKey === undefined ? undefined : promptHints.get(promptKey); const payloadJson = yield* encodeJson(encoded.payload); - // Deduplicate repeated progress prompts via anchor rows that ProjectionMaintenance never deletes. + // Deduplicate repeated progress prompts via anchor rows; pruning keeps anchors that retained rows reference. const storedPayload = hint !== undefined && subagent !== undefined ? sql`CASE WHEN EXISTS ( @@ -672,6 +679,15 @@ const makeEventStore = Effect.gen(function* () { ), ); + const prunedThroughSequence = sql<{ readonly last_sequence: number }>` + SELECT last_sequence FROM orchestration_v2_projection_metadata + WHERE projection_name = ${EVENT_LOG_PRUNED_THROUGH} + `.pipe( + NodeSqliteClient.readOnly, + Effect.map((rows) => rows[0]?.last_sequence ?? 0), + Effect.mapError(toPersistenceSqlError("OrchestrationEventStore.prunedThroughSequence:query")), + ); + // Advance over raw rows so even a full page of unknown types cannot end replay early. const catchUpApplicationEvents = (input: { readonly afterSequence: number; @@ -748,6 +764,7 @@ const makeEventStore = Effect.gen(function* () { getReplayStats, latestAgentSequence, latestApplicationSequence, + prunedThroughSequence, readApplicationEvents: catchUpApplicationEvents, publishCommitted: (events) => PubSub.publishAll(committedEvents, events).pipe(Effect.asVoid), streamApplicationEvents, diff --git a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts index d062961b3af57..5f51c237fd763 100644 --- a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts +++ b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts @@ -58,6 +58,7 @@ describe("V2 preview upgrade", () => { [76, "ProjectionProjectsPinArchive"], [77, "RepairBooleanSettledOverride"], [78, "DropOrchestrationEventCorrelationIndex"], + [79, "DelegatedRecoveryIndexes"], ]); assert.deepStrictEqual(yield* runMigrations(), []); assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports); @@ -159,6 +160,7 @@ describe("V2 preview upgrade", () => { [76, "ProjectionProjectsPinArchive"], [77, "RepairBooleanSettledOverride"], [78, "DropOrchestrationEventCorrelationIndex"], + [79, "DelegatedRecoveryIndexes"], ]); }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); diff --git a/apps/server/src/persistence/testkit/SqliteRouting.ts b/apps/server/src/persistence/testkit/SqliteRouting.ts new file mode 100644 index 0000000000000..2dcd65ec2b2f9 --- /dev/null +++ b/apps/server/src/persistence/testkit/SqliteRouting.ts @@ -0,0 +1,52 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Statement from "effect/sql/Statement"; +import * as Tracer from "effect/Tracer"; + +import * as Sqlite from "../Sqlite.ts"; + +export const layerReaderDatabase = Layer.unwrap( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-routing-" }); + return Sqlite.layerFromPath(path.join(directory, "state.sqlite")); + }), +).pipe(Layer.provide(NodeServices.layer)); + +export const traceSqlRouting = () => { + const queries: Array<{ query: string; connection: unknown }> = []; + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options); + const end = span.end.bind(span); + span.end = (endTime, exit) => { + end(endTime, exit); + const query = span.attributes.get("db.query.text"); + if (typeof query === "string") { + let owner: Tracer.AnySpan | undefined = span; + let connection: unknown; + while (owner?._tag === "Span") { + connection = owner.attributes.get("db.connection"); + if (connection !== undefined) break; + owner = Option.getOrUndefined(owner.parent); + } + queries.push({ query, connection }); + } + }; + return span; + }, + }); + return { + queries, + trace: (effect: Effect.Effect) => + effect.pipe( + Effect.withTracer(tracer), + Effect.provideService(Statement.SpanPropagationEnabled, true), + ), + }; +}; diff --git a/apps/server/src/policy/ManagedPolicy.test.ts b/apps/server/src/policy/ManagedPolicy.test.ts index 9fbe4d2c42791..26e74f05b0372 100644 --- a/apps/server/src/policy/ManagedPolicy.test.ts +++ b/apps/server/src/policy/ManagedPolicy.test.ts @@ -11,6 +11,7 @@ import { } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import { BUILT_IN_DRIVERS } from "../provider/builtInDrivers.ts"; import * as ManagedPolicy from "./ManagedPolicy.ts"; import { disallowedDriverReason, providerHostEnvironment } from "./managedProviders.ts"; @@ -349,6 +350,20 @@ it.layer(NodeServices.layer)("ManagedPolicy.load", (it) => { it.effect("tells clients which drivers run and which instance fields are locked", () => Effect.gen(function* () { + // A provider the policy cannot name would also escape its launch-setting rules. + const everyDriver = BUILT_IN_DRIVERS.map((driver) => driver.driverKind); + const allowingAll = yield* loadFrom( + JSON.stringify({ + schemaVersion: 1, + organization: "Example Corp", + limits: { providers: { allowedDrivers: everyDriver } }, + }), + ); + assert.deepEqual( + allowingAll.summary?.limits?.providers?.allowedDrivers, + everyDriver, + ); + const policy = yield* loadFrom( JSON.stringify({ schemaVersion: 1, diff --git a/apps/server/src/policy/ManagedPolicy.ts b/apps/server/src/policy/ManagedPolicy.ts index 15167dedd38c2..01651051893c0 100644 --- a/apps/server/src/policy/ManagedPolicy.ts +++ b/apps/server/src/policy/ManagedPolicy.ts @@ -185,7 +185,10 @@ const make = Effect.gen(function* () { const policy = yield* Option.match(devPath, { onSome: load, onNone: () => - readManagedPolicy(platform).pipe( + readManagedPolicy( + platform, + ManagedProviders.knownDriverLabels().map(({ driver }) => driver), + ).pipe( Effect.flatMap((found) => fromDocument(found?.source ?? managedPolicyPath(platform), found?.document), ), diff --git a/apps/server/src/policy/managedProviders.ts b/apps/server/src/policy/managedProviders.ts index cbf0f0a1d9b24..ac390097e150d 100644 --- a/apps/server/src/policy/managedProviders.ts +++ b/apps/server/src/policy/managedProviders.ts @@ -19,6 +19,7 @@ import { acpRegistryClient } from "@t3tools/provider-acp-registry/client"; import { cursorClient } from "@t3tools/provider-cursor/client"; import { fxClient } from "@t3tools/provider-fx/client"; import { grokClient } from "@t3tools/provider-grok/client"; +import { kiroClient } from "@t3tools/provider-kiro/client"; import { museClient } from "@t3tools/provider-muse/client"; import { openCodeClient } from "@t3tools/provider-opencode/client"; import { piClient } from "@t3tools/provider-pi/client"; @@ -45,6 +46,7 @@ const PACKAGED_CLIENTS: ReadonlyArray = [ cursorClient, grokClient, fxClient, + kiroClient, museClient, openCodeClient, piClient, @@ -75,6 +77,10 @@ const KNOWN_DRIVERS: ReadonlyMap = new Map([ ), ]); +/** Each driver a policy may name, with the label users see. */ +export const knownDriverLabels = () => + Array.from(KNOWN_DRIVERS, ([driver, { label }]) => ({ driver, label })); + const PINNED_CONFIG_FIELDS = ["binaryPath", "launchArgs"] as const; type PinnedConfigField = (typeof PINNED_CONFIG_FIELDS)[number]; diff --git a/apps/server/src/policy/managedUpdates.ts b/apps/server/src/policy/managedUpdates.ts index 32adc6e4f3812..d805cfa028e2f 100644 --- a/apps/server/src/policy/managedUpdates.ts +++ b/apps/server/src/policy/managedUpdates.ts @@ -5,6 +5,17 @@ import { } from "@t3tools/contracts"; import type { CliReleaseChannel } from "@t3tools/shared/cliRelease"; +/** + * The update mirror the policy sends updates to, or `undefined` for the public releases. It wins + * over the environment, which a user can set. + */ +export function managedUpdateMirror(document: ManagedPolicyDocument | undefined) { + const feedUrl = document?.limits?.updates?.feedUrl; + return feedUrl === undefined || document === undefined + ? undefined + : { organization: document.organization, feedUrl }; +} + /** * Why the managed policy refuses an update a user or maintainer asked for, or `undefined` when it * allows it. The server never updates on its own, so `auto` allows every explicit request. diff --git a/apps/server/src/policy/policyTemplates.test.ts b/apps/server/src/policy/policyTemplates.test.ts new file mode 100644 index 0000000000000..9d97f0f311994 --- /dev/null +++ b/apps/server/src/policy/policyTemplates.test.ts @@ -0,0 +1,37 @@ +import { assert, it } from "@effect/vitest"; +import { + WINDOWS_POLICY_GROUPS, + WINDOWS_POLICY_LISTS, + WINDOWS_POLICY_VALUES, +} from "@t3tools/shared/managedPolicyFile"; + +import { macosProfile, windowsAdmx } from "./policyTemplates.ts"; + +it("gives every registry value the Windows reader accepts a Group Policy control", () => { + const admx = windowsAdmx(); + for (const name of Object.keys(WINDOWS_POLICY_VALUES)) { + assert.include(admx, `valueName="${name}"`); + } + for (const key of Object.keys(WINDOWS_POLICY_LISTS)) { + assert.include(admx, `key="SOFTWARE\\Policies\\T3Tools\\T3Code\\${key}"`); + } + for (const name of Object.keys(WINDOWS_POLICY_GROUPS)) { + assert.include(admx, `valueName="${name}"`); + } +}); + +it("refuses a profile for a policy with a null value, which a plist cannot hold", () => { + const profile = macosProfile( + { + schemaVersion: 1, + organization: "Example Corp", + settings: { defaultModelSelection: { value: null, locked: true } }, + }, + { profile: "profile-uuid", payload: "payload-uuid" }, + ); + assert.deepStrictEqual(profile, { + _tag: "Invalid", + reason: + "settings.defaultModelSelection is null, which a configuration profile cannot hold; deploy this policy as a file.", + }); +}); diff --git a/apps/server/src/policy/policyTemplates.ts b/apps/server/src/policy/policyTemplates.ts new file mode 100644 index 0000000000000..55c15fb94f56c --- /dev/null +++ b/apps/server/src/policy/policyTemplates.ts @@ -0,0 +1,386 @@ +import type { ManagedPolicyDocument } from "@t3tools/contracts"; +import { + WINDOWS_POLICY_KEY, + WINDOWS_POLICY_GROUPS, + WINDOWS_POLICY_LISTS, + WINDOWS_POLICY_VALUES, +} from "@t3tools/shared/managedPolicyFile"; + +import { knownDriverLabels } from "./managedProviders.ts"; + +type ValueName = keyof typeof WINDOWS_POLICY_VALUES; +type ListKey = keyof typeof WINDOWS_POLICY_LISTS; + +type Element = + | { + readonly kind: "text"; + readonly name: ValueName; + readonly label: string; + readonly json?: true; + } + | { + readonly kind: "enum"; + readonly name: ValueName; + readonly label: string; + readonly items: ReadonlyArray<{ readonly value: string; readonly label: string }>; + readonly defaultItem: number; + } + | { + readonly kind: "checkbox"; + readonly list?: ListKey; + readonly name: string; + readonly label: string; + readonly defaultChecked: boolean; + }; + +interface Policy { + readonly id: string; + readonly title: string; + readonly help: string; + /** A policy whose Enabled and Disabled states are themselves the value. */ + readonly toggle?: ValueName; + readonly elements?: ReadonlyArray; +} + +const RUNTIME_MODES = [ + { value: "approval-required", label: "Supervised" }, + { value: "auto-accept-edits", label: "Auto-accept edits" }, + { value: "auto", label: "Auto" }, + { value: "full-access", label: "Full access" }, +]; + +const policies = (): ReadonlyArray => [ + { + id: "Organization", + title: "Organization", + help: 'Required whenever any other T3 Code policy is set: T3 Code will not start with policies that do not name an organization.\n\nUsers see the name as "Managed by " beside every control the policy locks. The support URL is optional.', + elements: [ + { kind: "text", name: "Organization", label: "Organization name" }, + { kind: "text", name: "SupportUrl", label: "Support URL (optional)" }, + ], + }, + { + id: "MaxRuntimeMode", + title: "Broadest permission mode", + help: "The broadest permission mode any thread may use, including threads agents and scheduled tasks start.\n\nA thread already above it cannot run another turn until someone lowers its mode. Under this limit, each provider's launch settings (binary path, launch arguments, environment variables) come only from the policy.\n\nDisabled or Not Configured: no restriction.", + elements: [ + { + kind: "enum", + name: "MaxRuntimeMode", + label: "Broadest permission mode", + items: RUNTIME_MODES, + defaultItem: 1, + }, + ], + }, + { + id: "Clients", + title: "Allowed clients", + help: "What may connect to T3 Code on this machine.\n\nWithout Remote, T3 Code only listens on this machine, does not create pairing links, signs out devices that paired earlier, and turns T3 Connect off. Check at least one client.\n\nDisabled or Not Configured: no restriction.", + elements: [ + { + kind: "checkbox", + list: "Clients", + name: "desktop", + label: "Desktop app on this machine", + defaultChecked: true, + }, + { + kind: "checkbox", + list: "Clients", + name: "local-browser", + label: "A browser on this machine", + defaultChecked: true, + }, + { + kind: "checkbox", + list: "Clients", + name: "remote", + label: "Remote: phones, other computers, app.t3.codes", + defaultChecked: false, + }, + ], + }, + { + id: "Connect", + title: "Allow T3 Connect", + help: "Disabled turns T3 Connect off and unlinks this machine. Other remote access still follows Allowed clients.\n\nEnabled or Not Configured leaves T3 Connect available.", + toggle: "Connect", + }, + { + id: "WebhookTasks", + title: "Allow webhook tasks", + help: "Disabled stops scheduled tasks from using webhook triggers. Existing webhook tasks are kept but do not run.", + toggle: "WebhookTasks", + }, + { + id: "BrowserCookieImport", + title: "Allow importing browser cookies", + help: "Disabled stops users from importing cookies from another browser into T3 Code's browser.", + toggle: "BrowserCookieImport", + }, + { + id: "Telemetry", + title: "Allow usage data", + help: "Disabled stops T3 Code from sending product usage data and relay connection traces. Enabled cannot turn them on for a user who turned them off.", + toggle: "Telemetry", + }, + { + id: "AllowedDrivers", + title: "Allowed providers", + help: "Only the checked providers may run. Threads on other providers stay, marked unavailable.\n\nDisabled or Not Configured: no restriction.", + elements: knownDriverLabels().map(({ driver, label }) => ({ + kind: "checkbox" as const, + list: "AllowedDrivers" as const, + name: driver, + label, + defaultChecked: driver === "codex" || driver === "claudeAgent", + })), + }, + { + id: "Updates", + title: "Updates", + help: "Off stops T3 Code from checking for updates, for machines that receive T3 Code through device management.\n\nA feed URL makes the desktop app update from your own mirror of the release feed instead of GitHub; it must be https. Unchecking nightly makes the nightly channel unavailable.\n\nDisabled or Not Configured: no restriction.", + elements: [ + { + kind: "enum", + name: "UpdatesMode", + label: "Updates", + items: [ + { value: "auto", label: "Automatic" }, + { value: "off", label: "Off" }, + ], + defaultItem: 0, + }, + { kind: "text", name: "UpdatesFeedUrl", label: "Feed URL (optional)" }, + { + kind: "checkbox", + name: "AllowNightly", + label: "Allow the nightly channel", + defaultChecked: true, + }, + ], + }, + { + id: "Settings", + title: "Managed settings (JSON)", + help: 'Defaults and locks for server settings, as the "settings" object of a T3 Code policy. Each key is a setting path; each value is {"value": ..., "locked": true|false}.\n\nExample: {"enableAgentBrowserAccess": {"value": false, "locked": true}}\n\nRun "t3 policy check" on a policy file with the same object to validate it first.', + elements: [{ kind: "text", name: "Settings", label: "Settings JSON", json: true }], + }, + { + id: "ProviderInstances", + title: "Provider settings (JSON)", + help: 'Fields set on every instance of a provider, as the "limits.providers.instances" object of a T3 Code policy: environment variables, binaryPath, launchArgs.\n\nExample: {"claudeAgent": {"locked": true, "environment": [{"name": "CLAUDE_CODE_USE_BEDROCK", "value": "1"}]}}\n\nEvery user on the machine can read this value, so do not put credentials in it.', + elements: [ + { kind: "text", name: "ProviderInstances", label: "Provider settings JSON", json: true }, + ], + }, +]; + +const escapeXml = (text: string) => + text + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """); + +// ADMX keys are relative to the hive the policy class picks. +const ADMX_KEY = WINDOWS_POLICY_KEY.replace(/^HKLM\\/, ""); +const elementId = (element: Element) => + element.kind === "checkbox" && element.list !== undefined + ? `${element.list}_${element.name}` + : element.name; + +const admxElement = (element: Element) => { + const id = elementId(element); + switch (element.kind) { + case "text": + return ``; + case "enum": + return [ + ``, + ...element.items.map( + (item) => + ` ${item.value}`, + ), + "", + ].join("\n "); + case "checkbox": + return ``; + } +}; + +/** The language-neutral Group Policy template. */ +export function windowsAdmx(): string { + const body = policies().map((policy) => { + const head = `\n `; + if (policy.toggle !== undefined) { + return ` ${head} valueName="${policy.toggle}"> + ${common} + + + `; + } + const grouped = Object.hasOwn(WINDOWS_POLICY_GROUPS, policy.id); + return ` ${head}${grouped ? ` valueName="${policy.id}"` : ""} presentation="$(presentation.${policy.id})"> + ${common}${grouped ? '\n \n ' : ""} + + ${(policy.elements ?? []).map(admxElement).join("\n ")} + + `; + }); + return ` + + + + + + + + + + + + + + +${body.join("\n")} + + +`; +} + +const presentationElement = (element: Element) => { + const id = elementId(element); + switch (element.kind) { + case "text": + return ``; + case "enum": + return `${escapeXml(element.label)}`; + case "checkbox": + return `${escapeXml(element.label)}`; + } +}; + +/** The en-US strings and dialog layouts for `windowsAdmx`. */ +export function windowsAdml(): string { + const strings: Array = [ + ["T3Code", "T3 Code"], + ["SUPPORTED_T3Code", "T3 Code with managed policy support"], + ]; + const presentations: Array = []; + for (const policy of policies()) { + strings.push([policy.id, policy.title], [`${policy.id}_Help`, policy.help]); + for (const element of policy.elements ?? []) { + if (element.kind === "enum") { + for (const item of element.items) { + strings.push([`${elementId(element)}_${item.value}`, item.label]); + } + } + } + if (policy.elements !== undefined) { + presentations.push( + ` \n ${policy.elements.map(presentationElement).join("\n ")}\n `, + ); + } + } + return ` + + T3 Code + Policies for T3 Code + + +${strings.map(([id, text]) => ` ${escapeXml(text)}`).join("\n")} + + +${presentations.join("\n")} + + + +`; +} + +const plistValue = (value: unknown, indent: string): string => { + const inner = `${indent} `; + if (typeof value === "string") return `${indent}${escapeXml(value)}`; + if (typeof value === "boolean") return `${indent}<${value} />`; + if (typeof value === "number") { + return Number.isInteger(value) + ? `${indent}${value}` + : `${indent}${value}`; + } + if (Array.isArray(value)) { + return `${indent}\n${value.map((item) => plistValue(item, inner)).join("\n")}\n${indent}`; + } + const entries = Object.entries(value as Record); + return `${indent}\n${entries + .map(([key, item]) => `${inner}${escapeXml(key)}\n${plistValue(item, inner)}`) + .join("\n")}\n${indent}`; +}; + +/** Where `value` holds a `null`, as path segments, or undefined when it holds none. */ +const nullPath = ( + value: unknown, + path: ReadonlyArray, +): ReadonlyArray | undefined => { + if (value === null) return path; + if (typeof value !== "object") return undefined; + for (const [key, item] of Object.entries(value)) { + const found = nullPath(item, [...path, key]); + if (found !== undefined) return found; + } + return undefined; +}; + +/** A device configuration profile that installs `document` for `com.t3tools.t3code`. */ +export function macosProfile( + document: ManagedPolicyDocument, + uuids: { readonly profile: string; readonly payload: string }, +): + | { readonly _tag: "Valid"; readonly profile: string } + | { readonly _tag: "Invalid"; readonly reason: string } { + // A plist has no null, and a setting may be locked empty. + const empty = nullPath(document, []); + if (empty !== undefined) { + const setting = + empty[0] === "settings" && empty.at(-1) === "value" ? empty.slice(0, -1) : empty; + return { + _tag: "Invalid", + reason: `${setting.join(".")} is null, which a configuration profile cannot hold; deploy this policy as a file.`, + }; + } + const profile = { + PayloadContent: [ + { + PayloadDisplayName: "T3 Code policy", + PayloadIdentifier: `com.t3tools.t3code.policy.${uuids.payload}`, + PayloadType: "com.t3tools.t3code", + PayloadUUID: uuids.payload, + PayloadVersion: 1, + ...document, + }, + ], + PayloadDescription: "Configures T3 Code for every user on this Mac.", + PayloadDisplayName: `T3 Code policy for ${document.organization}`, + PayloadIdentifier: `com.t3tools.t3code.policy`, + PayloadOrganization: document.organization, + PayloadScope: "System", + PayloadType: "Configuration", + PayloadUUID: uuids.profile, + PayloadVersion: 1, + }; + return { + _tag: "Valid", + profile: ` + + +${plistValue(profile, "")} + +`, + }; +} diff --git a/apps/server/src/preview/PortScanner.ts b/apps/server/src/preview/PortScanner.ts index e026b204c03d8..b2b6e1ac9fbe0 100644 --- a/apps/server/src/preview/PortScanner.ts +++ b/apps/server/src/preview/PortScanner.ts @@ -24,6 +24,7 @@ import * as Scope from "effect/Scope"; import * as Semaphore from "effect/Semaphore"; import { FetchHttpClient, HttpClient } from "effect/http"; +import * as WindowsTcpListeners from "../process/windowsTcpListeners.ts"; import * as ProcessRunner from "../processRunner.ts"; export class PortDiscovery extends Context.Service< @@ -76,8 +77,6 @@ const LSOF_TIMEOUT_MS = 5_000; const PROC_FD_WALK_LIMIT = 50_000; const SOCKET_OWNER_ATTEMPTS = 3; const WINDOWS_LISTENER_TIMEOUT_MS = 5_000; -/** `netstat -ano` lists every connection, not just listeners; ~80 bytes per row, sized for ~200k rows. */ -const WINDOWS_NETSTAT_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; const WEB_PROBE_TIMEOUT = Duration.seconds(1); const WEB_PROBE_CACHE_TTL_MS = Duration.toMillis(Duration.seconds(15)); /** Bounds how long a cached Windows image name can outlive its process if Windows reuses the PID. */ @@ -272,53 +271,6 @@ const parsePortFromLsofName = (name: string): number | null => { return port; }; -const parseWindowsListenerLine = ( - line: string, -): { - readonly host: string; - readonly port: number; - readonly pid: number | null; - readonly processName: string | null; -} | null => { - const trimmed = line.trim(); - if (trimmed.length === 0) return null; - - if (trimmed.includes("|")) { - const [hostRaw, portRaw, pidRaw, processNameRaw] = trimmed.split("|", 4); - const host = hostRaw?.trim() ?? ""; - if (!LSOF_LOCAL_HOST_TOKENS.has(host) && host !== "::") return null; - const port = Number(portRaw); - const pid = Number(pidRaw); - if (!Number.isInteger(port) || port <= 0 || port >= 65536) return null; - return { - host, - port, - pid: Number.isInteger(pid) && pid > 0 ? pid : null, - processName: processNameRaw?.trim() || null, - }; - } - - // Foreign port `:0` identifies a listening TCP socket regardless of Windows locale. - const fields = trimmed.split(/\s+/); - if (fields.length < 4 || fields[0]?.toUpperCase() !== "TCP") return null; - const localEndpoint = fields[1] ?? ""; - const foreignEndpoint = fields[2] ?? ""; - if (!foreignEndpoint.endsWith(":0")) return null; - const lastColon = localEndpoint.lastIndexOf(":"); - if (lastColon <= 0) return null; - const host = localEndpoint.slice(0, lastColon); - if (!LSOF_LOCAL_HOST_TOKENS.has(host) && host !== "::") return null; - const port = Number(localEndpoint.slice(lastColon + 1)); - if (!Number.isInteger(port) || port <= 0 || port >= 65536) return null; - const pid = Number(fields.at(-1)); - return { - host, - port, - pid: Number.isInteger(pid) && pid > 0 ? pid : null, - processName: null, - }; -}; - const exitedWithError = (result: ProcessRunner.ProcessRunOutput): boolean => result.code !== null && result.code !== 0; @@ -330,16 +282,13 @@ const parseTasklistProcessName = (raw: string, pid: number): string | null => { return null; }; -const parseWindowsListenerOutput = ( - raw: string, - owners: ReadonlyMap = new Map(), +const discoveredWindowsListeners = ( + listeners: ReadonlyArray, + owners: ReadonlyMap, ): ReadonlyArray => { const seen = new Map(); - for (const line of raw.split(/\r?\n/g)) { - const parsed = parseWindowsListenerLine(line); - if (!parsed) continue; - const { port, pid: normalizedPid, processName } = parsed; - const owner = normalizedPid === null ? undefined : owners.get(normalizedPid); + for (const { port, pid } of listeners) { + const owner = pid === null ? undefined : owners.get(pid); const existing = seen.get(port); if (existing) { if (owner === undefined) seen.set(port, { ...existing, ...UNOWNED }); @@ -349,8 +298,8 @@ const parseWindowsListenerOutput = ( host: "localhost", port, url: `http://localhost:${port}`, - processName, - pid: normalizedPid, + processName: null, + pid, ...(owner ?? UNOWNED), }); } @@ -857,21 +806,12 @@ export const make = Effect.gen(function* PortDiscoveryMake() { agentsPossible ? withAgentOwners(terminalOwners, pids) : Effect.succeed(terminalOwners); if (hostPlatform === "win32") { const recoverWindowsProbeFailure = recoverProcessProbeFailure("windows-listeners"); - const runNetstat = (protocol: "tcp" | "tcpv6") => - processRunner.run({ - command: "netstat.exe", - args: ["-ano", "-p", protocol], - timeout: Duration.millis(WINDOWS_LISTENER_TIMEOUT_MS), - maxOutputBytes: WINDOWS_NETSTAT_MAX_OUTPUT_BYTES, - }); - const listeners = yield* Effect.all([runNetstat("tcp"), runNetstat("tcpv6")], { - concurrency: 2, - }).pipe( - // Failed queries can hide other IPv4/IPv6 listeners on the port; fall back to configured URLs. - Effect.map(([tcp4, tcp6]) => - exitedWithError(tcp4) || exitedWithError(tcp6) - ? null - : parseWindowsListenerOutput(`${tcp4.stdout}\n${tcp6.stdout}`, terminalOwners), + const listeners = yield* WindowsTcpListeners.queryWindowsTcpListeners( + processRunner, + "netstat.exe", + ).pipe( + Effect.map((snapshot) => + snapshot === null ? null : discoveredWindowsListeners(snapshot, terminalOwners), ), Effect.catchTags({ ProcessSpawnError: recoverWindowsProbeFailure, diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 5736e62e795e5..1d7a5f9058368 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -53,6 +53,12 @@ vi.mock("./ServerBrowserContexts.ts", () => ({ if (contextFailure) throw contextFailure; await contextGate?.promise; const context = makeContext(this.onClose); + const setupFailure = pageSetupFailure; + if (setupFailure) { + context.newCDPSession = async () => { + throw setupFailure; + }; + } contexts.push(context); return context as unknown as BrowserContext; } @@ -299,6 +305,7 @@ let pageCopy = ""; let keyCopy = ""; let focusedFrame: ReturnType | null = null; let contextFailure: Error | null = null; +let pageSetupFailure: Error | null = null; let initialNavigation: (() => Promise) | null = null; let desktopRendersNext = false; let desktopPageUrl = "about:blank"; @@ -443,6 +450,7 @@ beforeEach(() => { contextRequests.length = 0; contextGate = null; contextFailure = null; + pageSetupFailure = null; initialNavigation = null; pageCopy = ""; keyCopy = ""; @@ -482,7 +490,8 @@ it.live("tells the desktop once per thread when its agent turns start and end", it.live("preview_open reports its navigation timeout instead of the request budget", () => Effect.scoped( Effect.gen(function* () { - const { broker } = yield* ready; + const { broker, tabId } = yield* ready; + const manager = yield* Manager.PreviewManager; initialNavigation = async () => { throw Object.assign(new Error("page.goto: Timeout 15000ms exceeded."), { name: "TimeoutError", @@ -506,6 +515,54 @@ it.live("preview_open reports its navigation timeout instead of the request budg timeoutMs: 15_000, }); expect(error.message).toBe("Preview automation open timed out after 15000ms."); + // The failed open returned no tabId, so its tab must not linger and block later opens. + const { sessions } = yield* manager.list({ threadId: scope.thread.threadId }); + expect(sessions.map((session) => session.tabId)).toEqual([tabId]); + expect(contexts[1]!.page.close).toHaveBeenCalled(); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("an initial download stays available after preview_open reports the navigation error", () => + Effect.scoped( + Effect.gen(function* () { + const { broker, tabId } = yield* ready; + const manager = yield* Manager.PreviewManager; + const released = Promise.withResolvers(); + initialNavigation = async () => { + contexts[1]!.page.emit("download", { + failure: async () => null, + delete: async () => released.resolve(), + saveAs: (path: string) => NodeFSP.writeFile(path, "a,b"), + suggestedFilename: () => "report.csv", + url: () => "http://localhost:1337/report.csv", + }); + throw new Error("page.goto: Download is starting"); + }; + const error = yield* broker + .invoke({ + scope, + operation: "open", + input: { url: "http://localhost:1337/report.csv", reuseExistingTab: false, open: false }, + }) + .pipe(Effect.flip); + expect(error.message).toContain("Download is starting"); + yield* Effect.promise(() => released.promise); + const { sessions } = yield* manager.list({ threadId: scope.thread.threadId }); + expect(sessions).toHaveLength(2); + const downloadTabId = PreviewTabId.make(sessions.find((tab) => tab.tabId !== tabId)!.tabId); + const status = yield* broker.invoke({ + scope, + tabId: downloadTabId, + operation: "status", + input: {}, + }); + expect(status.downloads).toEqual([ + expect.objectContaining({ fileName: "report.csv", sizeBytes: 3 }), + ]); + expect(yield* (yield* FileSystem.FileSystem).readFileString(status.downloads![0]!.path)).toBe( + "a,b", + ); }), ).pipe(Effect.provide(layer)), ); @@ -2636,6 +2693,13 @@ it.live("failed session and browser launches release capacity, as does closing a // A failed launch must not leave a session a viewer can relaunch past the budget. expect((yield* manager.list({ threadId: scope.thread.threadId })).sessions).toHaveLength(7); contextFailure = null; + pageSetupFailure = new Error("Target page, context or browser has been closed"); + expect(yield* open.pipe(Effect.result)).toMatchObject({ _tag: "Failure" }); + pageSetupFailure = null; + // Setup that fails after the page exists must not leave the page or its context open. + expect(contexts.at(-1)!.close).toHaveBeenCalled(); + expect(contexts.at(-1)!.page.isClosed()).toBe(true); + expect((yield* manager.list({ threadId: scope.thread.threadId })).sessions).toHaveLength(7); expect((yield* open).available).toBe(true); expect(yield* open.pipe(Effect.flip)).toMatchObject({ reason: "tabLimit" }); expect( diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 15221698ead02..afc2934cc2fd2 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -470,7 +470,11 @@ const make = Effect.gen(function* () { const runFork = Effect.runForkWith(yield* Effect.context()); const runPromise = Effect.runPromiseWith(yield* Effect.context()); const launchServices = yield* Effect.context(); - const setupCommand = yield* resolveRootCliCommand(PreviewBrowserHost.SETUP_SUBCOMMAND); + // sudo resets the environment, so the command has to name this server's home. + const setupCommand = yield* resolveRootCliCommand(PreviewBrowserHost.SETUP_SUBCOMMAND, [ + "--base-dir", + config.baseDir, + ]); const agentScope = yield* AgentScope; const launchersDir = NodePath.join(config.stateDir, "server-browser", "launchers"); @@ -781,12 +785,27 @@ const make = Effect.gen(function* () { }; const createTab = async (snapshot: PreviewSessionSnapshot): Promise => { + const opened: Array<() => Promise> = []; + try { + return await setUpTab(snapshot, opened); + } catch (cause) { + // An unpublished tab is in no list, so nothing else would ever close it. + for (const close of opened.reverse()) await close().catch(constVoid); + throw cause; + } + }; + + const setUpTab = async ( + snapshot: PreviewSessionSnapshot, + opened: Array<() => Promise>, + ): Promise => { const adopted = adoptedPages.get(tabKey(snapshot.threadId, snapshot.tabId)); adoptedPages.delete(tabKey(snapshot.threadId, snapshot.tabId)); const desktop = adopted === undefined && (await desktopRenders(snapshot)) ? await connectDesktop(snapshot) : null; + if (desktop) opened.push(desktop.close); // Otherwise a desktop that attaches late shows its own page, and the agent drives another. if (!desktop && desktopChannel.available) { await runPromise( @@ -807,9 +826,11 @@ const make = Effect.gen(function* () { snapshot.profileId ?? "default", isolatedContext ? tabKey(snapshot.threadId, snapshot.tabId) : undefined, )); + if (isolatedContext) opened.push(() => context.close()); if (adopted?.page.isClosed()) throw new Error("The popup closed before it opened."); if (!desktop) await prepareContext(context); const page = adopted?.page ?? desktop?.page ?? (await context.newPage()); + if (!desktop) opened.push(() => page.close()); const cdp = await context.newCDPSession(page); await installPdfDocumentNavigation(cdp); // The desktop's page is already a normal browser. A popup's first request @@ -983,9 +1004,6 @@ const make = Effect.gen(function* () { const key = tabKey(tab.threadId, tab.tabId); if (closedPendingTabs.delete(key)) { await control.close().catch(constVoid); - if (desktop) await desktop.close().catch(constVoid); - else await page.close().catch(constVoid); - if (isolatedContext) await context.close().catch(constVoid); throw new ServerBrowserTabNotFoundError({ threadId: tab.threadId, tabId: tab.tabId }); } page.on("close", () => dropTab(tab, true)); @@ -2015,8 +2033,16 @@ const make = Effect.gen(function* () { if (existing) { if (url) await navigate(tab, url, "load", navigationTimeout); } else { - // Await the original navigation failure even though background creation keeps the tab. - await tab.initialNavigation; + // A failed open returns no tabId; a leftover hidden tab would block later reuse. + await tab.initialNavigation?.catch(async (cause: unknown) => { + // Playwright rejects downloads too; their tab must survive to save the file. + if (cause instanceof Error && cause.message.includes("Download is starting")) { + throw cause; + } + await runPromise(manager.close({ threadId: tab.threadId, tabId: tab.tabId })); + dropTab(tab, false); + throw cause; + }); } signal.throwIfAborted(); const reveal = open.open ?? open.show; diff --git a/apps/server/src/process/windowsTcpListeners.ts b/apps/server/src/process/windowsTcpListeners.ts new file mode 100644 index 0000000000000..789915fa4b193 --- /dev/null +++ b/apps/server/src/process/windowsTcpListeners.ts @@ -0,0 +1,61 @@ +import { LSOF_LOCAL_HOST_TOKENS } from "@t3tools/shared/preview"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; + +import type * as ProcessRunner from "../processRunner.ts"; + +const NETSTAT_TIMEOUT = Duration.seconds(5); +/** `netstat -ano` lists every connection, not just listeners; ~80 bytes per row, sized for ~200k rows. */ +const NETSTAT_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; + +export interface WindowsTcpListener { + readonly port: number; + readonly pid: number | null; +} + +const parseWindowsTcpListenerLine = (line: string): WindowsTcpListener | null => { + // Foreign port `:0` identifies a listening TCP socket regardless of Windows locale. + const fields = line.trim().split(/\s+/); + if (fields.length < 4 || fields[0]?.toUpperCase() !== "TCP") return null; + const localEndpoint = fields[1] ?? ""; + const foreignEndpoint = fields[2] ?? ""; + if (!foreignEndpoint.endsWith(":0")) return null; + const lastColon = localEndpoint.lastIndexOf(":"); + if (lastColon <= 0) return null; + const host = localEndpoint.slice(0, lastColon); + if (!LSOF_LOCAL_HOST_TOKENS.has(host) && host !== "::") return null; + const port = Number(localEndpoint.slice(lastColon + 1)); + if (!Number.isInteger(port) || port <= 0 || port >= 65536) return null; + const pid = Number(fields.at(-1)); + return { port, pid: Number.isInteger(pid) && pid > 0 ? pid : null }; +}; + +const parseWindowsTcpListeners = (raw: string): ReadonlyArray => + raw.split(/\r?\n/g).flatMap((line) => { + const listener = parseWindowsTcpListenerLine(line); + return listener === null ? [] : [listener]; + }); + +const exitedWithError = (result: ProcessRunner.ProcessRunOutput): boolean => + result.code !== null && result.code !== 0; + +/** Resolves to `null` when either query exits nonzero. */ +export const queryWindowsTcpListeners = Effect.fn("windowsTcpListeners.query")(function* ( + processRunner: ProcessRunner.ProcessRunner["Service"], + netstatCommand: string, +) { + const runNetstat = (protocol: "tcp" | "tcpv6") => + processRunner.run({ + command: netstatCommand, + args: ["-ano", "-p", protocol], + timeout: NETSTAT_TIMEOUT, + maxOutputBytes: NETSTAT_MAX_OUTPUT_BYTES, + }); + const [tcp4, tcp6] = yield* Effect.all([runNetstat("tcp"), runNetstat("tcpv6")], { + concurrency: 2, + }); + // One failed query leaves a partial snapshot that could hide another process sharing a + // port across IPv4/IPv6. + if (exitedWithError(tcp4) || exitedWithError(tcp6)) return null; + return parseWindowsTcpListeners(`${tcp4.stdout}\n${tcp6.stdout}`); +}); diff --git a/apps/server/src/project/ManagedProjectFolders.test.ts b/apps/server/src/project/ManagedProjectFolders.test.ts index dc4e947f294cc..93e280bf83685 100644 --- a/apps/server/src/project/ManagedProjectFolders.test.ts +++ b/apps/server/src/project/ManagedProjectFolders.test.ts @@ -351,6 +351,12 @@ it.effect("starts a named project as a committed repository, and suffixes a take const path = yield* Path.Path; const root = path.resolve(baseDir, "projects"); assert.equal(folders.namedProjectsRoot, root); + const attributes = path.join(baseDir, "crlf.gitattributes"); + const crlfConfig = path.join(baseDir, "crlf.gitconfig"); + yield* fileSystem.writeFileString(attributes, "* text=auto eol=crlf\n"); + yield* git(baseDir, ["config", "--file", crlfConfig, "core.safecrlf", "true"]); + yield* git(baseDir, ["config", "--file", crlfConfig, "core.attributesFile", attributes]); + process.env.GIT_CONFIG_GLOBAL = crlfConfig; const first = yield* folders.createNamedProject({ name: "Pinball Stats" }); const second = yield* folders.createNamedProject({ name: "pinball stats" }); diff --git a/apps/server/src/project/ManagedProjectFolders.ts b/apps/server/src/project/ManagedProjectFolders.ts index c187424870c61..4b0991e1a64ce 100644 --- a/apps/server/src/project/ManagedProjectFolders.ts +++ b/apps/server/src/project/ManagedProjectFolders.ts @@ -395,11 +395,12 @@ const make = Effect.gen(function* () { path.join(cwd, "assets", "icon.svg"), namedProjectIconSvg(name), ); - // Force explicit filenames so global ignore rules cannot drop them. + // Force explicit filenames so global ignore rules cannot drop them, and keep + // core.safecrlf=true from refusing the LF files under CRLF conversion. yield* git.execute({ operation: "ManagedProjectFolders.add", cwd, - args: ["add", "--force", "--", "README.md", "assets/icon.svg"], + args: ["-c", "core.safecrlf=false", "add", "--force", "--", "README.md", "assets/icon.svg"], timeoutMs: 10_000, }); return yield* git diff --git a/apps/server/src/project/ProjectService.deletion.test.ts b/apps/server/src/project/ProjectService.deletion.test.ts index 41804842d5205..28b1c40541ddf 100644 --- a/apps/server/src/project/ProjectService.deletion.test.ts +++ b/apps/server/src/project/ProjectService.deletion.test.ts @@ -22,7 +22,7 @@ import * as EventSink from "../orchestration-v2/EventSink.ts"; import * as EventStore from "../orchestration-v2/EventStore.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as LegacyV1ThreadImporter from "../orchestration-v2/legacy/LegacyV1ThreadImporter.ts"; -import * as ProjectionMaintenance from "../orchestration-v2/ProjectionMaintenance.ts"; +import * as ProjectionReplay from "../orchestration-v2/testkit/ProjectionReplay.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; @@ -40,7 +40,7 @@ const layerEventPersistence = EventSink.layer.pipe( ); const layerServices = Layer.mergeAll( LegacyV1ThreadImporter.layer.pipe(Layer.provideMerge(layerEventPersistence)), - ProjectionMaintenance.layer.pipe(Layer.provide(layerEventPersistence)), + ProjectionReplay.layer.pipe(Layer.provide(layerEventPersistence)), ProjectStore.layer, IdAllocator.layer, ThreadCommandExecutor.layer, @@ -310,7 +310,7 @@ it.effect( yield* Effect.gen(function* () { const importer = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter; - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const maintenance = yield* ProjectionReplay.ProjectionReplay; const projections = yield* ProjectionStore.ProjectionStoreV2; const service = yield* ProjectService.make; assert.deepEqual(yield* importer.reconcileShells, { diff --git a/apps/server/src/project/RepositoryIdentityResolver.test.ts b/apps/server/src/project/RepositoryIdentityResolver.test.ts index 61e721a6d465b..0ad038b588a57 100644 --- a/apps/server/src/project/RepositoryIdentityResolver.test.ts +++ b/apps/server/src/project/RepositoryIdentityResolver.test.ts @@ -51,6 +51,25 @@ it.layer(NodeServices.layer)("RepositoryIdentityResolverLive", (it) => { }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), ); + it.effect("identifies the host of an SSH remote that leaves its user to ssh config", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-userless-remote-test-", + }); + + yield* git(cwd, ["init"]); + yield* git(cwd, ["remote", "add", "origin", "github.com:T3Tools/t3code.git"]); + + const resolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; + const identity = yield* resolver.resolve(cwd); + + expect(identity?.canonicalKey).toBe("github.com/t3tools/t3code"); + expect(identity?.provider).toBe("github"); + expect(identity?.displayName).toBe("T3Tools/t3code"); + }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), + ); + it.effect("resolves the fetch remote of a partial clone", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; diff --git a/apps/server/src/provider/AntigravityAuth.test.ts b/apps/server/src/provider/AntigravityAuth.test.ts index 4664c299e8a5c..d7099ec2bfa41 100644 --- a/apps/server/src/provider/AntigravityAuth.test.ts +++ b/apps/server/src/provider/AntigravityAuth.test.ts @@ -84,17 +84,21 @@ const makeHarness = Effect.fn("makeAuthTestHarness")(function* ( yield* Deferred.succeed(closed, undefined); }), ); + // Like AcpSessionRuntime, `start` reuses one memoized initialize handshake. + const initialize = yield* Effect.cached( + Effect.gen(function* () { + events.push("initialize"); + yield* options.beforeInitialize ?? Effect.void; + return options.supportsLogout === false + ? { ...initialized, agentCapabilities: {} } + : initialized; + }), + ); return { - initialize: () => - Effect.gen(function* () { - events.push("initialize"); - yield* options.beforeInitialize ?? Effect.void; - return options.supportsLogout === false - ? { ...initialized, agentCapabilities: {} } - : initialized; - }), + initialize: () => initialize, start: () => Effect.gen(function* () { + yield* initialize; events.push("authenticate"); yield* Deferred.succeed(authenticateCalled, undefined); if (options.interactive !== false && input.onAuthorizationUrl) { @@ -280,6 +284,42 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("AntigravityAuth" }), ); + it.effect("does not blame the network while the runtime is still starting", () => + Effect.gen(function* () { + const initializing = yield* Deferred.make(); + const ready = yield* Deferred.make(); + const harness = yield* makeHarness({ + interactive: false, + beforeInitialize: Deferred.succeed(initializing, undefined).pipe( + Effect.andThen(Deferred.await(ready)), + ), + }); + const hasHint = harness.auth.controller.subscribe(owner).pipe( + Stream.runHead, + Effect.map((state) => Option.getOrThrow(state).message?.includes("Broken IPv6") === true), + ); + yield* harness.auth.controller.start(owner); + yield* Deferred.await(initializing); + + // A one-file runtime can take about a minute to unpack before it answers initialize. + yield* TestClock.adjust("60 seconds"); + assert.isFalse(yield* hasHint); + + yield* Deferred.succeed(ready, undefined); + yield* Deferred.await(harness.authenticateCalled); + yield* TestClock.adjust("29 seconds"); + assert.isFalse(yield* hasHint); + + yield* TestClock.adjust("1 second"); + const stalled = yield* harness.auth.controller.subscribe(owner).pipe( + Stream.filter((state) => state.message?.includes("Broken IPv6") === true), + Stream.runHead, + Effect.map(Option.getOrThrow), + ); + assert.equal(stalled.phase, "starting"); + }), + ); + it.effect("keeps the expired message once the Google link was shown", () => Effect.gen(function* () { const harness = yield* makeHarness(); @@ -385,7 +425,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("AntigravityAuth" assert.isTrue(Exit.isFailure(competing)); const resumed = yield* phase(harness.auth, "waiting"); assert.equal(resumed.flowId, first.flowId); - assert.deepEqual(harness.events, ["process-open", "authenticate"]); + assert.deepEqual(harness.events, ["process-open", "initialize", "authenticate"]); yield* harness.auth.controller.cancel(owner, first.flowId!); }), ); diff --git a/apps/server/src/provider/AntigravityAuth.ts b/apps/server/src/provider/AntigravityAuth.ts index 88367f5b06e99..6fd3d29162fee 100644 --- a/apps/server/src/provider/AntigravityAuth.ts +++ b/apps/server/src/provider/AntigravityAuth.ts @@ -31,8 +31,10 @@ import { import type { ProviderAuthController } from "./ProviderAuthService.ts"; const AUTH_TIMEOUT_MS = 300_000; -// A healthy agent shows the Google link or finishes within seconds. Its bundled Python has no -// IPv4 fallback race, so a host with broken IPv6 waits about two minutes per Google request. +// Once initialized, a healthy agent shows the Google link or finishes within seconds. Its bundled +// Python has no IPv4 fallback race, so a host with broken IPv6 waits about two minutes per Google +// request. The hint is timed from initialize, not spawn: the one-file runtime unpacks itself on +// every launch, which can take about a minute on Windows before it answers at all. const NO_RESPONSE_HINT_MS = 30_000; const NO_RESPONSE_MESSAGE = "Antigravity has not reached Google after 30 seconds. Check this machine's network connection. Broken IPv6 is a common cause."; @@ -271,6 +273,11 @@ export const makeAntigravityAuth = Effect.fn("makeAntigravityAuth")(function* < const runSignIn = Effect.fn("AntigravityAuth.runSignIn")( function* (flow: AuthFlow, stopSessions: Effect.Effect) { yield* stopSessions.pipe(Effect.ensuring(stopOwnedProcesses)); + const runtime = yield* options.makeRuntime({ + onAuthorizationUrl: (url) => receiveAuthorizationUrl(flow, url), + }); + // `start` reuses this memoized handshake. + yield* runtime.initialize(); yield* Effect.sleep(NO_RESPONSE_HINT_MS).pipe( Effect.andThen( lock.withPermits(1)( @@ -282,9 +289,6 @@ export const makeAntigravityAuth = Effect.fn("makeAntigravityAuth")(function* < ), Effect.forkScoped, ); - const runtime = yield* options.makeRuntime({ - onAuthorizationUrl: (url) => receiveAuthorizationUrl(flow, url), - }); const started = yield* runtime.start(); yield* lock.withPermits(1)( Effect.gen(function* () { diff --git a/apps/server/src/provider/AntigravityProvider.test.ts b/apps/server/src/provider/AntigravityProvider.test.ts index 44f8842fd9591..791ed748de2ef 100644 --- a/apps/server/src/provider/AntigravityProvider.test.ts +++ b/apps/server/src/provider/AntigravityProvider.test.ts @@ -348,6 +348,21 @@ it.layer(layerTest)("Antigravity provider snapshots", (it) => { ), ); + it.effect("does not offer Auto, so Auto threads fall back to Supervised", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + // Antigravity has no automatic review; the shared ACP policy would approve every request. + expect((yield* harness.provider.snapshot.getSnapshot).supportedRuntimeModes).toEqual([ + "approval-required", + "auto-accept-edits", + "full-access", + ]); + }), + ), + ); + it.effect("publishes session metadata and native commands without another health probe", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/provider/AntigravityProvider.ts b/apps/server/src/provider/AntigravityProvider.ts index ff8805f43b371..ed047331e73ee 100644 --- a/apps/server/src/provider/AntigravityProvider.ts +++ b/apps/server/src/provider/AntigravityProvider.ts @@ -182,7 +182,12 @@ export const makeAntigravityProvider = Effect.fn("makeAntigravityProvider")(func const checkedAt = DateTime.formatIso(yield* DateTime.now); const initialDraft = { ...buildServerProvider({ - presentation: { displayName: "Antigravity", showInteractionModeToggle: false }, + presentation: { + displayName: "Antigravity", + showInteractionModeToggle: false, + // No automatic review, so Auto falls back to Supervised instead of approving everything. + supportedRuntimeModes: ["approval-required", "auto-accept-edits", "full-access"], + }, enabled: settings.enabled, checkedAt, models: [], diff --git a/apps/server/src/provider/ClaudeTurnTokenUsage.ts b/apps/server/src/provider/ClaudeTurnTokenUsage.ts index 2fa6a11bac1e5..118716c55c26d 100644 --- a/apps/server/src/provider/ClaudeTurnTokenUsage.ts +++ b/apps/server/src/provider/ClaudeTurnTokenUsage.ts @@ -77,6 +77,19 @@ export function claudeModelLedger(modelUsage: unknown): ClaudeModelLedger | unde return ledger; } +/** The window each model in the SDK's `modelUsage` is measured against, as Claude Code resolved it. */ +export function claudeReportedContextWindows(modelUsage: unknown): ReadonlyMap { + const windows = new Map(); + if (typeof modelUsage !== "object" || modelUsage === null) return windows; + for (const [model, entry] of Object.entries(modelUsage as Record)) { + const window = finiteNonNegativeInteger( + (entry as Record | null)?.contextWindow, + ); + if (window !== undefined && window > 0) windows.set(model, window); + } + return windows; +} + /** Ledger growth since `previous`. Unknown without a baseline or when a counter goes backwards. */ export function claudeLedgerDelta( previous: ClaudeModelLedger | undefined, diff --git a/apps/server/src/provider/CodexInstallation.test.ts b/apps/server/src/provider/CodexInstallation.test.ts index ab266b7a8c01d..daf1dac2654de 100644 --- a/apps/server/src/provider/CodexInstallation.test.ts +++ b/apps/server/src/provider/CodexInstallation.test.ts @@ -359,6 +359,33 @@ it.effect("requires an update before running an older activated managed installa }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); +it.effect("updates an older supported managed installation to the pinned release on startup", () => + Effect.gen(function* () { + const first = yield* makeHarness(); + yield* first.installation.start; + yield* terminalState(first.installation); + const executable = yield* first.installation.resolve(); + const oldDirectory = `${first.installation.managedDirectory}/0.156.0`; + yield* first.fs.rename(executable.managedVersionDirectory!, oldDirectory); + for (const name of ["codex-package.json", ".install-complete.json"]) { + const file = `${oldDirectory}/${name}`; + yield* first.fs.writeFileString( + file, + (yield* first.fs.readFileString(file)).replaceAll("0.156.1", "0.156.0"), + ); + } + yield* first.fs.writeFileString( + `${first.installation.managedDirectory}/active.json`, + '{"version":"0.156.0"}', + ); + const restarted = yield* makeHarness({ baseDir: first.baseDir }); + expect((yield* restarted.installation.state).operationId).not.toBeNull(); + expect((yield* terminalState(restarted.installation)).phase).toBe("succeeded"); + expect((yield* restarted.installation.resolve()).version).toBe("0.156.1"); + expect(restarted.downloads()).toBe(1); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), +); + it.effect("rejects corrupted downloads without publishing a runtime", () => Effect.gen(function* () { const { installation, fs } = yield* makeHarness({ diff --git a/apps/server/src/provider/CodexInstallation.ts b/apps/server/src/provider/CodexInstallation.ts index 83e9e4e15864d..8fdcfbebb328e 100644 --- a/apps/server/src/provider/CodexInstallation.ts +++ b/apps/server/src/provider/CodexInstallation.ts @@ -5,6 +5,7 @@ import { type ServerProviderCompatibilityAdvisory, } from "@t3tools/contracts"; import { cliArchiveTarCommand } from "@t3tools/shared/cliRelease"; +import { compareSemverVersions } from "@t3tools/shared/semver"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { resolveCommandPath, resolveSpawnCommand } from "@t3tools/shared/shell"; import * as Clock from "effect/Clock"; @@ -58,46 +59,46 @@ export interface CodexReleaseAsset { } const RELEASES: Readonly> = { "darwin-arm64": { - version: "0.156.1", + version: "0.166.0", target: "aarch64-apple-darwin", - url: "https://github.com/openai/codex/releases/download/rust-v0.156.1/codex-package-aarch64-apple-darwin.tar.gz", - sha256: "fea42f9625091f011e38f059da974d52e57ba31831648bb1c7f0b1a385fde547", - archiveBytes: 127394863, + url: "https://github.com/openai/codex/releases/download/rust-v0.166.0/codex-package-aarch64-apple-darwin.tar.gz", + sha256: "55a440cb579f970abe7c79ab181e54b0ea4c419be99f333bbd868eebac9c21f0", + archiveBytes: 165951869, }, "darwin-x64": { - version: "0.156.1", + version: "0.166.0", target: "x86_64-apple-darwin", - url: "https://github.com/openai/codex/releases/download/rust-v0.156.1/codex-package-x86_64-apple-darwin.tar.gz", - sha256: "618dbcd55419fa041871f777a14b107ceb3fe2d339ef81e21e6ab5374420dc71", - archiveBytes: 138670455, + url: "https://github.com/openai/codex/releases/download/rust-v0.166.0/codex-package-x86_64-apple-darwin.tar.gz", + sha256: "1cb65690230be5982c654ba2655e3a3b594dce9c55b780e78bfdfeff829d7ad9", + archiveBytes: 180076524, }, "linux-arm64": { - version: "0.156.1", + version: "0.166.0", target: "aarch64-unknown-linux-musl", - url: "https://github.com/openai/codex/releases/download/rust-v0.156.1/codex-package-aarch64-unknown-linux-musl.tar.gz", - sha256: "fdd47ed6aade0360796fd3f6f95a45096f327c15e19e8c7339f9dc5633041786", - archiveBytes: 136933361, + url: "https://github.com/openai/codex/releases/download/rust-v0.166.0/codex-package-aarch64-unknown-linux-musl.tar.gz", + sha256: "7821b77dda2e6283fb1f63f3165177a271da8833e857cf673c6ca1c9db3bfcc6", + archiveBytes: 187370368, }, "linux-x64": { - version: "0.156.1", + version: "0.166.0", target: "x86_64-unknown-linux-musl", - url: "https://github.com/openai/codex/releases/download/rust-v0.156.1/codex-package-x86_64-unknown-linux-musl.tar.gz", - sha256: "8b711520beddf385467b8da4d2c93736637c6ba1e46811cf0d8606b7c490b6f6", - archiveBytes: 145976992, + url: "https://github.com/openai/codex/releases/download/rust-v0.166.0/codex-package-x86_64-unknown-linux-musl.tar.gz", + sha256: "1d44b28bc4ac9937d7d3532264baa68a507ce364956a0d4ebb856f331cdee254", + archiveBytes: 199694962, }, "win32-arm64": { - version: "0.156.1", + version: "0.166.0", target: "aarch64-pc-windows-msvc", - url: "https://github.com/openai/codex/releases/download/rust-v0.156.1/codex-package-aarch64-pc-windows-msvc.tar.gz", - sha256: "85994caecdc7609c49fd585c1cdb5677fa9d0acbf789650cff23a13afc9505db", - archiveBytes: 142037952, + url: "https://github.com/openai/codex/releases/download/rust-v0.166.0/codex-package-aarch64-pc-windows-msvc.tar.gz", + sha256: "9383152a72f97bb5dd148493c8214e4fed38d805aa2c3297cb3739f7f130becf", + archiveBytes: 187024318, }, "win32-x64": { - version: "0.156.1", + version: "0.166.0", target: "x86_64-pc-windows-msvc", - url: "https://github.com/openai/codex/releases/download/rust-v0.156.1/codex-package-x86_64-pc-windows-msvc.tar.gz", - sha256: "a2e017db9807e6a2269a26fea0e1d9546469cef4d472a33016bc9f3ad7d3b733", - archiveBytes: 153839991, + url: "https://github.com/openai/codex/releases/download/rust-v0.166.0/codex-package-x86_64-pc-windows-msvc.tar.gz", + sha256: "d559bb88e74d471265857b93b7f19bc250dae527673b9191b33534478093f6bc", + archiveBytes: 202589789, }, }; const resolveCodexReleaseAsset = (platform: NodeJS.Platform, arch: string) => @@ -655,14 +656,12 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function } const operationId = yield* random.uuidV4(); const next: ProviderInstallState = { - driver: DRIVER, + ...current, operationId, phase: "downloading", downloadedBytes: 0, totalBytes: asset.archiveBytes, version: asset.version, - installedVersion: current.installedVersion, - canRemove: current.canRemove, message: "Downloading Codex.", }; yield* SubscriptionRef.set(state, next); @@ -807,6 +806,9 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function source: "managed", }) satisfies ProviderInstallState, ); + // A supported older runtime still resolves, so a raised pin needs its own update. + if (asset && compareSemverVersions(installed.version, asset.version) < 0) + yield* start.pipe(Effect.ignore); }).pipe( Effect.catch(() => SubscriptionRef.update( diff --git a/apps/server/src/provider/Drivers/ClaudeHome.test.ts b/apps/server/src/provider/Drivers/ClaudeHome.test.ts index ae2dacd0f2311..c772b79efe184 100644 --- a/apps/server/src/provider/Drivers/ClaudeHome.test.ts +++ b/apps/server/src/provider/Drivers/ClaudeHome.test.ts @@ -60,7 +60,7 @@ it.layer(NodeServices.layer)("ClaudeHome", (it) => { }); it.effect( - "drops the launching Claude session and labels processes t3-code unless overridden", + "drops the launching Claude session and Electron's run-as-Node flag, and labels processes t3-code unless overridden", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -72,6 +72,7 @@ it.layer(NodeServices.layer)("ClaudeHome", (it) => { CLAUDE_CODE_SESSION_ID: "host-session", CLAUDE_CODE_MESSAGING_SOCKET: "/tmp/host-claude.sock", CLAUDE_EFFORT: "high", + ELECTRON_RUN_AS_NODE: "1", KEEP: "1", }; const instance = [ diff --git a/apps/server/src/provider/Drivers/ClaudeHome.ts b/apps/server/src/provider/Drivers/ClaudeHome.ts index ed5d8fca2ad7c..8042a7f16e491 100644 --- a/apps/server/src/provider/Drivers/ClaudeHome.ts +++ b/apps/server/src/provider/Drivers/ClaudeHome.ts @@ -76,6 +76,9 @@ export const mergeClaudeProcessEnvironment = Effect.fn("mergeClaudeProcessEnviro CLAUDE_CODE_ENTRYPOINT: "t3-code", }; for (const name of CLAUDE_HOST_SESSION_VARIABLES) delete baseEnvironment[name]; + // The desktop runs the server as Node through Electron. Commands Claude runs inherit its env, so + // an Electron app they start (T3 Code itself, VS Code) would run as Node and exit. + delete baseEnvironment.ELECTRON_RUN_AS_NODE; const environment = yield* mergeProviderInstanceEnvironment(instanceEnvironment, baseEnvironment); const settingsEntrypoint = (yield* readClaudeSettingsEnv(config, undefined, environment)) .CLAUDE_CODE_ENTRYPOINT; diff --git a/apps/server/src/provider/Drivers/ClaudeSkills.test.ts b/apps/server/src/provider/Drivers/ClaudeSkills.test.ts index 0855d457f1f56..99a05534493f4 100644 --- a/apps/server/src/provider/Drivers/ClaudeSkills.test.ts +++ b/apps/server/src/provider/Drivers/ClaudeSkills.test.ts @@ -211,6 +211,71 @@ it.layer(NodeServices.layer)("discoverClaudeSkills", (it) => { }), ); + it.effect("resolves each install record's project repository once per discovery", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + // Real path: macOS temp dirs sit behind a /var -> /private/var symlink. + const tempDir = yield* fs.realPath( + yield* fs.makeTempDirectoryScoped({ prefix: "t3-claude-skills-" }), + ); + const configDir = path.join(tempDir, "claude-home"); + const workspace = path.join(tempDir, "workspace"); + const otherProject = path.join(tempDir, "other", "packages", "app"); + const sameRepoProject = path.join(workspace, "packages", "app"); + yield* fs.makeDirectory(path.join(workspace, ".git"), { recursive: true }); + yield* fs.makeDirectory(path.join(tempDir, "other", ".git"), { recursive: true }); + yield* fs.makeDirectory(otherProject, { recursive: true }); + yield* fs.makeDirectory(sameRepoProject, { recursive: true }); + const elsewhereInstall = path.join(configDir, "plugins", "elsewhere"); + const sameRepoInstall = path.join(configDir, "plugins", "same-repo"); + yield* writeSkill(path.join(elsewhereInstall, "skills"), "elsewhere", "# Elsewhere"); + yield* writeSkill(path.join(sameRepoInstall, "skills"), "same-repo", "# Same repo"); + const records = Array.from({ length: 4 }, () => ({ + scope: "project", + projectPath: otherProject, + installPath: elsewhereInstall, + })); + const plugins = ["a@shop", "b@shop", "c@shop"]; + yield* fs.writeFileString( + path.join(configDir, "plugins", "installed_plugins.json"), + JSON.stringify({ + plugins: { + ...Object.fromEntries(plugins.map((key) => [key, records])), + "d@shop": [ + { scope: "project", projectPath: sameRepoProject, installPath: sameRepoInstall }, + ], + }, + }), + ); + yield* fs.writeFileString( + path.join(configDir, "settings.json"), + JSON.stringify({ + enabledPlugins: Object.fromEntries([...plugins, "d@shop"].map((key) => [key, true])), + }), + ); + + const probed: Array = []; + const skills = yield* discoverClaudeSkills({ homePath: configDir }, workspace).pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fs, + exists: (target) => { + probed.push(target); + return fs.exists(target); + }, + }), + ); + assert.strictEqual( + probed.filter((target) => target === path.join(otherProject, ".git")).length, + 1, + ); + assert.deepEqual( + skills.map((skill) => skill.name), + ["d:same-repo"], + ); + }), + ); + it.effect("publishes and dispatches canonical plugin folder names regardless of aliases", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/provider/Drivers/ClaudeSkills.ts b/apps/server/src/provider/Drivers/ClaudeSkills.ts index c73385374fbc0..2bcf9ca472f0c 100644 --- a/apps/server/src/provider/Drivers/ClaudeSkills.ts +++ b/apps/server/src/provider/Drivers/ClaudeSkills.ts @@ -143,7 +143,8 @@ function claudeSettingsPaths( ]; } -const findRepositoryRoot = Effect.fn("findRepositoryRoot")(function* ( +// Untraced: skill discovery resolves every distinct plugin project path, too often for spans. +const findRepositoryRoot = Effect.fnUntraced(function* ( cwd: string, ): Effect.fn.Return { const fileSystem = yield* FileSystem.FileSystem; @@ -164,7 +165,7 @@ const findRepositoryRoot = Effect.fn("findRepositoryRoot")(function* ( } }); -const findPluginRepositoryRoot = Effect.fn("findPluginRepositoryRoot")(function* (cwd: string) { +const findPluginRepositoryRoot = Effect.fnUntraced(function* (cwd: string) { const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const root = yield* findRepositoryRoot(cwd); @@ -336,6 +337,8 @@ const discoverPluginSkillRoots = Effect.fn("discoverPluginSkillRoots")(function* cwd === undefined ? undefined : yield* fileSystem.realPath(cwd).pipe(Effect.orElseSucceed(() => path.resolve(cwd))); + // Install records repeat project paths, often hundreds of times per discovery. + const projectRepositoryRoots = new Map(); const repositoryRoot = workspace === undefined ? undefined : yield* findPluginRepositoryRoot(workspace); @@ -347,11 +350,13 @@ const discoverPluginSkillRoots = Effect.fn("discoverPluginSkillRoots")(function* Effect.gen(function* () { if (record.scope === "user" || record.scope === "managed") return true; if (record.projectPath === undefined || workspace === undefined) return false; - if (path.resolve(record.projectPath) === workspace) return true; - return ( - repositoryRoot !== undefined && - (yield* findPluginRepositoryRoot(record.projectPath)) === repositoryRoot - ); + const projectPath = path.resolve(record.projectPath); + if (projectPath === workspace) return true; + if (repositoryRoot === undefined) return false; + if (!projectRepositoryRoots.has(projectPath)) { + projectRepositoryRoots.set(projectPath, yield* findPluginRepositoryRoot(projectPath)); + } + return projectRepositoryRoots.get(projectPath) === repositoryRoot; }), ); // The native loader keeps registry order, preferring the first install that exists. diff --git a/apps/server/src/provider/EventNdjsonLogger.test.ts b/apps/server/src/provider/EventNdjsonLogger.test.ts index c04150dce8df6..3cb7870f3f518 100644 --- a/apps/server/src/provider/EventNdjsonLogger.test.ts +++ b/apps/server/src/provider/EventNdjsonLogger.test.ts @@ -156,6 +156,39 @@ it.layer(NodeServices.layer)("EventNdjsonLogger", (it) => { }), ); + it.effect("keeps long thread ids that share a prefix in separate files", () => + Effect.gen(function* () { + const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); + const basePath = NodePath.join(tempDir, "events.log"); + const prefix = + "thread:delegated-task:command%3Amcp%3A00000000-0000-4000-8000-000000000000%3Adelegate-task%3A"; + + try { + const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 0 }); + const logger = store.logger("native"); + yield* logger.write({ id: "first" }, ThreadId.make(`${prefix}review-the-diff`)); + yield* logger.write({ id: "second" }, ThreadId.make(`${prefix}write-the-tests`)); + yield* store.close(); + + const files = NodeFS.readdirSync(tempDir); + assert.equal(files.length, 2); + assert.deepEqual( + files + .map((name) => + NodeFS.readFileSync(NodePath.join(tempDir, name), "utf8") + .trim() + .split("\n") + .map((line) => parseLogLine(line).payload), + ) + .toSorted(), + [['{"id":"first"}'], ['{"id":"second"}']], + ); + } finally { + NodeFS.rmSync(tempDir, { recursive: true, force: true }); + } + }), + ); + it.effect("keeps shared store views non-owning when one adapter closes", () => Effect.gen(function* () { const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); diff --git a/apps/server/src/provider/EventNdjsonLogger.ts b/apps/server/src/provider/EventNdjsonLogger.ts index 0e8c9043dcf0f..6a1d3844b2373 100644 --- a/apps/server/src/provider/EventNdjsonLogger.ts +++ b/apps/server/src/provider/EventNdjsonLogger.ts @@ -1,5 +1,6 @@ // @effect-diagnostics nodeBuiltinImport:off /** Native and canonical views share write state to avoid races on the same thread file. */ +import * as NodeCrypto from "node:crypto"; import * as NodeFS from "node:fs"; import * as NodePath from "node:path"; @@ -18,7 +19,10 @@ import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as SynchronizedRef from "effect/SynchronizedRef"; -import { toSafeThreadAttachmentSegment } from "../attachmentStore.ts"; +import { + ATTACHMENT_ID_THREAD_SEGMENT_MAX_CHARS, + toSafeThreadAttachmentSegment, +} from "../attachmentStore.ts"; import type * as ResourceAttribution from "../resourceTelemetry/ResourceAttribution.ts"; const MEBIBYTE = 1024 * 1024; @@ -145,8 +149,13 @@ function logWarning(message: string, context: Record): Effect.E } function resolveThreadSegment(raw: string | null | undefined): string { - const normalized = typeof raw === "string" ? toSafeThreadAttachmentSegment(raw) : null; - return normalized ?? GLOBAL_THREAD_SEGMENT; + if (typeof raw !== "string") return GLOBAL_THREAD_SEGMENT; + const normalized = toSafeThreadAttachmentSegment(raw); + if (normalized === null) return GLOBAL_THREAD_SEGMENT; + // Delegated thread ids share a prefix longer than the segment, so ids that may be cut get a hash. + if (raw.length <= ATTACHMENT_ID_THREAD_SEGMENT_MAX_CHARS) return normalized; + const hash = NodeCrypto.createHash("sha256").update(raw).digest("hex").slice(0, 16); + return `${normalized}-${hash}`; } function resolveStreamLabel(stream: EventNdjsonStream): string { @@ -725,12 +734,13 @@ export const makeEventNdjsonLogStore = Effect.fnUntraced(function* ( const observedAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)); const line = `[${observedAt}] ${resolveStreamLabel(stream)}: ${payload}\n`; const bytes = Buffer.byteLength(line); + const threadSegment = resolveThreadSegment(threadId); const action = yield* SynchronizedRef.modifyEffect(stateRef, (state) => { if (state.closed) { return Effect.succeed([{ flush: false }, state] as const); } const pending = state.pending; - pending.push({ stream, threadSegment: resolveThreadSegment(threadId), line, bytes }); + pending.push({ stream, threadSegment, line, bytes }); const pendingBytes = state.pendingBytes + bytes; const flush = resolved.batchWindowMs === 0 || diff --git a/apps/server/src/provider/GrokProvider.test.ts b/apps/server/src/provider/GrokProvider.test.ts index 9cc1a9a7e3238..9be799112cc23 100644 --- a/apps/server/src/provider/GrokProvider.test.ts +++ b/apps/server/src/provider/GrokProvider.test.ts @@ -722,6 +722,36 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("readGrokAccount" } }), ); + + it.effect("tells accounts in Legacy (no storage) to switch to Privacy Mode", () => + Effect.gen(function* () { + const { usageLimits: limits } = yield* readGrokAccount({ + HOME: "/definitely/not/a/grok-home", + GROK_AUTH: '{"https://accounts.x.ai/sign-in":{"key":"private-token"}}', + }).pipe( + Effect.provideService( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + Response.json( + { + error: + "Grok Build is not supported in Legacy (no storage). Switch to Privacy Mode to use Grok Build. [WKE=privacy-mode:legacy]", + }, + { status: 403 }, + ), + ), + ), + ), + ), + ); + expect(limits.unavailable?.reason).toBe("probeFailed"); + expect(limits.unavailable?.message).toMatch(/Privacy Mode/); + expect(limits.unavailable?.message).not.toMatch(/WKE=|Grok Build is not supported/); + }), + ); }); it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("readGrokAccount email", (it) => { diff --git a/apps/server/src/provider/KiroProvider.test.ts b/apps/server/src/provider/KiroProvider.test.ts index b0b9143d97a55..1877c131e3158 100644 --- a/apps/server/src/provider/KiroProvider.test.ts +++ b/apps/server/src/provider/KiroProvider.test.ts @@ -3,13 +3,16 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, it } from "@effect/vitest"; import { KiroSettings } from "@t3tools/provider-kiro/settings"; import * as Deferred from "effect/Deferred"; +import type * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import * as TestClock from "effect/testing/TestClock"; import { checkKiroProviderStatus, @@ -23,18 +26,25 @@ const settings = Schema.decodeSync(KiroSettings)({ enabled: true }); interface CliOutput { readonly stdout: string; readonly code: number; + /** Holds stdout and the exit back this long, like a CLI slow to answer. */ + readonly delay?: Duration.Input; } function processHandle(output: CliOutput) { + const answered = Effect.sleep(output.delay ?? 0); return ChildProcessSpawner.makeHandle({ pid: ChildProcessSpawner.ProcessId(900_000_002), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(output.code)), + exitCode: Effect.as(answered, ChildProcessSpawner.ExitCode(output.code)), isRunning: Effect.succeed(false), kill: () => Effect.void, unref: Effect.succeed(Effect.void), stdin: Sink.drain, - stdout: - output.stdout.length === 0 ? Stream.empty : Stream.succeed(encoder.encode(output.stdout)), + stdout: Stream.unwrap( + Effect.as( + answered, + output.stdout.length === 0 ? Stream.empty : Stream.succeed(encoder.encode(output.stdout)), + ), + ), stderr: Stream.empty, all: Stream.empty, getInputFd: () => Sink.drain, @@ -123,6 +133,35 @@ const check = ( }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))); describe("KiroProvider", () => { + it.effect.each([ + { + label: "answers after 6 s", + delay: "6 seconds", + status: "ready", + version: "2.27.0", + message: undefined, + }, + { + label: "never answers", + delay: "Infinity", + status: "error", + version: null, + message: "Failed to run `kiro-cli --version`.", + }, + ] as const)("gives `kiro-cli --version` 10 s when it $label", (row) => + Effect.gen(function* () { + const fiber = yield* check({ + "--version": { stdout: "kiro-cli 2.27.0\n", code: 0, delay: row.delay }, + "whoami --format json": { stdout: '{"accountType":"ApiKey","email":null}\n', code: 0 }, + }).pipe(Effect.forkChild); + yield* TestClock.adjust("10 seconds"); + const { snapshot } = yield* Fiber.join(fiber); + assert.equal(snapshot.status, row.status); + assert.equal(snapshot.version, row.version); + assert.equal(snapshot.message, row.message); + }), + ); + it.effect("reports a CLI whose version probe exits nonzero as failing to run", () => Effect.gen(function* () { const { snapshot, invoked } = yield* check({ diff --git a/apps/server/src/provider/ProviderRegistry.test.ts b/apps/server/src/provider/ProviderRegistry.test.ts index 32d721d65bbcb..115e827cad4b6 100644 --- a/apps/server/src/provider/ProviderRegistry.test.ts +++ b/apps/server/src/provider/ProviderRegistry.test.ts @@ -2124,6 +2124,139 @@ it.layer( }), ); + it.effect("ignores a probe from an instance that settings replaced mid-refresh", () => + Effect.gen(function* () { + const scenario = Effect.fn(function* ( + probeFails: boolean, + run: ( + registry: ProviderRegistry.ProviderRegistry["Service"], + ) => Effect.Effect>, + ) { + const driver = ProviderDriverKind.make("codex"); + const instanceId = ProviderInstanceId.make("codex"); + const provider = { + instanceId, + driver, + status: "ready", + enabled: true, + installed: true, + auth: { status: "authenticated" }, + checkedAt: "2026-06-10T00:00:00.000Z", + version: "2.0.0", + models: [], + slashCommands: [], + skills: [], + } as const satisfies ServerProvider; + const retiredProbe = { + ...provider, + status: "error", + installed: false, + version: "1.0.0", + message: "Codex CLI is not installed.", + } as const satisfies ServerProvider; + const probeStarted = yield* Deferred.make(); + const releaseProbe = yield* Deferred.make(); + const makeInstance = (snapshot: ServerProvider, refresh: Effect.Effect) => + ({ + instanceId, + driverKind: driver, + continuationIdentity: { driverKind: driver, continuationKey: "codex:instance:codex" }, + displayName: undefined, + enabled: true, + snapshot: { + resolveMaintenance: () => + Effect.succeed( + makeManualOnlyProviderMaintenanceCapabilities({ + provider: driver, + packageName: null, + }), + ), + getSnapshot: Effect.succeed(snapshot), + refresh: Effect.as(refresh, snapshot), + streamChanges: Stream.empty, + applyUsageLimits: () => Effect.void, + }, + orchestrationAdapter: {} as ProviderInstance["orchestrationAdapter"], + textGeneration: {} as ProviderInstance["textGeneration"], + }) satisfies ProviderInstance; + const retiredInstance = makeInstance( + retiredProbe, + Deferred.succeed(probeStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseProbe)), + Effect.andThen(probeFails ? Effect.die(new Error("probe failed")) : Effect.void), + ), + ); + const rebuiltInstance = makeInstance(provider, Effect.void); + const instances = yield* Ref.make>([retiredInstance]); + const changes = yield* PubSub.unbounded(); + const instanceRegistryLayer = Layer.succeed( + ProviderInstanceRegistry.ProviderInstanceRegistry, + { + getInstance: (requestedId) => + Ref.get(instances).pipe( + Effect.map((current) => + current.find((instance) => instance.instanceId === requestedId), + ), + ), + listInstances: Ref.get(instances), + listUnavailable: Effect.succeed([]), + streamChanges: Stream.empty, + subscribeChanges: PubSub.subscribe(changes), + }, + ); + const scope = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); + const runtimeServices = yield* Layer.build( + ProviderRegistry.layer.pipe( + Layer.provide(layerUnusedMutator), + Layer.provideMerge(instanceRegistryLayer), + Layer.provideMerge( + ServerConfig.layerTest(process.cwd(), { + prefix: "t3-provider-registry-retired-refresh-", + }), + ), + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ), + ).pipe(Scope.provide(scope)); + + return yield* Effect.gen(function* () { + const registry = yield* ProviderRegistry.ProviderRegistry; + const rebuiltPublished = yield* Stream.toPull( + registry.streamChanges.pipe( + Stream.filter((providers) => providers[0]?.version === provider.version), + ), + ); + const refresh = yield* run(registry).pipe(Effect.forkChild); + yield* Deferred.await(probeStarted); + yield* Ref.set(instances, [rebuiltInstance]); + yield* PubSub.publish(changes, undefined); + yield* rebuiltPublished; + yield* Deferred.succeed(releaseProbe, undefined); + return { + provider, + result: yield* Fiber.join(refresh), + providers: yield* registry.getProviders, + }; + }).pipe(Effect.provide(runtimeServices)); + }); + const driver = ProviderDriverKind.make("codex"); + const instanceId = ProviderInstanceId.make("codex"); + + const manual = yield* scenario(false, (registry) => registry.refresh()); + expect(manual.result).toEqual([expect.objectContaining(manual.provider)]); + expect(manual.providers).toEqual([expect.objectContaining(manual.provider)]); + + // Neither the retired probe's failure nor its cached row can stand in for the replacement. + for (const probeFails of [true, false]) { + const afterUpdate = yield* scenario(probeFails, (registry) => + registry.refreshAfterUpdate(driver, instanceId), + ); + expect(afterUpdate.result).toEqual([]); + expect(afterUpdate.providers).toEqual([expect.objectContaining(afterUpdate.provider)]); + } + }), + ); + it.effect("keeps an enriched snapshot that streams in before a manual refresh returns", () => Effect.gen(function* () { const driver = ProviderDriverKind.make("acpRegistry"); diff --git a/apps/server/src/provider/ProviderRegistry.ts b/apps/server/src/provider/ProviderRegistry.ts index 95bfcd9230883..983e6e831be5f 100644 --- a/apps/server/src/provider/ProviderRegistry.ts +++ b/apps/server/src/provider/ProviderRegistry.ts @@ -469,10 +469,6 @@ export const layer = Layer.effect( ); const syncSemaphore = yield* Semaphore.make(1); - const getLiveSources: Effect.Effect> = Ref.get( - liveSubsRef, - ).pipe(Effect.map((map) => Array.from(map.values(), buildSnapshotSource))); - const persistProvider = Effect.fn("ProviderRegistry.persistProvider")(function* ( provider: ServerProvider, ) { @@ -626,17 +622,25 @@ export const layer = Layer.effect( }, ); - const refreshOneSource = Effect.fn("refreshOneSource")(function* ( - providerSource: ProviderSnapshotSource, - ) { + const isCurrentInstance = (instance: ProviderInstance) => + instanceRegistry + .getInstance(instance.instanceId) + .pipe(Effect.map((current) => current === instance)); + + const refreshOneSource = Effect.fn("refreshOneSource")(function* (instance: ProviderInstance) { + const providerSource = buildSnapshotSource(instance); // Enrichment can publish a newer snapshot before refresh returns its pre-enrichment probe. - return yield* providerSource.refresh.pipe( + const probe = yield* providerSource.refresh.pipe( Effect.andThen(providerSource.getSnapshot), - Effect.flatMap((nextProvider) => - correlateSnapshotWithSource(providerSource, nextProvider).pipe( - Effect.flatMap(syncProvider), - ), - ), + Effect.exit, + ); + // Settings can replace or remove the instance mid-probe; its late result or failure is stale. + if (!(yield* isCurrentInstance(instance))) { + return yield* Ref.get(providersRef); + } + const nextProvider = yield* probe; + return yield* correlateSnapshotWithSource(providerSource, nextProvider).pipe( + Effect.flatMap(syncProvider), ); }); @@ -665,9 +669,9 @@ export const layer = Layer.effect( // Detach shared passes so the first caller's interruption cannot cancel other callers' work. // Syncing first keeps a refresh right after a rebuild from probing the replaced instance. yield* syncLiveSourcesAndContinue.pipe( - Effect.andThen(getLiveSources), - Effect.flatMap((sources) => - Effect.forEach(sources, (source) => refreshOneSource(source), { + Effect.andThen(Ref.get(liveSubsRef)), + Effect.flatMap((instances) => + Effect.forEach(instances.values(), (instance) => refreshOneSource(instance), { concurrency: "unbounded", discard: true, }), @@ -688,27 +692,22 @@ export const layer = Layer.effect( if (provider === undefined) { return yield* refreshAll(); } - const defaultInstanceId = defaultInstanceIdForDriver(provider); - const sources = yield* getLiveSources; - const providerSource = sources.find( - (candidate) => candidate.instanceId === defaultInstanceId, - ); - if (!providerSource) { + const instance = (yield* Ref.get(liveSubsRef)).get(defaultInstanceIdForDriver(provider)); + if (!instance) { return yield* Ref.get(providersRef); } - return yield* refreshOneSource(providerSource); + return yield* refreshOneSource(instance); }); const refreshInstance = Effect.fn("refreshInstance")(function* ( instanceId: ProviderInstanceId, ) { yield* syncLiveSourcesAndContinue; - const sources = yield* getLiveSources; - const providerSource = sources.find((candidate) => candidate.instanceId === instanceId); - if (!providerSource) { + const instance = (yield* Ref.get(liveSubsRef)).get(instanceId); + if (!instance) { return yield* Ref.get(providersRef); } - return yield* refreshOneSource(providerSource); + return yield* refreshOneSource(instance); }); const reloadSecretsAndRefresh = Effect.fn("reloadSecretsAndRefresh")(function* ( @@ -769,15 +768,18 @@ export const layer = Layer.effect( yield* Effect.forEach( instances.filter((instance) => !failedInstances.has(instance.instanceId)), (instance) => - refreshOneSource(buildSnapshotSource(instance)).pipe( + refreshOneSource(instance).pipe( Effect.catchCause((cause) => recoverInstanceFailure(instance.instanceId, cause)), ), { concurrency: "unbounded", discard: true }, ); + // A replaced instance's cached row did not come from this pass's probe. + const refreshedInstances = yield* Effect.filter( + instances.filter((instance) => !failedInstances.has(instance.instanceId)), + isCurrentInstance, + ); const refreshedInstanceIds = new Set( - instances - .filter((instance) => !failedInstances.has(instance.instanceId)) - .map((instance) => instance.instanceId), + refreshedInstances.map((instance) => instance.instanceId), ); return (yield* Ref.get(providersRef)).filter((candidate) => refreshedInstanceIds.has(candidate.instanceId), diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts index d3ca431c50794..09b8730bd6001 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts @@ -22,9 +22,12 @@ const errorPayload = (error: string) => `Agent execution terminated due to error.\u0012agent executor error: ${error}: ${error}\n(1) attached stack trace\n -- stack trace:`, ), ]); -// An unparsable reset timestamp must not hide the reset delay. +// An unparsable reset timestamp must not hide the reset delay, nor a retry delay outrank it. const QUOTA_ERROR = errorPayload( - "Error 429, Message: Quota exhausted for test model., Status: RESOURCE_EXHAUSTED, Details: [map[@type:type.googleapis.com/google.rpc.ErrorInfo domain:googleapis.com metadata:map[quotaResetDelay:3600s quotaResetTimeStamp:soon] reason:QUOTA_EXHAUSTED]]", + "Error 429, Message: Quota exhausted for test model., Status: RESOURCE_EXHAUSTED, Details: [map[@type:type.googleapis.com/google.rpc.RetryInfo retryDelay:42s] map[@type:type.googleapis.com/google.rpc.ErrorInfo domain:googleapis.com metadata:map[quotaResetDelay:3600s quotaResetTimeStamp:soon] reason:QUOTA_EXHAUSTED]]", +); +const RATE_LIMIT_ERROR = errorPayload( + "Error 429, Message: Resource has been exhausted (e.g. check quota)., Status: RESOURCE_EXHAUSTED, Details: [map[@type:type.googleapis.com/google.rpc.ErrorInfo domain:cloudcode-pa.googleapis.com reason:RATE_LIMIT_EXCEEDED] map[@type:type.googleapis.com/google.rpc.RetryInfo retryDelay:42s]]", ); const INVALID_ARGUMENT_ERROR = errorPayload( "Error 400, Message: Request contains an invalid argument., Status: INVALID_ARGUMENT, Details: []", @@ -101,6 +104,19 @@ describe("watchAntigravityPromptOutcome", () => { }, ), ); + assert.deepEqual( + yield* turn([ + [USER_INPUT, input], + [ERROR_MESSAGE, RATE_LIMIT_ERROR], + ]), + { + class: "usage_limit", + message: "Resource has been exhausted (e.g. check quota).", + code: "RESOURCE_EXHAUSTED", + retryable: true, + resetAt: "1970-01-01T00:00:42.000Z", + }, + ); assert.deepEqual( yield* turn([ [USER_INPUT, input], diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts index 874c28eeb1537..00de1e836296e 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts @@ -21,6 +21,8 @@ const API_ERROR = /Error (\d{3}), Message: (.+?), Status: ([A-Z_]+)/s; const QUOTA_EXHAUSTED = /\breason:QUOTA_EXHAUSTED\b/; const RESET_TIMESTAMP = /\bquotaResetTimeStamp:([^\s\]]+)/; const RESET_DELAY = /\bquotaResetDelay:((?:\d+(?:\.\d+)?(?:ns|us|µs|μs|ms|h|m|s))+)(?=[\s\]])/; +// google.rpc.RetryInfo, which plain rate limits carry instead of a quota reset. +const RETRY_DELAY = /\bretryDelay:((?:\d+(?:\.\d+)?(?:ns|us|µs|μs|ms|h|m|s))+)(?=[\s\]])/; const GO_DURATION_PART = /(\d+(?:\.\d+)?)(ns|us|µs|μs|ms|h|m|s)/g; const GO_DURATION_UNIT_MS: Record = { ns: 1e-6, @@ -76,7 +78,7 @@ function resetAtFrom(text: string, now: DateTime.Utc): string | undefined { // makeProviderFailure drops a timestamp that does not parse, so fall back to the delay. const timestamp = RESET_TIMESTAMP.exec(text)?.[1]; if (timestamp !== undefined && Option.isSome(DateTime.make(timestamp))) return timestamp; - const delay = RESET_DELAY.exec(text)?.[1]; + const delay = RESET_DELAY.exec(text)?.[1] ?? RETRY_DELAY.exec(text)?.[1]; if (delay === undefined) return undefined; let delayMs = 0; for (const [, amount, unit] of delay.matchAll(GO_DURATION_PART)) { @@ -120,6 +122,7 @@ function turnFailure( message: message!.trim(), code: status ?? null, retryable: rateLimited || Number(httpStatus) >= 500, + resetAt: resetAtFrom(text, now) ?? null, }); } diff --git a/apps/server/src/provider/documentTextWorker.ts b/apps/server/src/provider/documentTextWorker.ts index 569493906d385..4abadfbdb173b 100644 --- a/apps/server/src/provider/documentTextWorker.ts +++ b/apps/server/src/provider/documentTextWorker.ts @@ -14,6 +14,20 @@ const CELL_END = Symbol("cell-end"); type TextPart = string | typeof CELL_START | typeof CELL_END | { fallback: string }; const TRUNCATED = "\n\n[Extracted text truncated at document extraction limits.]"; +// Parts the document never displays: tracked deletions and moves, field codes, phonetic guides, +// paragraph properties such as tab-stop definitions. +const HIDDEN = new Set([ + "script", + "style", + "binary-data", + "rPh", + "delText", + "moveFrom", + "instrText", + "delInstrText", + "tracked-changes", + "pPr", +]); class Truncation extends Error {} @@ -22,6 +36,8 @@ interface XmlNode { attrs: Record; children: Array; spaceCount?: number; + hasText?: boolean; + hasTextChoice?: boolean; } function xml(source: string, odfSpaces = false, htmlEntities = false): XmlNode { @@ -47,10 +63,22 @@ function xml(source: string, odfSpaces = false, htmlEntities = false): XmlNode { stack.push(node); }); parser.on("closetag", () => { - stack.pop(); + const node = stack.pop()!; + const parent = stack.at(-1)!; + // Markup-compatibility fallbacks repeat their mc:Choice content, e.g. Word text boxes. + // Keep one when no Choice carries text, such as a chart with a VML caption. + if (node.name === "Fallback" && parent.hasTextChoice) { + parent.children.pop(); + return; + } + if (!node.hasText) return; + parent.hasText = true; + if (node.name === "Choice") parent.hasTextChoice = true; }); const text = (value: string) => { - stack.at(-1)!.children.push(value); + const node = stack.at(-1)!; + node.children.push(value); + if (value.trim() !== "") node.hasText = true; }; parser.on("text", text); parser.on("cdata", text); @@ -67,7 +95,7 @@ function* all(node: XmlNode, name: string | readonly string[]): Generator { - if (["script", "style", "binary-data", "rPh"].includes(node.name)) return; + if (HIDDEN.has(node.name)) return; if (["tab", "cell"].includes(node.name)) yield "\t"; else if (["br", "line-break", "cr"].includes(node.name)) yield "\n"; else if (node.spaceCount !== undefined) { @@ -80,7 +108,7 @@ function* text(node: XmlNode): Generator { } } function* paragraphs(node: XmlNode): Generator { - if (["script", "style", "binary-data"].includes(node.name)) return; + if (HIDDEN.has(node.name)) return; for (const child of node.children) { if (typeof child === "string") yield child; else if (["br", "line-break", "cr"].includes(child.name)) yield "\n"; @@ -137,11 +165,7 @@ function odfValue(cell: XmlNode): string { function hasCellText(cell: XmlNode): boolean { if (odfValue(cell).trim()) return true; // Space markers can have enormous repeat counts; emptiness never requires expanding them. - if ( - ["script", "style", "binary-data", "rPh"].includes(cell.name) || - cell.spaceCount !== undefined - ) - return false; + if (HIDDEN.has(cell.name) || cell.spaceCount !== undefined) return false; return cell.children.some((child) => typeof child === "string" ? !!child.trim() : hasCellText(child), ); diff --git a/apps/server/src/provider/userInputAttachments.test.ts b/apps/server/src/provider/userInputAttachments.test.ts index 81c167c038d87..90793ce3f821e 100644 --- a/apps/server/src/provider/userInputAttachments.test.ts +++ b/apps/server/src/provider/userInputAttachments.test.ts @@ -554,6 +554,25 @@ describe("question answer paths", () => { expected: ["Start", "[Extracted text truncated at document extraction limits.]"], absent: ["Unneeded"], }, + { + ext: "docx", + bounded: true, + parts: { + "word/document.xml": + 'BodyCallout boxCallout box', + }, + expected: ["Body\nCallout box"], + occurrences: { text: "Callout box", count: 1 }, + }, + { + ext: "docx", + bounded: true, + parts: { + "word/document.xml": + 'Chart caption', + }, + expected: ["Chart caption"], + }, { ext: "docx", bounded: true, @@ -572,6 +591,26 @@ describe("question answer paths", () => { }, expected: ["First\nSecond\nThird"], }, + { + ext: "docx", + bounded: true, + parts: { + "word/document.xml": + 'Intro PAGEREF _Toc1 \\h 3Fee $500$700Moveddue', + }, + expected: ["Intro 3\nFee $700\tdue"], + absent: ["PAGEREF", "$500", "Moved"], + }, + { + ext: "odt", + bounded: true, + parts: { + "content.xml": + "Reviewer

$500

Fee $700

", + }, + expected: ["Fee $700"], + absent: ["Reviewer", "$500"], + }, { ext: "docx", bounded: true, diff --git a/apps/server/src/pullRequest/PullRequestReviewGuideService.test.ts b/apps/server/src/pullRequest/PullRequestReviewGuideService.test.ts index 4f567142fbed5..3ee929f923a57 100644 --- a/apps/server/src/pullRequest/PullRequestReviewGuideService.test.ts +++ b/apps/server/src/pullRequest/PullRequestReviewGuideService.test.ts @@ -467,7 +467,7 @@ it.layer(layerBase, { excludeTestServices: true })("PullRequestReviewGuideServic ), ); - it.effect("keeps excerpts of source lines a configured diff driver would convert away", () => + it.effect("keeps excerpts despite the checkout's diff driver and a failing hook", () => Effect.scoped( Effect.gen(function* () { const repositories = yield* makeRepositories; @@ -477,6 +477,9 @@ it.layer(layerBase, { excludeTestServices: true })("PullRequestReviewGuideServic "feature.txt diff=blank\n", ); yield* git(repositories.workspace, ["config", "diff.blank.textconv", "true"]); + const hook = `${repositories.workspace}/.git/hooks/post-checkout`; + yield* fs.writeFileString(hook, "#!/bin/sh\nexit 1\n"); + yield* fs.chmod(hook, 0o755); const stored = yield* Deferred.make(); const context = yield* Layer.build( makeLayer({ @@ -502,9 +505,10 @@ it.layer(layerBase, { excludeTestServices: true })("PullRequestReviewGuideServic context, PullRequestReviewGuideService.PullRequestReviewGuideService, ); - yield* service.generate(reference); - yield* Deferred.await(stored); - const result = yield* service.get(reference); + const runs = yield* watchRuns; + yield* runs.generate(service, reference); + const result = yield* runs.settled(service, reference); + assert.notStrictEqual(result.run?._tag, "Failed", JSON.stringify(result.run)); assert.deepStrictEqual(result.guide?.stops[0]?.excerpts, [ { path: "feature.txt", side: "new", start: 1, end: 3, caption: "" }, ]); diff --git a/apps/server/src/pullRequest/PullRequestReviewGuideService.ts b/apps/server/src/pullRequest/PullRequestReviewGuideService.ts index a7618a23d27ae..8994837fcdd80 100644 --- a/apps/server/src/pullRequest/PullRequestReviewGuideService.ts +++ b/apps/server/src/pullRequest/PullRequestReviewGuideService.ts @@ -305,7 +305,17 @@ const make = Effect.gen(function* () { yield* git.execute({ operation: "PullRequestReviewGuideService.addWorktree", cwd: workspaceRoot, - args: ["worktree", "add", "--detach", "--quiet", path, resolvedHead.commitSha], + // Scratch checkout for reading the change: repository hooks (installs, LFS) only slow or fail it. + args: [ + "-c", + "core.hooksPath=/dev/null", + "worktree", + "add", + "--detach", + "--quiet", + path, + resolvedHead.commitSha, + ], timeoutMs: 5 * 60_000, }); yield* Effect.addFinalizer(() => diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index 34d54af541e45..4b612a310d9e5 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -400,6 +400,8 @@ it.effect("thread cleanup notifies subscribers and a finishing run cannot recrea const run = yield* service.runNow({ id: task.id }).pipe(Effect.forkChild); yield* Deferred.await(started); yield* Deferred.await(running); + const refused = yield* service.runNow({ id: task.id }).pipe(Effect.flip); + assert.equal(refused.message, "Schedule task is already running."); yield* service.deleteForThread(threadId); yield* Deferred.await(removed); yield* Deferred.succeed(release, undefined); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 27e41281a3a65..737df297441d9 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -339,9 +339,8 @@ function requireFutureOnce( } function errorMessage(error: unknown): string { - if (Cause.isCause(error)) return Cause.pretty(error); - if (error instanceof Error) return error.message; - return String(error); + const failure = Cause.isCause(error) ? Cause.squash(error) : error; + return failure instanceof Error ? failure.message : String(failure); } function webhookPath(taskId: string, token: string): string { @@ -1558,7 +1557,9 @@ export const layer = Layer.effect( } return yield* runTask(task, "manual").pipe( Effect.mapError((cause) => - taskError("Could not run schedule task.", { taskId: input.id, cause }), + isScheduledTaskError(cause) + ? cause + : taskError("Could not run schedule task.", { taskId: input.id, cause }), ), ); }); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index 6303b38c05ce6..90d2ccc329cbf 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -1399,6 +1399,7 @@ it.effect("start failures release both replay keys and relay claims for redelive yield* Queue.take(launches); assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0); assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_relay_deliveries`).length, 0); + assert.equal((yield* service.list()).tasks[0]!.lastRunError, "injected start failure"); const retry = yield* waitForSpan( service.triggerWebhook(request), "ScheduledTaskService.runWebhookDelivery", diff --git a/apps/server/src/secrets/settingsSecretValue.ts b/apps/server/src/secrets/settingsSecretValue.ts index 83a0adc5f11dd..50a99b953d327 100644 --- a/apps/server/src/secrets/settingsSecretValue.ts +++ b/apps/server/src/secrets/settingsSecretValue.ts @@ -1,7 +1,12 @@ -import { secretReferencesInSettings, withSettingsSecrets } from "@t3tools/contracts"; +import { + isSecretReference, + secretReferencesInSettings, + type SettingsSecretValue, + withSettingsSecrets, +} from "@t3tools/contracts"; import type * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; import * as Effect from "effect/Effect"; -import type * as Schema from "effect/Schema"; +import * as Schema from "effect/Schema"; /** * Replaces each reference in the slots `schema` declares as `SettingsSecretValue` with the secret @@ -18,3 +23,29 @@ export const resolveSettingsSecrets = Effect.fn("SettingsSecrets.resolve")(funct pending.map((slot, index) => ({ ...slot, value: values[index] ?? "" })), ); }); + +export class SettingsSecretUnresolvedError extends Schema.TaggedError()( + "SettingsSecretUnresolvedError", + { provider: Schema.String }, +) { + override get message(): string { + return `Could not read the credential from the secret provider "${this.provider}". Unlock or fix it in Settings, then try again.`; + } +} + +/** + * The secret a credential setting holds or names. A named secret that cannot be read, or reads + * empty, fails instead of passing for a missing token, so callers that would fall back to another + * credential cannot silently switch accounts. + */ +export const requireSettingsSecret = Effect.fn("SettingsSecrets.require")(function* ( + resolver: SecretResolver.SecretResolver["Service"], + value: SettingsSecretValue, +) { + if (!isSecretReference(value)) return value; + const [secret] = yield* resolver.read([value]); + if (secret === undefined || secret.trim() === "") { + return yield* new SettingsSecretUnresolvedError({ provider: value.provider }); + } + return secret; +}); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 3ebd71cda0352..f5f86a71f3520 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -237,7 +237,6 @@ import * as ThreadSettlementService from "./orchestration-v2/ThreadSettlementSer import * as IdleCompactionService from "./orchestration-v2/IdleCompactionService.ts"; import * as ThreadPullRequestService from "./orchestration-v2/ThreadPullRequestService.ts"; import * as RunFinalizationService from "./orchestration-v2/RunFinalizationService.ts"; -import * as EventStoreV2 from "./orchestration-v2/EventStore.ts"; import * as ProjectionMaintenance from "./orchestration-v2/ProjectionMaintenance.ts"; import * as ProjectionStoreV2 from "./orchestration-v2/ProjectionStore.ts"; import { makePersistedServerRuntimeState } from "./serverRuntimeState.ts"; @@ -696,12 +695,7 @@ const layerRuntimeCoreDependenciesBase = Layer.mergeAll( layerThreadSettlementWorker, layerIdleCompactionWorker, StorageCleanup.layer.pipe(Layer.provide(ProjectionStoreV2.layer)), - DatabaseStorage.layer.pipe( - Layer.provide(ProjectionMaintenance.layer), - Layer.provide( - Layer.merge(EventStoreV2.layerFromOrchestrationEventStore, ProjectionStoreV2.layer), - ), - ), + DatabaseStorage.layer.pipe(Layer.provide(ProjectionMaintenance.layer)), layerThreadPullRequestWorker, DelegatedTaskAnalytics.layer, Layer.effectDiscard( diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index fbea8cfa8b605..d85b22adb2b65 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -30,6 +30,7 @@ import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; +import * as SchemaAST from "effect/SchemaAST"; import * as SqlClient from "effect/sql/SqlClient"; import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; import { sourceControlHostSecretName } from "./sourceControl/sourceControlHostSecrets.ts"; @@ -37,6 +38,7 @@ import * as ServerConfig from "./config.ts"; import * as SqlitePersistence from "./persistence/Sqlite.ts"; import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; import * as ServerSettingsModule from "./serverSettings.ts"; +import { SECRET_REDACTED } from "./settingsSecrets.ts"; import * as ManagedPolicy from "./policy/ManagedPolicy.ts"; import { resolveProviderInstanceTerminalEnvironment } from "./terminal/Manager.ts"; @@ -105,6 +107,36 @@ const recordProviderUsage = (provider: string, instanceId: string | null = provi `; }); +/** Paths of fields typed `SettingsSecretValue` that clients decode with the settings schema itself. */ +function strictSecretReferencePaths( + ast: SchemaAST.AST, + path = "", + seen = new Set(), +): ReadonlyArray { + if (seen.has(ast)) return []; + seen.add(ast); + const walk = (type: SchemaAST.AST, at: string): ReadonlyArray => + type.context?.annotations?.settingsSecretReference === true + ? [at] + : strictSecretReferencePaths(type, `${at}.`, seen); + if (SchemaAST.isObjects(ast)) { + return [ + ...ast.propertySignatures.flatMap((property) => + walk(property.type, `${path}${String(property.name)}`), + ), + ...ast.indexSignatures.flatMap((signature) => walk(signature.type, `${path}*`)), + ]; + } + if (SchemaAST.isArrays(ast)) { + return [...ast.elements, ...ast.rest].flatMap((element) => walk(element, `${path}[]`)); + } + if (SchemaAST.isUnion(ast)) { + return ast.types.flatMap((member) => strictSecretReferencePaths(member, path, seen)); + } + if (SchemaAST.isSuspend(ast)) return strictSecretReferencePaths(ast.thunk(), path, seen); + return []; +} + it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings", (it) => { it.effect("serializes concurrent model preference edits without cancelling duplicate adds", () => Effect.gen(function* () { @@ -313,7 +345,8 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" }, }); assert.deepStrictEqual( - ServerSettingsModule.redactServerSettingsForClient(settings).secretProviders, + ServerSettingsModule.redactServerSettingsForClient(settings, { secretReferences: true }) + .secretProviders, { personal: { kind: "one_password", label: "Personal", config: { account: "my" } }, vault: { kind: "openbao", label: "Vault" }, @@ -1461,7 +1494,9 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" "quota-secret", "expo-secret", ]; - const client = ServerSettingsModule.redactServerSettingsForClient(saved); + const client = ServerSettingsModule.redactServerSettingsForClient(saved, { + secretReferences: true, + }); const disk = yield* fs.readFileString(config.settingsPath); for (const value of values) { assert.include(JSON.stringify(saved), value); @@ -1495,7 +1530,9 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" instance: { ...client.providerInstances[instanceId]!, config: { apiKey } }, }); assert.propertyVal(changed.providerInstances[instanceId]?.config, "apiKey", apiKey); - const redacted = ServerSettingsModule.redactServerSettingsForClient(changed); + const redacted = ServerSettingsModule.redactServerSettingsForClient(changed, { + secretReferences: true, + }); assert.propertyVal( redacted.providerInstances[instanceId]?.config, "apiKey", @@ -1544,9 +1581,8 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" }); assert.isTrue(Option.isNone(yield* store.get(secretNames[1]!))); assert.deepEqual( - ServerSettingsModule.redactServerSettingsForClient(referenced).providerInstances[ - instanceId - ]?.environment, + ServerSettingsModule.redactServerSettingsForClient(referenced, { secretReferences: true }) + .providerInstances[instanceId]?.environment, [{ name: "API_TOKEN", value: secret, sensitive: true }], ); yield* service.updateSettings({ @@ -1593,7 +1629,9 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" "serverPassword", "legacy-password", ); - const client = ServerSettingsModule.redactServerSettingsForClient(saved); + const client = ServerSettingsModule.redactServerSettingsForClient(saved, { + secretReferences: true, + }); for (const value of ["legacy-api-key", "legacy-password"]) { assert.notInclude(yield* fs.readFileString(config.settingsPath), value); assert.notInclude(JSON.stringify(client), value); @@ -1637,8 +1675,9 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" for (const secret of ["bb-access", "ghp_dotcom", "ghp_ghe"]) assert.notInclude(raw, secret); assert.include(raw, "me@example.com"); - const forClient = - ServerSettingsModule.redactServerSettingsForClient(saved).sourceControlHosts; + const forClient = ServerSettingsModule.redactServerSettingsForClient(saved, { + secretReferences: true, + }).sourceControlHosts; assert.equal(forClient.bitbucket?.email, "me@example.com"); assert.notInclude(String(forClient.bitbucket?.accessToken), "bb-access"); assert.isAbove(String(forClient.bitbucket?.accessToken).length, 0); @@ -1700,15 +1739,78 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" assert.isTrue(Option.isNone(yield* stored("bitbucket", "accessToken", null))); assert.isTrue(Option.isNone(yield* stored("github", "tokens", "github.com"))); assert.isTrue(Option.isSome(yield* stored("github", "tokens", "ghe.acme.test"))); - const redacted = - ServerSettingsModule.redactServerSettingsForClient(referenced).sourceControlHosts; + const redacted = ServerSettingsModule.redactServerSettingsForClient(referenced, { + secretReferences: true, + }).sourceControlHosts; assert.deepEqual(redacted.bitbucket?.accessToken, reference); const tokens = redacted.github?.tokens as Record; assert.deepEqual(tokens["github.com"], gitHubReference); assert.notEqual(tokens["ghe.acme.test"], "ghp_ghe"); + + // A client that cannot decode references sees saved secrets and can send them back. + const usageReference = { + provider: SecretProviderId.make("personal"), + reference: "op://Private/Hub/key", + }; + const easReference = { + provider: SecretProviderId.make("personal"), + reference: "op://Private/Expo/token", + }; + const hub = UsageLimitSourceId.make("hub"); + const withUsage = yield* serverSettings.updateSettings({ + usageLimitSources: { + [hub]: { + kind: "cliproxy", + url: "https://hub.test", + managementKey: usageReference, + enabled: true, + }, + }, + easDeviceHosts: [ + { + kind: "eas", + id: "eas", + label: "EAS Simulator", + accessToken: easReference, + maxSessionMinutes: 30, + idleMinutes: 10, + maxConcurrentSessions: 2, + }, + ], + }); + const legacy = ServerSettingsModule.redactServerSettingsForClient(withUsage, { + secretReferences: false, + }); + assert.notInclude(JSON.stringify(legacy), "op://"); + assert.equal(legacy.sourceControlHosts.bitbucket?.accessToken, SECRET_REDACTED); + assert.equal(legacy.usageLimitSources[hub]?.managementKey, SECRET_REDACTED); + assert.equal(legacy.easDeviceHosts[0]?.accessToken, SECRET_REDACTED); + const echoed = yield* serverSettings.updateSettings({ + sourceControlHosts: legacy.sourceControlHosts, + usageLimitSources: legacy.usageLimitSources, + easDeviceHosts: legacy.easDeviceHosts, + }); + assert.deepEqual(echoed.sourceControlHosts.bitbucket?.accessToken, reference); + assert.deepEqual(echoed.sourceControlHosts.github?.tokens, { + "github.com": gitHubReference, + "ghe.acme.test": "ghp_ghe", + }); + assert.deepEqual(echoed.usageLimitSources[hub]?.managementKey, usageReference); + assert.deepEqual(echoed.easDeviceHosts[0]?.accessToken, easReference); + assert.isTrue(Option.isSome(yield* stored("github", "tokens", "ghe.acme.test"))); }).pipe(Effect.provide(layerServerSettingsWithSecrets())), ); + it.effect("knows every strictly decoded field that can hold a secret reference", () => + Effect.sync(() => { + // Shipped clients decode these as strings. Each must be covered by the older-client case above. + assert.deepEqual(strictSecretReferencePaths(ServerSettings.ast), [ + "easDeviceHosts.[].accessToken", + "usageLimitSources.*.managementKey", + ]); + }), + ); + it.effect("keeps the retired keys for another load when a token cannot be copied", () => { const cause = new ServerSecretStore.SecretStoreReadError({ resource: "bitbucket-access-token", diff --git a/apps/server/src/serverSettings.ts b/apps/server/src/serverSettings.ts index e9fbe510a0db0..498006641484a 100644 --- a/apps/server/src/serverSettings.ts +++ b/apps/server/src/serverSettings.ts @@ -18,6 +18,8 @@ import { type ServerSettingsPatch, type ManagedPolicySummary, globalIdleCompaction, + isSecretReference, + type SecretReference, withIdleCompaction, } from "@t3tools/contracts"; import * as Cache from "effect/Cache"; @@ -60,7 +62,12 @@ import { sourceControlHostSecretName, } from "./sourceControl/sourceControlHostSecrets.ts"; -import { mapSettingsSecrets, SECRET_REDACTED, type SettingsSecret } from "./settingsSecrets.ts"; +import { + mapSettingsSecrets, + mapSettingsSecretValues, + SECRET_REDACTED, + type SettingsSecret, +} from "./settingsSecrets.ts"; export { resolveReviewGuideModelSelection, @@ -159,13 +166,35 @@ const normalizeServerSettings = ( ), ); -export function redactServerSettingsForClient(settings: ServerSettings): ServerSettings { +export function redactServerSettingsForClient( + settings: ServerSettings, + client: { readonly secretReferences: boolean }, +): ServerSettings { const redacted = mapSettingsSecrets(settings, (secret) => secret.sensitive ? { value: "", redacted: secret.redacted || secret.value.length > 0 } : { value: secret.value, redacted: false }, ); - return { ...redacted, secretProviders: withoutUnknownSecretProviderConfigs(redacted) }; + // Older clients decode secret fields as strings and fail on a reference, so they see a saved secret. + const shaped = client.secretReferences + ? redacted + : mapSettingsSecretValues(redacted, (value) => + isSecretReference(value) ? SECRET_REDACTED : value, + ); + return { ...shaped, secretProviders: withoutUnknownSecretProviderConfigs(shaped) }; +} + +/** A client shown a reference as a saved secret sends that marker back; it keeps the reference. */ +function withEchoedSecretReferences(current: ServerSettings, next: ServerSettings): ServerSettings { + const references = new Map(); + mapSettingsSecretValues(current, (value, secretName) => { + if (isSecretReference(value)) references.set(secretName, value); + return value; + }); + if (references.size === 0) return next; + return mapSettingsSecretValues(next, (value, secretName) => + value === SECRET_REDACTED ? (references.get(secretName) ?? value) : value, + ); } // A newer kind's config may hold credentials this build cannot recognize, so it never leaves the server. @@ -1107,7 +1136,7 @@ const make = Effect.gen(function* () { }); const changes: SecretChange[] = []; const nextSecretNames = new Set(); - const settings = mapSettingsSecrets(next, (secret) => { + const settings = mapSettingsSecrets(withEchoedSecretReferences(current, next), (secret) => { const metadata = { secretName: secret.secretName, ...(secret.providerInstanceId !== undefined diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 70550f2379272..1da6c2e195140 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -1,5 +1,7 @@ // @effect-diagnostics-next-line nodeBuiltinImport:off - observes the children the launcher spawns with Node built-ins. import type * as NodeChildProcess from "node:child_process"; +// @effect-diagnostics-next-line nodeBuiltinImport:off - the free-space probe is a plain Promise callback. +import * as NodeFS from "node:fs"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; @@ -618,6 +620,35 @@ setInterval(() => {}, 1_000); }), ); + it.effect("reclaims partial backups left by killed updates before checking space", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-orphan-" }); + const databasePath = path.join(root, "state.sqlite"); + yield* writeSparseFile(databasePath, 10 * 1024 ** 3); + const backupRoot = path.join(root, "runtime", "db-backup"); + const orphan = path.join(backupRoot, "earlier-update.staging"); + const completed = path.join(backupRoot, "completed-update"); + yield* fs.makeDirectory(orphan, { recursive: true }); + yield* fs.writeFileString(path.join(orphan, "database"), "partial copy"); + yield* fs.makeDirectory(completed); + yield* fs.writeFileString(path.join(completed, "database"), "complete copy"); + + // The orphan is what keeps this disk too full for a full copy. + const blocker = yield* Effect.promise(() => + databaseBackupBlocker(root, databasePath, { + readFreeBytes: () => Promise.resolve((NodeFS.existsSync(orphan) ? 8 : 11) * 1024 ** 3), + canClone: () => Promise.resolve(false), + }), + ); + + assert.isUndefined(blocker); + assert.isFalse(yield* fs.exists(orphan)); + assert.isTrue(yield* fs.exists(path.join(completed, "database"))); + }), + ); + it.effect("checks database backup space against clone or full-copy needs", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 9f764b788efe2..ee05c63297843 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -219,6 +219,17 @@ const filesystemBackupSpaceProbe: BackupSpaceProbe = { canClone: backupCanClone, }; +// A killed launcher skips the backup's cleanup, and retries only clear their own update's copy. +// Update requests are refused while one is pending, so no staging directory is still in use here. +async function removeAbandonedBackupCopies(backupRoot: string): Promise { + for (const entry of await NodeFSP.readdir(backupRoot)) { + if (!entry.endsWith(".staging")) continue; + await NodeFSP.rm(NodePath.join(backupRoot, entry), { recursive: true, force: true }).catch( + () => undefined, + ); + } +} + /** Check before stopping the server so full disks refuse updates without an outage. */ export async function databaseBackupBlocker( baseDir: string, @@ -228,6 +239,7 @@ export async function databaseBackupBlocker( const { readFreeBytes, canClone } = { ...filesystemBackupSpaceProbe, ...probe }; const backupRoot = NodePath.join(baseDir, "runtime", "db-backup"); await NodeFSP.mkdir(backupRoot, { recursive: true, mode: 0o700 }); + await removeAbandonedBackupCopies(backupRoot); const [files, free] = await Promise.all([statDatabaseFiles(dbPath), readFreeBytes(backupRoot)]); const size = files.reduce((total, file) => total + file.size, 0); const fullCopyBytes = size * 1.05; diff --git a/apps/server/src/settingsSecrets.ts b/apps/server/src/settingsSecrets.ts index 1bebce51b3876..17640237ff5ca 100644 --- a/apps/server/src/settingsSecrets.ts +++ b/apps/server/src/settingsSecrets.ts @@ -3,7 +3,9 @@ import { ClaudeSettings, CodexSettings, EasDeviceHostConfig, + type ProviderInstanceConfig, type ServerSettings, + type SettingsSecretValue, SimlockDeviceHostConfig, UsageLimitSourceConfig, isSecretReference, @@ -58,16 +60,19 @@ export interface SettingsSecret { } type SecretValue = Pick; +type SecretSlotName = (field: string, serverHost: string | null) => string; +type SecretFieldsMapper = ( + fields: Readonly>, + secretFields: ReadonlyArray, + name: SecretSlotName, +) => Record; +type ProviderEnvironment = NonNullable; export function mapSettingsSecrets( settings: ServerSettings, map: (secret: SettingsSecret) => SecretValue, ): ServerSettings { - const mapFields = ( - fields: Readonly>, - secretFields: ReadonlyArray, - name: (field: string, serverHost: string | null) => string, - ) => { + const mapFields: SecretFieldsMapper = (fields, secretFields, name) => { const next = { ...fields }; for (const field of secretFields) { if (!(field in next) || isSecretReference(next[field])) continue; @@ -88,6 +93,70 @@ export function mapSettingsSecrets( return next; }; + return mapSecretFieldGroups(settings, mapFields, (instanceId, environment) => + environment.map((variable) => { + const { valueRedacted: _omit, ...rest } = variable; + // A secret reference is not itself secret; storage sees the value cleared. + const reference = typeof variable.value === "string" ? undefined : variable.value; + const result = map({ + secretName: `provider-env-${encode(instanceId)}-${encode(variable.name)}`, + kind: "environment", + value: typeof variable.value === "string" ? variable.value : "", + redacted: reference === undefined && variable.valueRedacted === true, + sensitive: variable.sensitive, + providerInstanceId: instanceId, + environmentVariable: variable.name, + }); + if (reference !== undefined) return { ...rest, value: reference }; + return { + ...rest, + value: result.value, + ...(result.redacted ? { valueRedacted: true } : {}), + }; + }), + ); +} + +/** + * Maps each value a secret field holds, literal or reference, with the secret name of its slot. + * Provider environment variables are left as they are. + */ +export function mapSettingsSecretValues( + settings: ServerSettings, + map: (value: SettingsSecretValue, secretName: string) => SettingsSecretValue, +): ServerSettings { + const mapValue = (value: unknown, secretName: string) => + typeof value === "string" || isSecretReference(value) ? map(value, secretName) : value; + return mapSecretFieldGroups(settings, (fields, secretFields, name) => { + const next = { ...fields }; + for (const field of secretFields) { + if (!(field in next)) continue; + const value = next[field]; + next[field] = + value !== null && + typeof value === "object" && + !Array.isArray(value) && + !isSecretReference(value) + ? Object.fromEntries( + Object.entries(value).map(([serverHost, entry]) => [ + serverHost, + mapValue(entry, name(field, serverHost.trim().toLowerCase())), + ]), + ) + : mapValue(value, name(field, null)); + } + return next; + }); +} + +function mapSecretFieldGroups( + settings: ServerSettings, + mapFields: SecretFieldsMapper, + mapEnvironment: (instanceId: string, environment: ProviderEnvironment) => ProviderEnvironment = ( + _instanceId, + environment, + ) => environment, +): ServerSettings { const providerInstances = Object.fromEntries( Object.entries(settings.providerInstances).map(([instanceId, instance]) => { const config = instance.config; @@ -107,28 +176,7 @@ export function mapSettingsSecrets( } : {}), ...(instance.environment - ? { - environment: instance.environment.map((variable) => { - const { valueRedacted: _omit, ...rest } = variable; - // A secret reference is not itself secret; storage sees the value cleared. - const reference = typeof variable.value === "string" ? undefined : variable.value; - const result = map({ - secretName: `provider-env-${encode(instanceId)}-${encode(variable.name)}`, - kind: "environment", - value: typeof variable.value === "string" ? variable.value : "", - redacted: reference === undefined && variable.valueRedacted === true, - sensitive: variable.sensitive, - providerInstanceId: instanceId, - environmentVariable: variable.name, - }); - if (reference !== undefined) return { ...rest, value: reference }; - return { - ...rest, - value: result.value, - ...(result.redacted ? { valueRedacted: true } : {}), - }; - }), - } + ? { environment: mapEnvironment(instanceId, instance.environment) } : {}), }, ]; diff --git a/apps/server/src/sourceControl/SourceControlProviderRegistry.test.ts b/apps/server/src/sourceControl/SourceControlProviderRegistry.test.ts index 05b6a1a8d0e42..c532d93e81ad9 100644 --- a/apps/server/src/sourceControl/SourceControlProviderRegistry.test.ts +++ b/apps/server/src/sourceControl/SourceControlProviderRegistry.test.ts @@ -199,6 +199,24 @@ it.effect("leaves a remote no integration claims unknown, with its server's addr }), ); +it.effect.each([ + ["github.com:team/project.git", "github"], + ["gitlab.com:team/project.git", "gitlab"], + ["bitbucket.org:team/project.git", "bitbucket"], + ["code.example.test:team/project.git", "unknown"], +] as const)( + "reads the host of an SSH remote that leaves its user to ssh config: %s", + ([url, expected]) => + Effect.gen(function* () { + const registry = yield* makeRegistry({ remotes: [{ name: "origin", url }] }); + + const handle = yield* registry.resolveHandle({ cwd: "/repo" }); + + assert.strictEqual(handle.provider.kind, kind(expected)); + assert.strictEqual(handle.context?.remoteUrl, url); + }), +); + it.effect("refines the caller-selected remote instead of choosing another configured remote", () => Effect.gen(function* () { const registry = yield* makeRegistry({ diff --git a/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts b/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts index dedf3636df4ec..326b7931da292 100644 --- a/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts +++ b/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts @@ -648,6 +648,52 @@ it.effect("explains a rejected SSH push after creating the repository", () => }).pipe(Effect.provide(layer({ git: failingPush("authentication_failed") }))), ); +// regression: an auto-selected SSH remote kept failing every later fetch after a refused push +it.effect("switches an auto-selected remote to HTTPS when the SSH push is refused", () => { + const gitArgs: Array> = []; + let pushAttempts = 0; + return Effect.gen(function* () { + const service = yield* SourceControlRepositoryService.SourceControlRepositoryService; + const result = yield* service.publishRepository({ + cwd: "/workspace", + provider: SourceControlProviderKind.make("github"), + repository: "octocat/t3code", + visibility: "private", + }); + + assert.strictEqual(result.status, "pushed"); + assert.strictEqual(result.remoteUrl, CLONE_URLS.url); + assert.strictEqual(pushAttempts, 2); + assert.deepStrictEqual(gitArgs, [ + ["rev-parse", "--verify", "HEAD"], + ["remote", "set-url", "origin", CLONE_URLS.url], + ]); + }).pipe( + Effect.provide( + layer({ + git: { + execute: (input) => + Effect.sync(() => { + gitArgs.push(input.args); + return processOutput(); + }), + pushCurrentBranch: (cwd) => { + pushAttempts += 1; + return pushAttempts === 1 + ? failingPush("authentication_failed").pushCurrentBranch(cwd) + : Effect.succeed({ + status: "pushed" as const, + branch: "main", + upstreamBranch: "origin/main", + setUpstream: true, + }); + }, + }, + }), + ), + ); +}); + // regression: #19219 publishing did not explain why the first push failed it.effect("says the repository exists when an unclassified publish push fails", () => Effect.gen(function* () { diff --git a/apps/server/src/sourceControl/SourceControlRepositoryService.ts b/apps/server/src/sourceControl/SourceControlRepositoryService.ts index 1e9c48893dad7..64242b72d4d58 100644 --- a/apps/server/src/sourceControl/SourceControlRepositoryService.ts +++ b/apps/server/src/sourceControl/SourceControlRepositoryService.ts @@ -502,28 +502,50 @@ export const make = Effect.gen(function* () { }; } - const pushResult = yield* git.pushCurrentBranch(input.cwd, null, { remoteName }).pipe( - Effect.mapError( - (cause) => - new SourceControlRepositoryError({ - operation: "publishRepository", - provider: providerKind, - detail: publishPushFailureDetail({ - urls, - remoteName, - remoteUrl, - reason: cause.reason, + const push = (pushUrl: string) => + git.pushCurrentBranch(input.cwd, null, { remoteName }).pipe( + Effect.map((result) => ({ branch: result.branch, remoteUrl: pushUrl })), + Effect.mapError( + (cause) => + new SourceControlRepositoryError({ + operation: "publishRepository", + provider: providerKind, + detail: publishPushFailureDetail({ + urls, + remoteName, + remoteUrl: pushUrl, + reason: cause.reason, + }), + cause, }), - cause, - }), + ), + ); + + // Like clone, an auto-selected SSH remote switches to HTTPS when SSH is refused; + // otherwise the remote keeps failing every later fetch and push. + const canSwitchToHttps = input.protocol !== "ssh" && remoteUrl === urls.sshUrl; + const pushed = yield* push(remoteUrl).pipe( + Effect.catchIf( + (error) => canSwitchToHttps && isSshRejection(error), + (sshError) => + git + .execute({ + operation: "SourceControlRepositoryService.publishRepository.switchToHttps", + cwd: input.cwd, + args: ["remote", "set-url", remoteName, urls.url], + }) + .pipe( + Effect.mapError(() => sshError), + Effect.andThen(push(urls.url)), + ), ), ); return { repository: toRepositoryInfo(providerKind, urls), remoteName, - remoteUrl, - branch: pushResult.branch, + remoteUrl: pushed.remoteUrl, + branch: pushed.branch, status: "pushed" as const, }; }, diff --git a/apps/server/src/sourceControl/sourceControlDefinitions.test.ts b/apps/server/src/sourceControl/sourceControlDefinitions.test.ts index e21efd8173a73..36996bdf3f370 100644 --- a/apps/server/src/sourceControl/sourceControlDefinitions.test.ts +++ b/apps/server/src/sourceControl/sourceControlDefinitions.test.ts @@ -10,3 +10,15 @@ it("runs a driver for every host the server and clients read, in their order", ( sourceControlDefinitions.definitions, ); }); + +it.each([ + "C:/repos/project", + "C:\\repos\\project", + "/srv/git/repo.git", + "./team:group/repo", + "file:///srv/git/repo.git", + "hg::team/repo", + "codecommit::us-east-1://repo", +])("finds no host in %s, which git does not read as SSH", (remoteUrl) => { + expect(sourceControlDefinitions.recognizeRemote(remoteUrl)).toBeNull(); +}); diff --git a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts index be1da6f3d610c..974fc7cc91c49 100644 --- a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts +++ b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts @@ -1,5 +1,5 @@ import { assert, it } from "@effect/vitest"; -import { SecretProviderId } from "@t3tools/contracts"; +import { DeviceHostId, SecretProviderId, UsageLimitSourceId } from "@t3tools/contracts"; import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; import * as Effect from "effect/Effect"; @@ -125,12 +125,44 @@ it.effect("reads a collected reference again once it is invalidated", () => }), }); const read = () => resolveSourceControlHostSecrets(resolver, hosts, "bitbucket"); + const usageLimitSources = { + [UsageLimitSourceId.make("referenced")]: { + kind: "cliproxy", + url: "https://hub.test", + managementKey: source("cliproxy#key"), + enabled: true, + }, + [UsageLimitSourceId.make("literal")]: { + kind: "cliproxy", + url: "https://hub.test", + managementKey: "plain-key", + enabled: true, + }, + } as const; + const easDeviceHosts = [ + { + kind: "eas" as const, + id: DeviceHostId.make("ref"), + label: "Ref", + maxSessionMinutes: 30, + idleMinutes: 10, + maxConcurrentSessions: 2, + accessToken: source("expo#token"), + }, + ]; + cache.set("cliproxy#key", "stale"); + cache.set("expo#token", "stale"); assert.deepEqual((yield* read()).bitbucket, { email: "me@example.test", accessToken: "stale", }); current = "rotated"; - yield* invalidateSettingsSecrets(resolver, Effect.succeed({ sourceControlHosts: hosts })); + yield* invalidateSettingsSecrets( + resolver, + Effect.succeed({ sourceControlHosts: hosts, usageLimitSources, easDeviceHosts }), + ); + assert.isFalse(cache.has("cliproxy#key")); + assert.isFalse(cache.has("expo#token")); assert.deepEqual((yield* read()).bitbucket, { email: "me@example.test", accessToken: "rotated", diff --git a/apps/server/src/sourceControl/sourceControlHostSecrets.ts b/apps/server/src/sourceControl/sourceControlHostSecrets.ts index 82de25be4de69..21213d27bdb8b 100644 --- a/apps/server/src/sourceControl/sourceControlHostSecrets.ts +++ b/apps/server/src/sourceControl/sourceControlHostSecrets.ts @@ -97,13 +97,32 @@ export const resolveSourceControlHostSecrets = Effect.fn("SourceControlHost.reso }, ); +const collectUsageLimitSecretReferences = ( + sources: ServerSettings["usageLimitSources"], +): ReadonlyArray => + Object.values(sources).flatMap(({ managementKey }) => + isSecretReference(managementKey) ? [managementKey] : [], + ); + +const collectEasSecretReferences = ( + hosts: ServerSettings["easDeviceHosts"], +): ReadonlyArray => + hosts.flatMap(({ accessToken }) => (isSecretReference(accessToken) ? [accessToken] : [])); + /** Settings that cannot be read clear every cached secret, so a refresh never keeps a stale one. */ export const invalidateSettingsSecrets = Effect.fn("SettingsSecrets.invalidate")(function* ( resolver: SecretResolver.SecretResolver["Service"], - settings: Effect.Effect, E>, + settings: Effect.Effect< + Pick, + E + >, ) { const references = yield* settings.pipe( - Effect.map((current) => collectSourceControlHostSecretReferences(current.sourceControlHosts)), + Effect.map((current) => [ + ...collectSourceControlHostSecretReferences(current.sourceControlHosts), + ...collectUsageLimitSecretReferences(current.usageLimitSources), + ...collectEasSecretReferences(current.easDeviceHosts), + ]), Effect.orElseSucceed(() => undefined), ); yield* resolver.invalidate(references); diff --git a/apps/server/src/storageCleanup.test.ts b/apps/server/src/storageCleanup.test.ts index 9be6202362a4c..97d61fb809666 100644 --- a/apps/server/src/storageCleanup.test.ts +++ b/apps/server/src/storageCleanup.test.ts @@ -858,6 +858,162 @@ it.live("preserves late tracked edits when submodules were never initialized", ( ); describe("subagent worktree ownership", () => { + it.effect.each([ + "child", + "grandchild", + "immediate", + "active-parent", + "recent-settle", + "settle-grace", + "recent-child-activity", + "running-child", + "pending-child", + "running-parent", + "pending-parent", + "unsettled-child", + "pinned-child", + "fork", + "missing-parent", + "cycle", + ] as const)("checks isolated subagent settlement (%s)", (scenario) => + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const fixture = yield* cleanupFixture({ + ...DEFAULT_SERVER_SETTINGS.storageCleanup, + worktreeSettledAfterDays: scenario === "immediate" ? 0 : 8, + }); + const root = shell({ + settledAt: + scenario === "active-parent" + ? null + : scenario === "recent-settle" || scenario === "settle-grace" + ? at(-30_000) + : scenario === "immediate" + ? at(-120_000) + : at(-20 * DAY_MS), + settledOverride: scenario === "active-parent" ? null : "settled", + updatedAt: scenario === "settle-grace" ? at(-30_000) : at(-10 * DAY_MS), + }); + const child = shell({ + id: ThreadId.make("child"), + branch: "feature", + worktreePath: fixture.worktree, + lineage: { + parentThreadId: + scenario === "missing-parent" + ? ThreadId.make("missing") + : scenario === "cycle" + ? ThreadId.make("child") + : ["grandchild", "running-parent", "pending-parent"].includes(scenario) + ? ThreadId.make("middle") + : root.id, + rootThreadId: root.id, + relationshipToParent: scenario === "fork" ? "fork" : "subagent", + }, + status: scenario === "running-child" ? "running" : "completed", + pendingBackgroundTasks: + scenario === "pending-child" ? [{ taskId: "task", kind: "command" }] : [], + latestRunCompletedAt: scenario === "recent-child-activity" ? at(-2 * DAY_MS) : null, + settledOverride: scenario === "unsettled-child" ? "active" : null, + pinnedAt: scenario === "pinned-child" ? at(-DAY_MS) : null, + }); + const middle = shell({ + id: ThreadId.make("middle"), + status: scenario === "running-parent" ? "running" : "idle", + pendingBackgroundTasks: + scenario === "pending-parent" ? [{ taskId: "task", kind: "command" }] : [], + lineage: { + parentThreadId: root.id, + rootThreadId: root.id, + relationshipToParent: "subagent", + }, + }); + fixture.setThreads([child, middle, root]); + const report = yield* fixture.service.runNow; + const removed = ["child", "grandchild", "immediate"].includes(scenario); + expect(yield* fixture.fs.exists(fixture.worktree)).toBe(!removed); + expect(report.entries[0]).toMatchObject({ + threadId: child.id, + outcome: removed ? "removed" : "kept", + reason: removed + ? scenario === "immediate" + ? "Settled for 0 days" + : "Settled for 20 days" + : scenario === "settle-grace" + ? "Settled moments ago" + : scenario === "recent-settle" + ? "Settled for 0 of 8 days" + : scenario === "recent-child-activity" + ? "Settled for 2 of 8 days" + : scenario === "running-child" || scenario === "pending-child" + ? "Thread is running or has pending work" + : "Not settled", + }); + }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + ); + + it.live.each([ + "parent-unsettled", + "parent-resettled", + "child-unsettled", + "intermediate-running", + "intermediate-pending", + ] as const)("cancels inherited settlement during measurement (%s)", (change) => + runCleanupTest( + Effect.gen(function* () { + const fixture = yield* pausedMeasurementFixture({ + ...DEFAULT_SERVER_SETTINGS.storageCleanup, + worktreeSettledAfterDays: 0, + }); + const root = shell({ settledAt: at(-20 * DAY_MS), settledOverride: "settled" }); + const middle = shell({ + id: ThreadId.make("middle"), + lineage: { + parentThreadId: root.id, + rootThreadId: root.id, + relationshipToParent: "subagent", + }, + }); + const child = shell({ + id: ThreadId.make("child"), + branch: "feature", + worktreePath: fixture.worktree, + lineage: { + parentThreadId: middle.id, + rootThreadId: root.id, + relationshipToParent: "subagent", + }, + }); + fixture.setThreads([child, middle, root]); + const run = yield* fixture.service.runNow.pipe(Effect.forkChild); + yield* Deferred.await(fixture.measuring); + fixture.setThreads([ + { ...child, settledOverride: change === "child-unsettled" ? "active" : null }, + { + ...middle, + status: change === "intermediate-running" ? "running" : "idle", + pendingBackgroundTasks: + change === "intermediate-pending" ? [{ taskId: "task", kind: "command" }] : [], + }, + change === "child-unsettled" || change.startsWith("intermediate-") + ? root + : { + ...root, + settledAt: change === "parent-resettled" ? at(0) : null, + settledOverride: change === "parent-resettled" ? "settled" : "active", + updatedAt: at(0), + }, + ]); + yield* Deferred.succeed(fixture.resume, undefined); + expect((yield* Fiber.join(run)).entries[0]).toMatchObject({ + outcome: "kept", + reason: "Parent settlement changed since check", + }); + expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true); + }), + ), + ); + it.effect.each([ "idle", "grandchild", diff --git a/apps/server/src/storageCleanup.ts b/apps/server/src/storageCleanup.ts index 2ab0b7263b7b7..bb82e44253470 100644 --- a/apps/server/src/storageCleanup.ts +++ b/apps/server/src/storageCleanup.ts @@ -18,6 +18,7 @@ import type { WorktreeCleanupRules, StorageCleanupReport, StorageCleanupReportEntry, + ThreadId, } from "@t3tools/contracts"; import { resolveWorktreeCleanup } from "@t3tools/shared/projectSettings"; import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; @@ -404,9 +405,16 @@ export function storageCleanupThreadIdle( thread: OrchestrationV2ThreadShell, now: number, settings: LimitRecoverySettings, +): boolean { + return thread.worktreePath !== null && threadIdle(thread, now, settings); +} + +function threadIdle( + thread: OrchestrationV2ThreadShell, + now: number, + settings: LimitRecoverySettings, ): boolean { return ( - thread.worktreePath !== null && thread.activeRunId === null && (thread.status === "idle" || thread.status === "completed" || @@ -493,6 +501,36 @@ function worktreeOwner, + now: number, + settings: LimitRecoverySettings, +) { + const chain = [thread]; + let current = thread; + while (current.settledAt === null) { + if ( + current.lineage.relationshipToParent !== "subagent" || + current.settledOverride === "active" || + current.pinnedAt != null + ) + return []; + const parent = byId.get(current.lineage.parentThreadId!); + if ( + parent === undefined || + chain.some((member) => member.id === parent.id) || + parent.projectId !== thread.projectId || + parent.lineage.rootThreadId !== thread.lineage.rootThreadId || + !threadIdle(parent, now, settings) + ) + return []; + chain.push(parent); + current = parent; + } + return chain; +} + /** Squash/rebase merges require the merged PR to name this exact head commit. */ export function storageCleanupPullRequestMerged( pullRequest: Pick< @@ -1061,6 +1099,7 @@ const make = Effect.gen(function* () { ), )).filter((thread) => thread.worktreePath !== null && thread.branch !== null); const snapshot = yield* readThreads(); + const threadsById = new Map(snapshot.threads.map((thread) => [thread.id, thread])); const refreshedDefaultRefs = new Map>(); const defaultBranches = new Map(); const groups = Map.groupBy( @@ -1094,6 +1133,10 @@ const make = Effect.gen(function* () { const group = groups.get(worktreePath) ?? []; const activityAt = Math.max(...group.map(storageCleanupActivityAt)); const deleted = "workspaceRoot" in thread; + const settlementChain = deleted + ? [] + : worktreeSettlementChain(thread, threadsById, now, serverSettings); + const settledThread = settlementChain.at(-1); const members = deleted ? [thread, ...group] : group; const project = deleted ? { workspaceRoot: thread.workspaceRoot } @@ -1152,7 +1195,7 @@ const make = Effect.gen(function* () { ); if (changes !== null) return keep(changes); if ( - group.some( + [...group, ...settlementChain.slice(1)].some( (member) => member.settledAt !== null && now - Math.max(DateTime.toEpochMillis(member.updatedAt), activityAt) < @@ -1166,9 +1209,9 @@ const make = Effect.gen(function* () { settings.worktreeAfterDays !== null && activityAt < now - settings.worktreeAfterDays * DAY_MS; const settledFor = - deleted || thread.settledAt === null + settledThread?.settledAt == null ? null - : now - Math.max(DateTime.toEpochMillis(thread.settledAt), activityAt); + : now - Math.max(DateTime.toEpochMillis(settledThread.settledAt), activityAt); const settledDays = settledFor === null ? null : Math.floor(settledFor / DAY_MS); const settled = settledFor !== null && @@ -1335,6 +1378,27 @@ const make = Effect.gen(function* () { }) ) return waitForStop("thread activity or shared worktree changed since check"); + if (settlementChain.length > 1) { + const ancestors = yield* Effect.forEach(settlementChain.slice(1), (ancestor) => + projections.getThreadShell(ancestor.id), + ); + const latestSettlement = worktreeSettlementChain( + latestGroup.find((shell) => shell.id === thread.id)!, + new Map( + [...latestGroup, ...ancestors.filter((ancestor) => ancestor !== null)].map( + (ancestor) => [ancestor.id, ancestor], + ), + ), + now, + serverSettings, + ).at(-1); + if ( + latestSettlement?.id !== settledThread?.id || + !Equal.equals(latestSettlement?.settledAt, settledThread?.settledAt) || + !Equal.equals(latestSettlement?.updatedAt, settledThread?.updatedAt) + ) + return "parent settlement changed since check"; + } // Sessions can outlive their run and can be shared across app threads. const sessionRows = yield* sql<{ payload_json: string }>` SELECT payload_json FROM orchestration_v2_projection_provider_sessions diff --git a/apps/server/src/telemetry/AnalyticsService.test.ts b/apps/server/src/telemetry/AnalyticsService.test.ts index 98e3d640c604c..4d3c2d5421279 100644 --- a/apps/server/src/telemetry/AnalyticsService.test.ts +++ b/apps/server/src/telemetry/AnalyticsService.test.ts @@ -5,8 +5,11 @@ import { assert, it } from "@effect/vitest"; import * as Clock from "effect/Clock"; import * as ConfigProvider from "effect/ConfigProvider"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; import * as TestClock from "effect/testing/TestClock"; import * as HttpClient from "effect/http/HttpClient"; import * as HttpClientError from "effect/http/HttpClientError"; @@ -110,6 +113,30 @@ const recordTwentyEvents = Effect.gen(function* () { } }); +const analyticsLayerWith = (client: HttpClient.HttpClient, policy?: ManagedPolicyDocument) => + AnalyticsService.layer.pipe( + Layer.provideMerge( + ServerConfig.ServerConfig.layerTest(process.cwd(), { prefix: "t3-telemetry-retry-" }), + ), + Layer.provide( + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + T3CODE_TELEMETRY_ENABLED: true, + T3CODE_POSTHOG_KEY: "phc_test_key", + T3CODE_POSTHOG_HOST: "http://localhost", + }), + ), + ), + Layer.provide( + Layer.mergeAll( + Layer.succeed(HostProcess.Platform, "win32"), + Layer.succeed(HostProcess.Architecture, "x64"), + Layer.succeed(HttpClient.HttpClient, client), + ManagedPolicy.layerTest(policy), + ), + ), + ); + /** * Runs `record` (20 events by default), then lets `minutes` of background * flushes run against a client that answers every send with `respond`. @@ -138,28 +165,7 @@ const sendFor = ( return yield* respond(request); }), ); - const runtimeLayer = AnalyticsService.layer.pipe( - Layer.provideMerge( - ServerConfig.ServerConfig.layerTest(process.cwd(), { prefix: "t3-telemetry-retry-" }), - ), - Layer.provide( - ConfigProvider.layer( - ConfigProvider.fromUnknown({ - T3CODE_TELEMETRY_ENABLED: true, - T3CODE_POSTHOG_KEY: "phc_test_key", - T3CODE_POSTHOG_HOST: "http://localhost", - }), - ), - ), - Layer.provide( - Layer.mergeAll( - Layer.succeed(HostProcess.Platform, "win32"), - Layer.succeed(HostProcess.Architecture, "x64"), - Layer.succeed(HttpClient.HttpClient, client), - ManagedPolicy.layerTest(policy), - ), - ), - ); + const runtimeLayer = analyticsLayerWith(client, policy); yield* Effect.gen(function* () { yield* record; @@ -271,6 +277,21 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("AnalyticsService }), ); + it.effect("shutdown stops waiting for a telemetry endpoint that never answers", () => + Effect.gen(function* () { + const scope = yield* Scope.make(); + const context = yield* Layer.buildWithScope( + analyticsLayerWith(HttpClient.make(() => Effect.never)), + scope, + ); + yield* recordTwentyEvents.pipe(Effect.provide(context)); + + const shutdown = yield* Scope.close(scope, Exit.void).pipe(Effect.forkChild); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(shutdown); + }), + ); + it.effect("flush drains all buffered events across multiple batches", () => Effect.gen(function* () { const capturedRequests: Array = []; diff --git a/apps/server/src/telemetry/AnalyticsService.ts b/apps/server/src/telemetry/AnalyticsService.ts index 91b00eeb12335..384da92b3f312 100644 --- a/apps/server/src/telemetry/AnalyticsService.ts +++ b/apps/server/src/telemetry/AnalyticsService.ts @@ -41,8 +41,10 @@ interface DeliveryState { // Retry delays shorter than this still wait for the next background check. const FLUSH_INTERVAL_MS = 10_000; -// A hung send would hold the flush lock, and with it the shutdown flush. +// A hung send would hold the flush lock for the next background flush. const SEND_TIMEOUT = "10 seconds"; +// Quit and restart wait for this flush; events it cannot send in time are dropped. +const SHUTDOWN_FLUSH_TIMEOUT = "1 second"; // A failed batch is tried again until it has failed for this long. Ingest answers 503 when it // cannot store a batch, and a broken Cloudflare location can take about 5 minutes to fail over. // The waiting batch holds newer events back; `maxBufferedEvents` bounds them meanwhile. @@ -352,7 +354,9 @@ export const make = Effect.gen(function* () { disableYield: true, }).pipe(Effect.forkScoped); - yield* Effect.addFinalizer(() => flush); + yield* Effect.addFinalizer(() => + flush.pipe(Effect.timeout(SHUTDOWN_FLUSH_TIMEOUT), Effect.ignore), + ); return AnalyticsService.of({ record, flush, recordOwners }); }); diff --git a/apps/server/src/terminal/Manager.test.ts b/apps/server/src/terminal/Manager.test.ts index eaabe14784b83..5620221473e3c 100644 --- a/apps/server/src/terminal/Manager.test.ts +++ b/apps/server/src/terminal/Manager.test.ts @@ -1122,7 +1122,8 @@ it.layer( const { manager, ptyAdapter } = yield* createManager(); yield* manager.open(openInput()); const process = ptyAdapter.processes[0]!; - const before = "café\t界🙂\u0007\u009c\u0085"; + // Mojibake like "â€\u009d" (for ”) carries C1 code points that libghostty prints as text. + const before = "café\t界🙂\u0007\u009c\u0085 â€\u009d Ð\u0090 \u009e\u009f\u009b6n"; const after = "e\u0301\r\n"; process.emitData(before); @@ -1146,17 +1147,18 @@ it.layer( process.emitData("\u001b[?2026$p\u001b[?2026;2$y\u001b[>q\u001b[?u\u001b[?31u"); process.emitData("\u001bP$q m\u001b\\\u001bP1$r0m\u001b\\"); process.emitData("\u001bP+q544e\u001b\\\u001bP1+r544e=1b\u001b\\"); - process.emitData("\u0090$q m\u009c\u00901$r0m\u009c"); - process.emitData("\u0090+q544e\u009c\u00901+r544e=1b\u009c"); process.emitData('\u001b[!p\u001b["p\u001b[4 q\u001b[u'); + process.emitData("\u001b]0;x\u001b\u001b[?2026$p"); process.emitData("done\n"); yield* manager.close({ threadId: "thread-1" }); const reopened = yield* manager.open(openInput()); - expect(reopened.history.startsWith('prompt \u001b[!p\u001b["p\u001b[4 q\u001b[udone\n')).toBe( - true, - ); + expect( + reopened.history.startsWith( + 'prompt \u001b[!p\u001b["p\u001b[4 q\u001b[u\u001b]0;x\u001bdone\n', + ), + ).toBe(true); }), ); @@ -1166,17 +1168,13 @@ it.layer( { name: "PM", sequence: "\u001b^unfinished" }, { name: "APC", sequence: "\u001b_unfinished" }, { name: "CSI", sequence: "\u001b[5" }, - { name: "C1 OSC", sequence: "\u009d0;unfinished" }, - { name: "C1 DCS", sequence: "\u0090unfinished" }, - { name: "C1 PM", sequence: "\u009eunfinished" }, - { name: "C1 APC", sequence: "\u009funfinished" }, - { name: "C1 CSI", sequence: "\u009b5" }, ])("keeps output after a cancelled $name sequence in history", ({ sequence }) => Effect.gen(function* () { - const { manager, ptyAdapter } = yield* createManager(); + const { manager, ptyAdapter } = yield* createManager(10); yield* manager.open(openInput()); const process = ptyAdapter.processes[0]!; - for (const cancel of ["\u0018", "\u001a"]) { + // A new escape sequence also aborts one, as in xterm.js; keep it last so CAN cannot end it. + for (const cancel of ["\u0018", "\u001a", "\u001b[0m"]) { process.emitData(sequence); process.emitData(cancel); process.emitData("next line\n"); @@ -1185,7 +1183,7 @@ it.layer( yield* manager.close({ threadId: "thread-1" }); const reopened = yield* manager.open(openInput()); - expect(reopened.history.match(/next line/g)).toHaveLength(4); + expect(reopened.history.match(/next line/g)).toHaveLength(6); }), ); @@ -1203,10 +1201,10 @@ it.layer( process.emitData("\u001bP$q "); process.emitData("m\u001b"); process.emitData("\\after "); - process.emitData("\u009b?3"); + process.emitData("\u001b[?3"); process.emitData("1uafter "); - process.emitData("\u0090+q544e"); - process.emitData("\u009cafter\n"); + process.emitData("\u001bP+q544e"); + process.emitData("\u001b\\after\n"); yield* manager.close({ threadId: "thread-1" }); @@ -1229,10 +1227,10 @@ it.layer( yield* Effect.addFinalizer(() => Effect.sync(unsubscribe)); const sequences = [ { prefix: `\u001b]52;c;${"YWJj".repeat(16_384)}\u001b`, terminator: "\\" }, - { prefix: `\u0090q${"?".repeat(65_536)}`, terminator: "\u009c" }, + { prefix: `\u001bPq${"?".repeat(65_536)}`, terminator: "\u001b\\" }, ]; - const stripped = "\u001b]11;?\u0007\u0090$q m\u009c"; - const preserved = "\u001b[31m\u009d52;c;YWJj\u009c"; + const stripped = "\u001b]11;?\u0007\u001bP$q m\u001b\\"; + const preserved = "\u001b[31m\u001b]52;c;YWJj\u0007"; const expected = sequences.map(({ prefix, terminator }) => prefix + terminator + preserved).join("") + "done\n"; diff --git a/apps/server/src/terminal/Manager.ts b/apps/server/src/terminal/Manager.ts index 4bc86f5b12965..89f1299b35a05 100644 --- a/apps/server/src/terminal/Manager.ts +++ b/apps/server/src/terminal/Manager.ts @@ -1154,8 +1154,10 @@ function findStringTerminatorIndex(input: string, start: number): number | null if (codePoint === 0x07 || codePoint === 0x9c || isControlSequenceCancel(codePoint)) { return index + 1; } - if (codePoint === 0x1b && input.charCodeAt(index + 1) === 0x5c) { - return index + 2; + // Like xterm.js, an ESC that does not start ST ends the string and begins the next sequence. + if (codePoint === 0x1b) { + if (index + 1 >= input.length) return null; + return input.charCodeAt(index + 1) === 0x5c ? index + 2 : index; } } return null; @@ -1177,10 +1179,15 @@ function findEscapeSequenceEndIndex(input: string, start: number): number | null if (cursor >= input.length) { return null; } + // A second ESC restarts the sequence, so leave it for the next pass. + if (input.charCodeAt(cursor) === 0x1b) { + return cursor; + } return isEscapeFinalByte(input.charCodeAt(cursor)) ? cursor + 1 : start + 1; } -const HISTORY_SEQUENCE_START = /[\u001b\u0090\u009b\u009d\u009e\u009f]/; +// libghostty prints UTF-8 C1 code points (U+0080-U+009F) as text, so only ESC starts a sequence. +const HISTORY_SEQUENCE_START = /\u001b/; const NEXT_HISTORY_SEQUENCE_START = new RegExp(HISTORY_SEQUENCE_START.source, "g"); function sanitizeTerminalHistoryChunk( @@ -1265,47 +1272,6 @@ function sanitizeTerminalHistoryChunk( continue; } - if (codePoint === 0x9b) { - let cursor = index + 1; - while (cursor < input.length) { - const cancelled = isControlSequenceCancel(input.charCodeAt(cursor)); - if (cancelled || isCsiFinalByte(input.charCodeAt(cursor))) { - const sequence = input.slice(index, cursor + 1); - const body = input.slice(index + 1, cursor); - if (cancelled || !shouldStripCsiSequence(body, input[cursor] ?? "")) { - append(sequence); - } - index = cursor + 1; - break; - } - cursor += 1; - } - if (cursor >= input.length) { - return { visibleText, pendingControlSequence: input.slice(index) }; - } - continue; - } - - if (codePoint === 0x9d || codePoint === 0x90 || codePoint === 0x9e || codePoint === 0x9f) { - const terminatorIndex = findStringTerminatorIndex( - input, - Math.max(index + 1, pendingScanStart), - ); - if (terminatorIndex === null) { - return { visibleText, pendingControlSequence: input.slice(index) }; - } - const sequence = input.slice(index, terminatorIndex); - const content = stripStringTerminator(input.slice(index + 1, terminatorIndex)); - const strip = - (codePoint === 0x9d && shouldStripOscSequence(content)) || - (codePoint === 0x90 && shouldStripDcsSequence(content)); - if (!strip) { - append(sequence); - } - index = terminatorIndex; - continue; - } - // One slice per printable run: every chunk of coloured output reaches this loop. NEXT_HISTORY_SEQUENCE_START.lastIndex = index + 1; const nextSequenceStart = NEXT_HISTORY_SEQUENCE_START.exec(input)?.index ?? input.length; diff --git a/apps/server/src/textGeneration/AntigravityTextGeneration.test.ts b/apps/server/src/textGeneration/AntigravityTextGeneration.test.ts index 7e1f1446d1a55..62205d2835985 100644 --- a/apps/server/src/textGeneration/AntigravityTextGeneration.test.ts +++ b/apps/server/src/textGeneration/AntigravityTextGeneration.test.ts @@ -16,7 +16,7 @@ import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; -import { type AcpError } from "effect-acp/errors"; +import { type AcpError, AcpRequestError } from "effect-acp/errors"; import type * as AcpSchema from "effect-acp/compat"; import { expect } from "vite-plus/test"; @@ -452,6 +452,27 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { }).pipe(Effect.scoped), ); + it.effect.each([ + { + code: -32000, + detail: "Sign in to Antigravity in Settings before you continue.", + }, + { code: -32603, detail: "Antigravity text generation failed." }, + ])("maps ACP error $code from the prompt request to a fixed detail", ({ code, detail }) => + Effect.gen(function* () { + const fixture = yield* makeFixture({ + prompt: () => + Effect.fail(new AcpRequestError({ code, errorMessage: "Raw provider failure text." })), + }); + const error = yield* fixture.textGeneration + .generateThreadTitle(fixture.titleInput) + .pipe(Effect.flip); + expect(error.detail).toBe(detail); + expect(error.detail).not.toContain("Raw provider failure text."); + yield* fixture.assertCleaned; + }).pipe(Effect.scoped), + ); + it.effect.each([ { relativePath: "unexpected.txt", kind: "file" }, { relativePath: ".agents/skills/unexpected.txt", kind: "file" }, @@ -551,6 +572,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { .generateThreadTitle(fixture.titleInput) .pipe(Effect.flip); expect(error._tag).toBe("TextGenerationError"); + expect(error.detail).toBe("Sign in before starting Antigravity."); expect(fixture.state.workspaces).toEqual([]); yield* fixture.assertCleaned; }).pipe(Effect.scoped), diff --git a/apps/server/src/textGeneration/AntigravityTextGeneration.ts b/apps/server/src/textGeneration/AntigravityTextGeneration.ts index 22fce172b1836..b7e7cfaeb40e8 100644 --- a/apps/server/src/textGeneration/AntigravityTextGeneration.ts +++ b/apps/server/src/textGeneration/AntigravityTextGeneration.ts @@ -1,4 +1,4 @@ -import { type ProviderSetupError, TextGenerationError } from "@t3tools/contracts"; +import { ProviderSetupError, TextGenerationError } from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; @@ -13,12 +13,17 @@ import { type AcpError, AcpRequestError } from "effect-acp/errors"; import { applyAntigravityAcpModelSelection } from "../provider/acp/AntigravityAcpSupport.ts"; import { removeAntigravitySessionFiles } from "../provider/acp/AntigravitySessionFiles.ts"; +import { + ANTIGRAVITY_SIGN_IN_REQUIRED_MESSAGE, + isAntigravitySignInRequiredError, +} from "../provider/antigravityAuthSupport.ts"; import type * as AcpSessionRuntime from "@t3tools/provider-acp/server/AcpSessionRuntime"; import * as TextGenerationOperations from "@t3tools/provider-core/server/textGenerationOperations"; const ANTIGRAVITY_TIMEOUT_MS = 180_000; const MAX_OUTPUT_CHARS = 128_000; const isTextGenerationError = Schema.is(TextGenerationError); +const isProviderSetupError = Schema.is(ProviderSetupError); const isNativeSessionId = Schema.is(Schema.String.check(Schema.isUUID(4))); const Configuration = Schema.Record(Schema.String, Schema.Unknown); const decodeConfiguration = Schema.decodeEffect(Schema.fromJsonString(Configuration)); @@ -322,7 +327,11 @@ export const makeAntigravityTextGeneration = Effect.fn("makeAntigravityTextGener ? cause : new TextGenerationError({ operation: input.operation, - detail: "Antigravity text generation failed.", + detail: isProviderSetupError(cause) + ? cause.detail + : isAntigravitySignInRequiredError(cause) + ? ANTIGRAVITY_SIGN_IN_REQUIRED_MESSAGE + : "Antigravity text generation failed.", cause, }), ), diff --git a/apps/server/src/textGeneration/OpenCode2TextGeneration.test.ts b/apps/server/src/textGeneration/OpenCode2TextGeneration.test.ts index 2cfe556ebf58f..869cd1b4fb5e7 100644 --- a/apps/server/src/textGeneration/OpenCode2TextGeneration.test.ts +++ b/apps/server/src/textGeneration/OpenCode2TextGeneration.test.ts @@ -1,12 +1,15 @@ import { assert, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { ProviderInstanceId } from "@t3tools/contracts"; +import * as NodeURL from "node:url"; +import { ChatAttachmentId, ChatImageAttachment, ProviderInstanceId } from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import { TestClock } from "effect/testing"; +import * as ProviderHost from "@t3tools/provider-core/server/ProviderHost"; import * as TestProviderHost from "@t3tools/provider-testing/TestProviderHost"; import * as OpenCode2AdapterV2Testkit from "../orchestration-v2/Adapters/OpenCode2AdapterV2.testkit.ts"; import * as OpenCode2Server from "@t3tools/provider-opencode/server/v2/OpenCode2Server"; @@ -16,28 +19,58 @@ import { OPENCODE2_TITLE_GENERATION } from "./OpenCode2TextGeneration.fixture.ts const layer = TestProviderHost.layer().pipe(Layer.provideMerge(NodeServices.layer)); it.layer(layer)("OpenCode2TextGeneration", (it) => { - it.effect("generates a title in a temporary session on the free tier and removes it", () => - Effect.gen(function* () { - const server = yield* OpenCode2AdapterV2Testkit.replayServer({ - provider: "opencode", - protocol: OpenCode2AdapterV2Testkit.OPENCODE2_HTTP_PROTOCOL, - version: "2.0.18", - scenario: "opencode2_title_generation", - entries: [...OPENCODE2_TITLE_GENERATION, { type: "runtime_exit", status: "success" }], - }); - const textGeneration = yield* OpenCode2TextGeneration.make().pipe( - Effect.provideService(OpenCode2Server.OpenCode2Server, server), - ); - const title = yield* textGeneration.generateThreadTitle({ - cwd: process.cwd(), - message: "fix the login redirect loop after oauth", - modelSelection: { - instanceId: ProviderInstanceId.make("opencode"), - model: "opencode/big-pickle", - }, - }); - assert.equal(title.title, "Fix OAuth Login Redirect Loop"); - }).pipe(Effect.scoped), + it.effect.each([false, true] as const)( + "generates a title in a temporary session on the free tier and removes it (external: %s)", + (external) => + Effect.gen(function* () { + const host = yield* ProviderHost.ProviderHost; + const fileSystem = yield* FileSystem.FileSystem; + const image = (id: string, name: string, mimeType: string) => + ChatImageAttachment.make({ + type: "image", + id: ChatAttachmentId.make(id), + name, + mimeType, + sizeBytes: 4, + }); + const screenshot = image("oc2-title-screenshot", "screen.png", "image/png"); + const missing = image("oc2-title-missing", "missing.png", "image/png"); + const unsupported = image("oc2-title-diagram", "diagram.svg", "image/svg+xml"); + const screenshotPath = host.resolveAttachmentPath(screenshot)!; + yield* fileSystem.writeFileString(screenshotPath, "data"); + yield* fileSystem.writeFileString(host.resolveAttachmentPath(unsupported)!, ""); + const prompts: Array = []; + const server = yield* OpenCode2AdapterV2Testkit.replayServer( + { + provider: "opencode", + protocol: OpenCode2AdapterV2Testkit.OPENCODE2_HTTP_PROTOCOL, + version: "2.0.18", + scenario: "opencode2_title_generation", + entries: [...OPENCODE2_TITLE_GENERATION, { type: "runtime_exit", status: "success" }], + }, + { external, onPrompt: (input) => prompts.push(input) }, + ); + const textGeneration = yield* OpenCode2TextGeneration.make().pipe( + Effect.provideService(OpenCode2Server.OpenCode2Server, server), + ); + const title = yield* textGeneration.generateThreadTitle({ + cwd: process.cwd(), + message: "fix the login redirect loop after oauth", + attachments: [screenshot, missing, unsupported], + modelSelection: { + instanceId: ProviderInstanceId.make("opencode"), + model: "opencode/big-pickle", + }, + }); + assert.equal(title.title, "Fix OAuth Login Redirect Loop"); + if (external) { + assert.notProperty(prompts[0], "files"); + } else { + assert.deepInclude(prompts[0], { + files: [{ uri: NodeURL.pathToFileURL(screenshotPath).href, name: "screen.png" }], + }); + } + }).pipe(Effect.scoped), ); it.effect("keeps a provider's own error message out of the caller-visible detail", () => diff --git a/apps/server/src/textGeneration/ThreadTitleLinks.test.ts b/apps/server/src/textGeneration/ThreadTitleLinks.test.ts index ccb4cbfde9fce..d7a34a70884e9 100644 --- a/apps/server/src/textGeneration/ThreadTitleLinks.test.ts +++ b/apps/server/src/textGeneration/ThreadTitleLinks.test.ts @@ -20,7 +20,7 @@ it.effect( const result = yield* resolveThreadTitleLinks({ cwd: "/tmp/project", message: - "https://docs.test/guide [https://forge.test/change/1] https://forge.test/change/1#discussion https://forge.test/change/1?view=full `https://forge.test/change/2` https://forge.test/change/2. https://forge.test/change/3", + "https://docs.test/guide **https://forge.test/change/1**: [https://forge.test/change/1] https://forge.test/change/1#discussion https://forge.test/change/1?view=full `https://forge.test/change/2` https://forge.test/change/2. https://forge.test/change/3", }).pipe( Effect.provide( layerRegistry({ diff --git a/apps/server/src/textGeneration/ThreadTitleLinks.ts b/apps/server/src/textGeneration/ThreadTitleLinks.ts index 695e93e1f2d33..cb6f0d924d275 100644 --- a/apps/server/src/textGeneration/ThreadTitleLinks.ts +++ b/apps/server/src/textGeneration/ThreadTitleLinks.ts @@ -13,7 +13,7 @@ export const resolveThreadTitleLinks = Effect.fn("resolveThreadTitleLinks")(func const providers = yield* SourceControlProviderRegistry.SourceControlProviderRegistry; const links = new Map>>(); for (const match of input.message.matchAll(/https:\/\/[^\s<>"')\]`]+/g)) { - const candidate = match[0].replace(/[.,;!?]+$/, ""); + const candidate = match[0].replace(/[.,:;!?*]+$/, ""); if (!URL.canParse(candidate)) continue; const url = new URL(candidate); url.hash = ""; diff --git a/apps/server/src/usage/UsageLimitSources.test.ts b/apps/server/src/usage/UsageLimitSources.test.ts index b8976fca81e7d..7b078b7b61202 100644 --- a/apps/server/src/usage/UsageLimitSources.test.ts +++ b/apps/server/src/usage/UsageLimitSources.test.ts @@ -1,10 +1,12 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { expect, it } from "@effect/vitest"; -import { ServerConfigStreamEvent, UsageLimitSourceId } from "@t3tools/contracts"; +import { SecretProviderId, ServerConfigStreamEvent, UsageLimitSourceId } from "@t3tools/contracts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; -import { HttpClient } from "effect/http"; +import * as Stream from "effect/Stream"; +import { HttpClient, HttpClientResponse } from "effect/http"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as ServerSettings from "../serverSettings.ts"; @@ -36,6 +38,10 @@ it.effect("keeps malformed source URLs encodable for config subscribers", () => ); }).pipe( Effect.provide([ + Layer.succeed(SecretResolver.SecretResolver, { + read: (references) => Effect.succeed(references.map(() => undefined)), + invalidate: () => Effect.void, + }), ServerSettings.layerTest({ usageLimitSources: { [UsageLimitSourceId.make("local-hub")]: { @@ -83,3 +89,69 @@ it.effect("keeps malformed source URLs encodable for config subscribers", () => Effect.scoped, ), ); + +it.effect("reads a management key from its secret provider", () => { + const authorizations: Array = []; + const keySource = (reference: string) => ({ + provider: SecretProviderId.make("homelab"), + reference, + }); + return Effect.gen(function* () { + const sources = yield* UsageLimitSources.make; + const snapshots = yield* sources.streamChanges.pipe( + Stream.filter((next) => next.length > 0), + Stream.runHead, + Effect.flatMap(Effect.fromOption), + ); + expect(snapshots.map(({ id, error }) => ({ id, error }))).toEqual([ + { id: "resolved-hub", error: undefined }, + { + id: "unresolved-hub", + error: "Could not read the management key from its secret provider.", + }, + ]); + expect(authorizations).toEqual(["Bearer resolved-key"]); + }).pipe( + Effect.provide([ + ServerSettings.layerTest({ + usageLimitSources: { + [UsageLimitSourceId.make("resolved-hub")]: { + kind: "cliproxy", + url: "https://resolved.test", + managementKey: keySource("cliproxy#key"), + enabled: true, + }, + [UsageLimitSourceId.make("unresolved-hub")]: { + kind: "cliproxy", + url: "https://unresolved.test", + managementKey: keySource("missing#key"), + enabled: true, + }, + }, + }), + Layer.mock(BackgroundPolicy.BackgroundPolicy)({ + shouldRunScopeWork: () => Effect.succeed(false), + }), + Layer.succeed(SecretResolver.SecretResolver, { + read: (references) => + Effect.succeed( + references.map((reference) => + reference.reference === "cliproxy#key" ? "resolved-key" : undefined, + ), + ), + invalidate: () => Effect.void, + }), + Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.sync(() => { + authorizations.push(request.headers.authorization); + return HttpClientResponse.fromWeb(request, Response.json({ files: [] })); + }), + ), + ), + NodeCrypto.layer, + ]), + Effect.scoped, + ); +}); diff --git a/apps/server/src/usage/UsageLimitSources.ts b/apps/server/src/usage/UsageLimitSources.ts index 7a068cde85f6c..7d40b0fb9ecea 100644 --- a/apps/server/src/usage/UsageLimitSources.ts +++ b/apps/server/src/usage/UsageLimitSources.ts @@ -4,7 +4,8 @@ import { type UsageLimitSourceConsumeResetCreditInput, type ProviderConsumeResetCreditResult, type ServerSettings, - type UsageLimitSourceConfig, + UsageLimitSourceConfig, + settingsSecretLiteral, type UsageLimitSourceId, type UsageLimitSourceSnapshot, } from "@t3tools/contracts"; @@ -21,6 +22,9 @@ import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import * as SecretResolverLayer from "../secrets/SecretResolver.ts"; +import { resolveSettingsSecrets } from "../secrets/settingsSecretValue.ts"; import * as Settings from "../serverSettings.ts"; import { makeCliproxyApi } from "./cliproxyApi.ts"; @@ -46,11 +50,24 @@ function sourceLabel(id: string, config: UsageLimitSourceConfig): string { } } +function missingManagementKey(config: UsageLimitSourceConfig): string { + return typeof config.managementKey === "string" + ? "No management key configured." + : "Could not read the management key from its secret provider."; +} + /** @public */ export const make = Effect.gen(function* () { const api = yield* makeCliproxyApi; const settingsService = yield* Settings.ServerSettingsService; + const secrets = yield* SecretResolver.SecretResolver; const backgroundPolicy = yield* BackgroundPolicy.BackgroundPolicy; + const readManagementKey = Effect.fn("UsageLimitSources.readManagementKey")(function* ( + config: UsageLimitSourceConfig, + ) { + const resolved = yield* resolveSettingsSecrets(secrets, UsageLimitSourceConfig, config); + return settingsSecretLiteral(resolved.managementKey); + }); const stateRef = yield* Ref.make>([]); const changes = yield* Effect.acquireRelease( PubSub.unbounded>(), @@ -63,10 +80,13 @@ export const make = Effect.gen(function* () { ) { const checkedAt = DateTime.formatIso(yield* DateTime.now); const base = { id, kind: config.kind, label: sourceLabel(id, config), checkedAt } as const; - if (config.managementKey.length === 0) { - return { ...base, accounts: [], error: "No management key configured." }; + const managementKey = yield* readManagementKey(config); + if (managementKey.length === 0) { + return { ...base, accounts: [], error: missingManagementKey(config) }; } - const accounts = yield* api.readAccounts(config).pipe(Effect.result); + const accounts = yield* api + .readAccounts({ url: config.url, managementKey }) + .pipe(Effect.result); if (accounts._tag === "Failure") { yield* Effect.logDebug("usage limit source read failed", { id, cause: accounts.failure }); return { ...base, accounts: [], error: accounts.failure.detail }; @@ -108,12 +128,20 @@ export const make = Effect.gen(function* () { Effect.mapError(() => new UsageLimitSourceError({ detail: "Could not read hub settings." })), ); const config = settings.usageLimitSources[input.sourceId]; - if (!config?.enabled || !config.managementKey) { + if (!config?.enabled) { return yield* new UsageLimitSourceError({ detail: "The usage limit source is missing or disabled.", }); } - const result = yield* api.consume(config, input.accountId, input.creditId); + const managementKey = yield* readManagementKey(config); + if (managementKey.length === 0) { + return yield* new UsageLimitSourceError({ detail: missingManagementKey(config) }); + } + const result = yield* api.consume( + { url: config.url, managementKey }, + input.accountId, + input.creditId, + ); const snapshot = yield* readSource(input.sourceId, config); const previous = yield* Ref.get(stateRef); yield* publish(previous.map((source) => (source.id === input.sourceId ? snapshot : source))); @@ -167,4 +195,6 @@ export const make = Effect.gen(function* () { } satisfies UsageLimitSources["Service"]; }); -export const layer = Layer.effect(UsageLimitSources, make); +export const layer = Layer.effect(UsageLimitSources, make).pipe( + Layer.provide(SecretResolverLayer.layer), +); diff --git a/apps/server/src/usage/cliproxyApi.ts b/apps/server/src/usage/cliproxyApi.ts index 56aab970a1639..d277209b290d4 100644 --- a/apps/server/src/usage/cliproxyApi.ts +++ b/apps/server/src/usage/cliproxyApi.ts @@ -25,6 +25,12 @@ import { makeUsageLimits, } from "@t3tools/provider-core/server/usageLimits"; +/** A hub with its management key already read from wherever the setting keeps it. */ +export interface CliproxyHub { + readonly url: UsageLimitSourceConfig["url"]; + readonly managementKey: string; +} + const AuthFile = Schema.Struct({ id: Schema.String, auth_index: Schema.String, @@ -589,7 +595,7 @@ export const makeCliproxyApi = Effect.gen(function* () { const digest = yield* Digest.Digest; const management = Effect.fn("CliproxyApi.management")(function* ( - config: UsageLimitSourceConfig, + hub: CliproxyHub, path: string, body?: unknown, ) { @@ -597,13 +603,13 @@ export const makeCliproxyApi = Effect.gen(function* () { try: () => new URL( `v0/management/${path}`, - config.url.endsWith("/") ? config.url : `${config.url}/`, + hub.url.endsWith("/") ? hub.url : `${hub.url}/`, ).toString(), catch: () => new UsageLimitSourceError({ detail: "The hub URL is not valid." }), }); const request = ( body === undefined ? HttpClientRequest.get(url) : HttpClientRequest.post(url) - ).pipe(HttpClientRequest.setHeader("Authorization", `Bearer ${config.managementKey}`)); + ).pipe(HttpClientRequest.setHeader("Authorization", `Bearer ${hub.managementKey}`)); const response = yield* client .execute(body === undefined ? request : request.pipe(HttpClientRequest.bodyJsonUnsafe(body))) .pipe( @@ -617,18 +623,18 @@ export const makeCliproxyApi = Effect.gen(function* () { return response; }); - const authFiles = Effect.fn("CliproxyApi.authFiles")(function* (config: UsageLimitSourceConfig) { - const response = yield* management(config, "auth-files"); + const authFiles = Effect.fn("CliproxyApi.authFiles")(function* (hub: CliproxyHub) { + const response = yield* management(hub, "auth-files"); return (yield* decodeAuthFiles(response)).files; }); const apiCall = Effect.fn("CliproxyApi.apiCall")(function* ( - config: UsageLimitSourceConfig, + hub: CliproxyHub, account: typeof AuthFile.Type, url: string, options: { readonly header: Record; readonly data?: unknown }, ) { - const raw = yield* management(config, "api-call", { + const raw = yield* management(hub, "api-call", { auth_index: account.auth_index, method: options.data === undefined ? "GET" : "POST", url, @@ -645,10 +651,10 @@ export const makeCliproxyApi = Effect.gen(function* () { }); const credits = Effect.fn("CliproxyApi.credits")(function* ( - config: UsageLimitSourceConfig, + hub: CliproxyHub, account: typeof AuthFile.Type, ) { - const body = yield* apiCall(config, account, CREDIT_URL, { header: codexHeaders(account) }); + const body = yield* apiCall(hub, account, CREDIT_URL, { header: codexHeaders(account) }); const response = yield* decodeCreditList(body); const now = DateTime.toEpochMillis(yield* DateTime.now); return response.credits @@ -662,7 +668,7 @@ export const makeCliproxyApi = Effect.gen(function* () { }); const readAntigravityLimits = Effect.fn("CliproxyApi.readAntigravityLimits")(function* ( - config: UsageLimitSourceConfig, + hub: CliproxyHub, account: typeof AuthFile.Type, checkedAt: string, ) { @@ -670,7 +676,7 @@ export const makeCliproxyApi = Effect.gen(function* () { if (!project) { const discovered = yield* Effect.gen(function* () { const body = yield* apiCall( - config, + hub, account, `${ANTIGRAVITY_HOSTS[0]}/v1internal:loadCodeAssist`, { @@ -692,7 +698,7 @@ export const makeCliproxyApi = Effect.gen(function* () { return yield* new UsageLimitSourceError({ detail: "Antigravity account has no project." }); } for (const host of ANTIGRAVITY_HOSTS) { - const body = yield* apiCall(config, account, `${host}/v1internal:retrieveUserQuotaSummary`, { + const body = yield* apiCall(hub, account, `${host}/v1internal:retrieveUserQuotaSummary`, { header: ANTIGRAVITY_HEADERS, data: { project }, }).pipe(Effect.result); @@ -709,7 +715,7 @@ export const makeCliproxyApi = Effect.gen(function* () { }); const readAccount = Effect.fn("CliproxyApi.readAccount")(function* ( - config: UsageLimitSourceConfig, + hub: CliproxyHub, account: typeof AuthFile.Type, provider: HubProvider, ) { @@ -725,14 +731,9 @@ export const makeCliproxyApi = Effect.gen(function* () { const read = Effect.gen(function* () { switch (provider) { case "claude": { - const body = yield* apiCall( - config, - account, - "https://api.anthropic.com/api/oauth/usage", - { - header: CLAUDE_HEADERS, - }, - ); + const body = yield* apiCall(hub, account, "https://api.anthropic.com/api/oauth/usage", { + header: CLAUDE_HEADERS, + }); const usage = yield* decodeClaudeUsage(body); const model_scoped = (usage.limits ?? []).flatMap((limit) => limit.kind === "weekly_scoped" && @@ -767,7 +768,7 @@ export const makeCliproxyApi = Effect.gen(function* () { }; } case "grok": { - const body = yield* apiCall(config, account, GROK_USAGE_URL, { header: GROK_HEADERS }); + const body = yield* apiCall(hub, account, GROK_USAGE_URL, { header: GROK_HEADERS }); return { ...base, usageLimits: grokUsageResponseToLimits(yield* decodeGrokUsage(body), checkedAt), @@ -776,10 +777,10 @@ export const makeCliproxyApi = Effect.gen(function* () { case "antigravity": return { ...base, - usageLimits: yield* readAntigravityLimits(config, account, checkedAt), + usageLimits: yield* readAntigravityLimits(hub, account, checkedAt), }; case "devin": { - const body = yield* apiCall(config, account, DEVIN_QUOTA_URL, { + const body = yield* apiCall(hub, account, DEVIN_QUOTA_URL, { header: DEVIN_HEADERS, data: DEVIN_QUOTA_BODY, }); @@ -791,7 +792,7 @@ export const makeCliproxyApi = Effect.gen(function* () { }; } case "kimi": { - const body = yield* apiCall(config, account, KIMI_USAGE_URL, { header: KIMI_HEADERS }); + const body = yield* apiCall(hub, account, KIMI_USAGE_URL, { header: KIMI_HEADERS }); return { ...base, usageLimits: makeUsageLimits({ @@ -801,7 +802,7 @@ export const makeCliproxyApi = Effect.gen(function* () { }; } case "codex": { - const body = yield* apiCall(config, account, `${CODEX_BASE}/usage`, { + const body = yield* apiCall(hub, account, `${CODEX_BASE}/usage`, { header: codexHeaders(account), }); const usage = yield* decodeCodexUsage(body); @@ -816,7 +817,7 @@ export const makeCliproxyApi = Effect.gen(function* () { } : null; // A credits outage must not hide successfully fetched quota windows. - const available = yield* credits(config, account).pipe( + const available = yield* credits(hub, account).pipe( Effect.orElseSucceed(() => undefined), ); const next = available?.[0]; @@ -869,9 +870,9 @@ export const makeCliproxyApi = Effect.gen(function* () { }); const readAccounts = Effect.fn("CliproxyApi.readAccounts")(function* ( - config: UsageLimitSourceConfig, + hub: CliproxyHub, ): Effect.fn.Return, UsageLimitSourceError> { - const accounts = yield* authFiles(config).pipe( + const accounts = yield* authFiles(hub).pipe( Effect.mapError((error) => isUsageLimitSourceError(error) ? error @@ -884,24 +885,24 @@ export const makeCliproxyApi = Effect.gen(function* () { const provider = hubProvider(account.provider); return provider ? [{ account, provider }] : []; }), - ({ account, provider }) => readAccount(config, account, provider), + ({ account, provider }) => readAccount(hub, account, provider), { concurrency: 4 }, ); }); const consume = Effect.fn("CliproxyApi.consume")(function* ( - config: UsageLimitSourceConfig, + hub: CliproxyHub, accountId: string, creditId: string, ): Effect.fn.Return { const operation = Effect.gen(function* () { - const account = (yield* authFiles(config)).find((account) => account.id === accountId); + const account = (yield* authFiles(hub)).find((account) => account.id === accountId); if (!account || account.disabled || account.provider !== "codex") { return yield* new UsageLimitSourceError({ detail: "The Codex hub account is missing or disabled.", }); } - const body = yield* apiCall(config, account, `${CREDIT_URL}/consume`, { + const body = yield* apiCall(hub, account, `${CREDIT_URL}/consume`, { header: codexHeaders(account), data: { redeem_request_id: yield* creditRedeemRequestId( @@ -921,7 +922,7 @@ export const makeCliproxyApi = Effect.gen(function* () { } as const )[response.code]; if (outcome !== "reset" && outcome !== "alreadyRedeemed") return { outcome }; - const cleared = yield* management(config, "reset-quota", { + const cleared = yield* management(hub, "reset-quota", { auth_index: account.auth_index, }).pipe(Effect.result); return { diff --git a/apps/server/src/vcs/GitVcsDriver.test.ts b/apps/server/src/vcs/GitVcsDriver.test.ts index 9262688fd10b5..fed1c484d0d16 100644 --- a/apps/server/src/vcs/GitVcsDriver.test.ts +++ b/apps/server/src/vcs/GitVcsDriver.test.ts @@ -323,6 +323,26 @@ it.effect("checkpoint capture still fails when a clean filter rejects a file", ( }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); +it.effect("checkpoint capture keeps LF files that core.safecrlf would refuse to commit", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-safecrlf-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* fileSystem.writeFileString(path.join(cwd, ".gitattributes"), "*.txt text eol=crlf\n"); + yield* git(["config", "core.safecrlf", "true"]); + yield* fileSystem.writeFileString(path.join(cwd, "untracked.txt"), "new\n"); + const originalIndex = yield* fileSystem.readFile(path.join(cwd, ".git", "index")); + + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + + assert.strictEqual((yield* git(["show", `${checkpointRef}:file.txt`])).stdout, "unstaged\n"); + assert.strictEqual((yield* git(["show", `${checkpointRef}:untracked.txt`])).stdout, "new\n"); + assert.deepEqual(yield* fileSystem.readFile(path.join(cwd, ".git", "index")), originalIndex); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + it.effect("checkpoint capture refuses a truncated nested repository listing", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -1134,6 +1154,47 @@ it.effect("restores empty checkpoints without changing paths outside the workspa }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); +it.effect("restores a checkpoint before the first commit without staging its files", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + for (const nested of [false, true]) { + const root = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-unborn-restore-" }); + yield* runGit(root, ["init"]); + yield* runGit(root, ["config", "user.email", "test@test.com"]); + yield* runGit(root, ["config", "user.name", "Test"]); + if (nested) { + yield* fileSystem.writeFileString(path.join(root, "outside.txt"), "staged\n"); + yield* runGit(root, ["add", "outside.txt"]); + } + const cwd = nested ? path.join(root, "nested") : root; + yield* fileSystem.makeDirectory(cwd, { recursive: true }); + yield* fileSystem.writeFileString(path.join(cwd, "file.txt"), "original\n"); + const checkpointRef = CheckpointRef.make("refs/t3/checkpoints/unborn"); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fileSystem.writeFileString(path.join(cwd, "file.txt"), "changed\n"); + yield* fileSystem.writeFileString(path.join(cwd, "added.txt"), "new\n"); + + assert.isTrue( + yield* driver.checkpoints.restoreCheckpoint({ cwd, checkpointRef, fallbackToHead: false }), + ); + + assert.strictEqual( + yield* fileSystem.readFileString(path.join(cwd, "file.txt")), + "original\n", + ); + assert.isFalse(yield* fileSystem.exists(path.join(cwd, "added.txt"))); + const staged = yield* driver.execute({ + operation: "test", + cwd: root, + args: ["diff", "--cached", "--name-only"], + }); + assert.strictEqual(staged.stdout.trim(), nested ? "outside.txt" : ""); + } + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + // The real VCS process, spawning `command` wherever the driver runs git. const vcsProcessSpawning = (command: string) => Layer.effect( diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index 15392cb96ce5e..48f1f56b9b06e 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -1168,6 +1168,9 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( ...indexConfig, ...durableWrite, ...longPaths, + // safecrlf=true refuses LF files under CRLF conversion; capture them as Git's default does. + "-c", + "core.safecrlf=false", "add", ...(sparseCheckout ? ["--sparse"] : []), // add -A rejects ignored directories even when they contain tracked files. @@ -1367,14 +1370,11 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( // On Windows the same clean can remove './' and exit 0. yield* recreateWorkspace; - const headExists = yield* hasHeadCommit(input.cwd); - if (headExists) { - yield* execute({ - operation, - cwd: input.cwd, - args: [...longPaths, "reset", "--quiet", "--", "."], - }); - } + yield* execute({ + operation, + cwd: input.cwd, + args: [...longPaths, "reset", "--quiet", "--", "."], + }); return true; }), @@ -1428,6 +1428,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( "--no-color", "--no-ext-diff", "--no-textconv", + "--submodule=short", ...PATCH_RENDER_PREFIX_ARGS, ...(input.ignoreWhitespace ? ["--ignore-all-space"] : []), `${fromRevision}^{commit}`, diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index 0c45bf6ea42b2..e46d2b4666d6d 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -1516,6 +1516,36 @@ it.effect("retries a fetch that lost a ref update to another Git process", () => ), ); +it.effect("does not fetch for status when the branch tracks a local branch", () => + Effect.scoped( + Effect.gen(function* () { + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const fetches: Array> = []; + const spawner = ChildProcessSpawner.make((command) => { + if (ChildProcess.isStandardCommand(command) && command.args.includes("fetch")) { + fetches.push(command.args.map(String)); + } + return delegate.spawn(command); + }); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-vcs-driver-remote-"); + yield* initRepoWithCommit(cwd); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "feature", remote]); + yield* git(cwd, ["branch", "feature/base"]); + yield* git(cwd, ["checkout", "-b", "stacked", "--track", "feature/base"]); + + const status = yield* driver.statusDetailsRemote(cwd); + + assert.equal(status.upstreamRef, "feature/base"); + assert.deepStrictEqual(fetches, []); + }), + ).pipe(Effect.provide(layerTest)), +); + it.effect("lets a worktree checkout that is still working run past five minutes", () => Effect.gen(function* () { const checkoutStarted = yield* Deferred.make(); @@ -2280,6 +2310,9 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { yield* writeTextFile(cwd, "untracked.txt", "new\n"); const driver = yield* GitVcsDriver.GitVcsDriver; const nestedCwd = path.join(cwd, "nested"); + const clone = path.join(nestedCwd, "clone"); + yield* fileSystem.makeDirectory(clone); + yield* initRepoWithCommit(clone); const projectDirectories = [nestedCwd]; if (symlinksSupported) { const alias = path.join(directory, "alias"); @@ -2293,7 +2326,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }); assert.equal( preview.sources.find((source) => source.kind === "working-tree")!.files!.length, - 2, + 3, ); for (const source of preview.sources) { for (const file of source.files ?? []) { @@ -2459,6 +2492,27 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("reviews untracked files outside the sparse-checkout cone", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* writeTextFile(cwd, "excluded/tracked.ts", "tracked\n"); + yield* git(cwd, ["add", "excluded/tracked.ts"]); + yield* git(cwd, ["commit", "-m", "add excluded directory"]); + yield* git(cwd, ["sparse-checkout", "set", "--cone", "included"]); + yield* writeTextFile(cwd, "excluded/new.ts", "outside the cone\n"); + + const preview = yield* driver.getReviewDiffPreview({ cwd, ignoreWhitespace: false }); + + const source = preview.sources.find((candidate) => candidate.kind === "working-tree")!; + assert.deepStrictEqual(source.files, [ + { path: "excluded/new.ts", previousPath: null, additions: 1, deletions: 0 }, + ]); + assert.include(source.diff, "+outside the cone"); + }), + ); + it.effect("detects an unstaged rename with edits without mutating a split index", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -2515,33 +2569,40 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); - it.effect("keeps tracked changes visible when untracked discovery fails", () => - Effect.gen(function* () { - const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; - const failingLsFilesSpawner = ChildProcessSpawner.make((command) => { - if (!ChildProcess.isStandardCommand(command)) { - return Effect.die("expected a standard Git command"); - } - return command.args[0] === "ls-files" && command.args[1] === "--others" - ? Effect.succeed(makeNonRepositoryHandle()) - : delegate.spawn(command); - }); - const driver = yield* makeGitVcsDriverCore().pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, failingLsFilesSpawner), - Effect.provide(layerServerConfig), - ); - const cwd = yield* makeTmpDir(); - yield* initRepoWithCommit(cwd).pipe( - Effect.provideService(GitVcsDriver.GitVcsDriver, driver), - ); - yield* writeTextFile(cwd, "README.md", "# tracked change\n"); + it.effect.each(["fails", "exceeds the output limit"] as const)( + "keeps tracked changes visible when untracked discovery %s", + (scenario) => + Effect.gen(function* () { + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const failingLsFilesSpawner = ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command)) { + return Effect.die("expected a standard Git command"); + } + if (command.args[0] !== "ls-files" || command.args[1] !== "--others") { + return delegate.spawn(command); + } + return Effect.succeed( + scenario === "fails" + ? makeNonRepositoryHandle() + : makeSuccessfulHandle("a".repeat(16 * 1024 * 1024 + 1)), + ); + }); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, failingLsFilesSpawner), + Effect.provide(layerServerConfig), + ); + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd).pipe( + Effect.provideService(GitVcsDriver.GitVcsDriver, driver), + ); + yield* writeTextFile(cwd, "README.md", "# tracked change\n"); - const preview = yield* driver.getReviewDiffPreview({ cwd, ignoreWhitespace: false }); - const diff = preview.sources.find((source) => source.kind === "working-tree")?.diff ?? ""; + const preview = yield* driver.getReviewDiffPreview({ cwd, ignoreWhitespace: false }); + const diff = preview.sources.find((source) => source.kind === "working-tree")?.diff ?? ""; - assert.include(diff, "-# test"); - assert.include(diff, "+# tracked change"); - }), + assert.include(diff, "-# test"); + assert.include(diff, "+# tracked change"); + }), ); it.effect.each([ @@ -2853,24 +2914,46 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); - it.effect("reads range patches from source without textconv or color", () => + it.effect("reads raw range context regardless of Git display configuration", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); const { initialBranch } = yield* initRepoWithCommit(cwd); yield* git(cwd, ["config", "diff.probe.textconv", "touch textconv-ran; echo converted"]); yield* git(cwd, ["config", "color.ui", "always"]); + yield* git(cwd, ["config", "diff.relative", "true"]); + yield* git(cwd, ["config", "log.decorate", "short"]); yield* git(cwd, ["checkout", "-b", "feature/range"]); yield* writeTextFile(cwd, ".gitattributes", "*.doc diff=probe\n"); yield* writeTextFile(cwd, "notes.doc", "source line\n"); yield* writeTextFile(cwd, "plain.txt", "plain line\n"); yield* writeTextFile(cwd, "binary.dat", "binary\0data"); yield* git(cwd, ["add", "."]); - yield* git(cwd, ["commit", "-m", "range files"]); + const signingKey = `${yield* makeTmpDir("git-vcs-driver-signing-key-")}/key`; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const keygen = ChildProcess.make("ssh-keygen", [ + "-q", + "-t", + "ed25519", + "-N", + "", + "-f", + signingKey, + ]); + assert.equal(Number(yield* spawner.exitCode(keygen)), 0); + yield* git(cwd, ["config", "gpg.format", "ssh"]); + // A global signer such as 1Password's cannot use the fixture's generated key. + yield* git(cwd, ["config", "gpg.ssh.program", "ssh-keygen"]); + yield* git(cwd, ["config", "user.signingkey", `${signingKey}.pub`]); + yield* git(cwd, ["config", "log.showSignature", "true"]); + yield* git(cwd, ["commit", "-S", "-m", "range files"]); const driver = yield* GitVcsDriver.GitVcsDriver; const fileSystem = yield* FileSystem.FileSystem; + const projectCwd = `${cwd}/app`; + yield* fileSystem.makeDirectory(projectCwd); - const context = yield* driver.readRangeContext(cwd, initialBranch); + const context = yield* driver.readRangeContext(projectCwd, initialBranch); + assert.match(context.commitSummary.trim(), /^[a-f0-9]+ range files$/); assert.include(context.diffPatch, "+source line"); assert.notInclude(context.diffPatch, "converted"); assert.include(context.diffPatch, "+plain line"); @@ -2882,7 +2965,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.isFalse(yield* fileSystem.exists(`${cwd}/textconv-ran`)); yield* writeTextFile(cwd, "notes.doc", "updated source line\n"); - const staged = yield* driver.prepareCommitContext(cwd, ["notes.doc"]); + const staged = yield* driver.prepareCommitContext(projectCwd, ["notes.doc"]); assert.include(staged?.stagedPatch ?? "", "+updated source line"); assert.notInclude(staged?.stagedPatch ?? "", "converted"); assert.notInclude(staged?.stagedPatch ?? "", "\u001b["); diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 4ace91f8017d2..5e5aaa0fddce2 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -81,7 +81,7 @@ const REVIEW_BRANCH_SUBJECT_LIMIT = 300; const REVIEW_BRANCH_COMMIT_MAX_OUTPUT_BYTES = 128 * 1024; // Client patch parsers require a/ and b/ prefixes; override user diff prefix settings. export const PATCH_RENDER_PREFIX_ARGS = ["--src-prefix=a/", "--dst-prefix=b/"] as const; -// Client patch parsers drop blank context lines that diff.suppressBlankEmpty prints without a space. +// Client patch parsers drop blank context lines under diff.suppressBlankEmpty and summaries under diff.submodule. const REVIEW_DIFF_ARGS = [ "-c", "diff.suppressBlankEmpty=false", @@ -90,6 +90,7 @@ const REVIEW_DIFF_ARGS = [ "--no-color", "--no-ext-diff", "--no-textconv", + "--submodule=short", ...PATCH_RENDER_PREFIX_ARGS, ]; const STATUS_UPSTREAM_REFRESH_INTERVAL = Duration.seconds(15); @@ -1652,15 +1653,25 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const refreshStatusUpstreamIfStale = Effect.fn("refreshStatusUpstreamIfStale")(function* ( cwd: string, ) { - const upstream = yield* resolveCurrentUpstream(cwd); - if (!upstream) return; + const headRef = yield* runGitStdout( + "GitVcsDriver.refreshStatusUpstreamIfStale.headRef", + cwd, + ["symbolic-ref", "--quiet", "HEAD"], + true, + ).pipe(Effect.map((stdout) => stdout.trim())); + if (headRef.length === 0) return; + const remoteName = yield* runGitStdout( + "GitVcsDriver.refreshStatusUpstreamIfStale.upstreamRemote", + cwd, + ["for-each-ref", "--format=%(upstream:remotename)", headRef], + true, + ).pipe(Effect.map((stdout) => stdout.trim())); + // Empty means no upstream; `.` means the upstream is a local branch. + if (remoteName.length === 0 || remoteName === ".") return; const gitCommonDir = yield* resolveGitCommonDir(cwd); yield* Cache.get( statusRemoteRefreshCache, - new StatusRemoteRefreshCacheKey({ - gitCommonDir, - remoteName: upstream.remoteName, - }), + new StatusRemoteRefreshCacheKey({ gitCommonDir, remoteName }), ).pipe(Effect.ignore); }); @@ -2525,7 +2536,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const stagedSummary = yield* runGitStdoutWithOptions( "GitVcsDriver.prepareCommitContext.stagedSummary", cwd, - ["diff", "--cached", "--name-status"], + // cwd can be a subdirectory; diff.relative would hide changes outside it. + ["-c", "diff.relative=false", "diff", "--cached", "--name-status"], { env, maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, ).pipe(Effect.map((stdout) => stdout.trim())); if (stagedSummary.length === 0) { @@ -2539,6 +2551,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* "GitVcsDriver.prepareCommitContext.stagedPatch", cwd, [ + "-c", + "diff.relative=false", "diff", "--no-color", "--no-ext-diff", @@ -2939,7 +2953,15 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* runGitStdoutWithOptions( "GitVcsDriver.readRangeContext.log", cwd, - ["log", "--oneline", "--no-color", "--end-of-options", commitRange], + [ + "log", + "--oneline", + "--no-color", + "--no-decorate", + "--no-show-signature", + "--end-of-options", + commitRange, + ], { maxOutputBytes: RANGE_COMMIT_SUMMARY_MAX_OUTPUT_BYTES, appendTruncationMarker: true, @@ -2948,7 +2970,15 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* runGitStdoutWithOptions( "GitVcsDriver.readRangeContext.diffStat", cwd, - ["diff", "--stat", "--no-color", "--end-of-options", diffRange], + [ + "-c", + "diff.relative=false", + "diff", + "--stat", + "--no-color", + "--end-of-options", + diffRange, + ], { maxOutputBytes: RANGE_DIFF_SUMMARY_MAX_OUTPUT_BYTES, appendTruncationMarker: true, @@ -2958,6 +2988,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* "GitVcsDriver.readRangeContext.diffPatch", cwd, [ + "-c", + "diff.relative=false", "diff", "--no-color", "--no-ext-diff", @@ -3056,6 +3088,9 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* // Avoid redundant tracked-entry pathspec matching in this disposable index. "-c", "core.preloadIndex=false", + // Sparse checkout refuses to add untracked files outside the cone, failing the whole review. + "-c", + "core.sparseCheckout=false", "--literal-pathspecs", "add", "--intent-to-add", @@ -3087,15 +3122,12 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* cwd, ["ls-files", "--others", "--exclude-standard", "-z", "--", ...pathArgs], { maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, - ).pipe( - Effect.catchIf( - (error) => error.outputLength === undefined, - () => Effect.succeed(null), - ), - ); + ).pipe(Effect.orElseSucceed(() => null)); if (untracked === null) return null; + // ls-files lists a nested repository with a trailing slash; its diff path has none. const paths = splitNullSeparatedGitStdoutPaths(untracked).filter( - (candidate) => onlyPath === undefined || candidate === onlyPath, + (candidate) => + onlyPath === undefined || candidate === onlyPath || candidate === `${onlyPath}/`, ); if (paths.length === 0) return { env: undefined }; if (paths.length > REVIEW_UNTRACKED_MAX_FILES) return null; diff --git a/apps/server/src/workspace/WorkspaceEntries.test.ts b/apps/server/src/workspace/WorkspaceEntries.test.ts index 154a260eea0f4..eedff4fc79c9a 100644 --- a/apps/server/src/workspace/WorkspaceEntries.test.ts +++ b/apps/server/src/workspace/WorkspaceEntries.test.ts @@ -108,6 +108,14 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { yield* writeTextFile(cwd, "node_modules/pkg/index.js"); yield* writeTextFile(cwd, "src/index.ts"); yield* fileSystem.makeDirectory(path.join(cwd, "empty")); + yield* git(cwd, ["init", "sub"]); + yield* writeTextFile(cwd, "sub/.gitignore", "dist/\n"); + yield* writeTextFile(cwd, "sub/dist/out.js"); + const gitlink = yield* git(cwd, ["hash-object", "tracked.txt"]); + yield* git(cwd, ["update-index", "--add", "--cacheinfo", `160000,${gitlink},sub`]); + // A nested clone that isn't a submodule keeps the outer repository's ignore rules. + yield* git(cwd, ["init", "node_modules/pkg"]); + yield* git(cwd, ["init", "sub/dist"]); const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; const root = yield* workspaceEntries.list({ cwd, directoryPath: "" }); @@ -127,6 +135,17 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { entries: [{ path: "node_modules/pkg/index.js", kind: "file", ignored: true }], truncated: false, }); + // Git refuses pathspecs inside a submodule from the outer repository. + expect((yield* workspaceEntries.list({ cwd, directoryPath: "sub" })).entries).toEqual( + expect.arrayContaining([ + { path: "sub/.gitignore", kind: "file" }, + { path: "sub/dist", kind: "directory", ignored: true }, + ]), + ); + expect(yield* workspaceEntries.list({ cwd, directoryPath: "sub/dist" })).toEqual({ + entries: [{ path: "sub/dist/out.js", kind: "file", ignored: true }], + truncated: false, + }); expect(yield* workspaceEntries.list({ cwd, directoryPath: "empty" })).toEqual({ entries: [], truncated: false, diff --git a/apps/server/src/workspace/WorkspaceEntries.ts b/apps/server/src/workspace/WorkspaceEntries.ts index 0b853ccdaa437..d388170f93f5e 100644 --- a/apps/server/src/workspace/WorkspaceEntries.ts +++ b/apps/server/src/workspace/WorkspaceEntries.ts @@ -306,7 +306,7 @@ export const make = Effect.gen(function* () { relativePath: directoryPath, }) .pipe(Effect.mapError(toError)); - const { entries, throughSymlink } = yield* Effect.tryPromise({ + const { entries, throughSymlink, nestedGitRoots } = yield* Effect.tryPromise({ try: async () => { const root = await NodeFSP.realpath(normalizedCwd); const directory = await NodeFSP.realpath(target.absolutePath); @@ -378,7 +378,17 @@ export const make = Effect.gen(function* () { )), ); } + const nestedGitRoots: Array = []; + const segments = target.relativePath.split("/").filter(Boolean); + for (let index = 1; index <= segments.length; index += 1) { + const candidate = segments.slice(0, index).join("/"); + const dotGit = await NodeFSP.lstat(path.join(normalizedCwd, candidate, ".git")).catch( + () => null, + ); + if (dotGit) nestedGitRoots.push(candidate); + } return { + nestedGitRoots, entries: children.flatMap((child, index): ProjectEntry[] => { const kind = kinds[index]; if (kind === undefined) return []; @@ -395,6 +405,8 @@ export const make = Effect.gen(function* () { }, catch: toError, }); + let gitRoot = ""; + let gitPrefix = ""; // stdin avoids argument limits; ignore classification is optional without git/repositories. const checkIgnore = (paths: ReadonlyArray) => vcsProcess @@ -402,20 +414,27 @@ export const make = Effect.gen(function* () { operation: "WorkspaceEntries.list", command: "git", args: ["-c", "core.fsmonitor=false", "check-ignore", "-z", "--stdin"], - cwd: normalizedCwd, - stdin: `${paths.join("\0")}\0`, + cwd: path.join(normalizedCwd, gitRoot), + stdin: `${paths.map((entryPath) => entryPath.slice(gitPrefix.length)).join("\0")}\0`, allowNonZeroExit: true, timeoutMs: 10_000, maxOutputBytes: 16 * 1024 * 1024, }) .pipe(Effect.orElseSucceed(() => undefined)); + // Git refuses paths inside a submodule (128) but still answers for nested clones. + for (const nestedGitRoot of nestedGitRoots) { + if ((yield* checkIgnore([`${nestedGitRoot}/.git`]))?.exitCode !== 128) continue; + gitRoot = nestedGitRoot; + gitPrefix = `${gitRoot}/`; + } + const directoryGitPath = target.relativePath.slice(gitPrefix.length); // Check ignored folders once to avoid child index lookups; exit 0 guarantees no tracked descendants. // Symlinks and pathspec magic still need per-child checks. if ( entries.length > IGNORE_CHECK_CHUNK_SIZE && - target.relativePath !== "" && + directoryGitPath !== "" && !throughSymlink && - !/^:|[*?[\\]/.test(target.relativePath) && + !/^:|[*?[\\]/.test(directoryGitPath) && (yield* checkIgnore([target.relativePath]))?.exitCode === 0 ) { return { @@ -429,7 +448,7 @@ export const make = Effect.gen(function* () { entries.slice(offset, offset + IGNORE_CHECK_CHUNK_SIZE).map((entry) => entry.path), ); if (!result || (result.exitCode !== 0 && result.exitCode !== 1)) break; - for (const ignoredPath of result.stdout.split("\0")) ignored.add(ignoredPath); + for (const ignoredPath of result.stdout.split("\0")) ignored.add(gitPrefix + ignoredPath); } return { entries: entries.map((entry) => diff --git a/apps/server/src/ws.test.ts b/apps/server/src/ws.test.ts index d8d156c31eee4..5ca0878f224f0 100644 --- a/apps/server/src/ws.test.ts +++ b/apps/server/src/ws.test.ts @@ -51,6 +51,8 @@ import { import * as Deferred from "effect/Deferred"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Cause from "effect/Cause"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as PubSub from "effect/PubSub"; @@ -60,6 +62,7 @@ import * as TestClock from "effect/testing/TestClock"; import { Headers } from "effect/http"; import { RpcGroup, RpcMessage, RpcServer } from "effect/rpc"; import * as SqlClient from "effect/sql/SqlClient"; +import * as Statement from "effect/sql/Statement"; import * as PreviewAutomationBroker from "./mcp/PreviewAutomationBroker.ts"; import * as ServerBrowser from "./preview/ServerBrowser.ts"; @@ -144,9 +147,6 @@ import * as Keybindings from "./keybindings.ts"; import * as ServerSettings from "./serverSettings.ts"; import * as ManagedPolicy from "./policy/ManagedPolicy.ts"; -import * as EventStore from "./orchestration-v2/EventStore.ts"; -import * as ProjectionMaintenance from "./orchestration-v2/ProjectionMaintenance.ts"; -import * as ProjectionStore from "./orchestration-v2/ProjectionStore.ts"; import * as ThreadManagement from "./orchestration-v2/ThreadManagementService.ts"; import * as ProjectStore from "./orchestration-v2/ProjectStore.ts"; import * as OrchestrationEventStore from "./persistence/OrchestrationEventStore.ts"; @@ -947,6 +947,7 @@ it.effect.each([ {}, yield* ServerBrowser.ServerBrowser, () => Effect.void, + { secretReferences: true }, ).pipe(Layer.provide(configRpcServices)), ), ); @@ -991,8 +992,8 @@ it.effect.each([ key: "mod+alt+z", command: "terminal.split", }); - const expectedSettings = [duringDiscovery, latestSettings].map( - ServerSettings.redactServerSettingsForClient, + const expectedSettings = [duringDiscovery, latestSettings].map((value) => + ServerSettings.redactServerSettingsForClient(value, { secretReferences: true }), ); const expectedKeybindings = [duringAck.keybindings, latestKeybindings.keybindings]; // The two sources can interleave, but each must retain its own publication order. @@ -1020,7 +1021,7 @@ it.effect.each([ assert.deepEqual(resolved.config.remoteOpenTargets, targets); assert.deepEqual( resolved.config.settings, - ServerSettings.redactServerSettingsForClient(latestSettings), + ServerSettings.redactServerSettingsForClient(latestSettings, { secretReferences: true }), ); assert.deepEqual(resolved.config.keybindings, latestKeybindings.keybindings); yield* acknowledge; @@ -1035,7 +1036,9 @@ it.effect.each([ assert.deepEqual(resolved.config.directEndpoints, endpoints); assert.deepEqual( resolved.config.settings, - ServerSettings.redactServerSettingsForClient(latestSettings), + ServerSettings.redactServerSettingsForClient(latestSettings, { + secretReferences: true, + }), ); assert.deepEqual(resolved.config.keybindings, latestKeybindings.keybindings); yield* acknowledge; @@ -1165,6 +1168,7 @@ it.effect("on-demand clients get provider config without workspace snapshots", ( {}, yield* ServerBrowser.ServerBrowser, () => Effect.void, + { secretReferences: true }, ).pipe(Layer.provide(configRpcServices)), ), ); @@ -1222,93 +1226,176 @@ it.effect("on-demand clients get provider config without workspace snapshots", ( ), ); -// Compaction keeps the oldest and newest event per message, so a cursor older -// than the deleted rows still replays each message where the client would -// first have seen it (reducers append on first sight) and reaches the same -// final state. No snapshot fallback is needed. -it.effect("thread resume across compacted history keeps message order and final state", () => +// Pruned history cannot be replayed: a cursor below the boundary must resync from a snapshot. +it.effect.each(["thread", "shell"] as const)( + "%s resume from a cursor below the pruned boundary loads a snapshot", + (scope) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const now = "2026-10-13T00:00:00.000Z"; + const threadId = ThreadId.make("thread:pruned"); + const insert = (version: number) => sql<{ sequence: number }>` + INSERT INTO orchestration_events ( + event_id, aggregate_kind, stream_id, stream_version, event_type, + occurred_at, actor_kind, payload_json, metadata_json, application_event_version + ) VALUES ( + ${`pruned:${version}`}, 'thread', ${threadId}, ${version}, 'provider-session.detached', + ${now}, 'server', ${JSON.stringify({ providerSessionId: "session:pruned", detachedAt: now })}, + '{}', 2 + ) RETURNING sequence + `; + const [first] = yield* insert(0); + const [second] = yield* insert(1); + yield* insert(2); + yield* sql` + INSERT INTO orchestration_v2_projection_metadata ( + projection_name, schema_version, last_sequence, updated_at + ) VALUES (${OrchestrationEventStore.EVENT_LOG_PRUNED_THROUGH}, 2, ${second!.sequence}, ${now}) + `; + const subscription = Effect.gen(function* () { + const stream = + scope === "thread" + ? yield* subscribeOrchestrationV2Thread({ threadId, afterSequence: first!.sequence }) + : yield* subscribeOrchestrationV2Shell({ afterSequence: first!.sequence }); + return stream; + }); + const exit = yield* subscription.pipe( + Effect.provideService(ThreadManagement.ThreadManagementService, { + ensureLegacyTranscript: () => Effect.void, + getThreadSnapshot: () => Effect.die("snapshot"), + listActiveShellIndex: () => Effect.die("snapshot"), + readShellSnapshot: () => Effect.die("snapshot"), + listActiveSubagentThreadIds: () => Effect.die("replay"), + } as unknown as ThreadManagement.ThreadManagementService["Service"]), + Effect.provideService(ProjectStore.ProjectStoreV2, { + listShells: () => Effect.succeed([]), + } as unknown as ProjectStore.ProjectStoreV2["Service"]), + Effect.provideService( + ProjectService.ProjectService, + {} as ProjectService.ProjectService["Service"], + ), + Effect.provideService(ProjectEnrichment.ProjectEnrichmentService, { + subscribeChanges: PubSub.subscribe(yield* PubSub.unbounded()), + } as ProjectEnrichment.ProjectEnrichmentService["Service"]), + Effect.exit, + ); + assert.deepEqual(Exit.isFailure(exit) ? Cause.squash(exit.cause) : exit, "snapshot"); + }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)), +); + +// A prune pass can delete replayed rows after the boundary check; the read must not be trusted. +it.effect.each(["thread", "shell"] as const)( + "%s resume loads a snapshot when pruning advances during the replay read", + (scope) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const now = "2026-10-13T00:00:00.000Z"; + const threadId = ThreadId.make("thread:prune-race"); + const insert = (version: number) => sql<{ sequence: number }>` + INSERT INTO orchestration_events ( + event_id, aggregate_kind, stream_id, stream_version, event_type, + occurred_at, actor_kind, payload_json, metadata_json, application_event_version + ) VALUES ( + ${`prune-race:${version}`}, 'thread', ${threadId}, ${version}, 'provider-session.detached', + ${now}, 'server', ${JSON.stringify({ providerSessionId: "session:race", detachedAt: now })}, + '{}', 2 + ) RETURNING sequence + `; + const [first] = yield* insert(0); + const [second] = yield* insert(1); + yield* insert(2); + + const readStarted = yield* Deferred.make(); + const releaseRead = yield* Deferred.make(); + let gated = false; + const gateReplayRead: Statement.Transformer = (statement) => { + const [query] = statement.compile(); + // The paged replay read (readApplicationRows), not the stats or high-water queries. + if (!gated && /^\s*SELECT\s+sequence,\s+event_id,/.test(query)) { + gated = true; + return Deferred.succeed(readStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseRead)), + Effect.as(statement), + ); + } + return Effect.succeed(statement); + }; + const subscription = Effect.gen(function* () { + const stream = + scope === "thread" + ? yield* subscribeOrchestrationV2Thread({ threadId, afterSequence: first!.sequence }) + : yield* subscribeOrchestrationV2Shell({ afterSequence: first!.sequence }); + return stream; + }).pipe( + Effect.provideService(Statement.CurrentTransformer, gateReplayRead), + Effect.provideService(ThreadManagement.ThreadManagementService, { + ensureLegacyTranscript: () => Effect.void, + getThreadSnapshot: () => Effect.die("snapshot"), + listActiveShellIndex: () => Effect.die("snapshot"), + readShellSnapshot: () => Effect.die("snapshot"), + listActiveSubagentThreadIds: () => Effect.succeed([]), + } as unknown as ThreadManagement.ThreadManagementService["Service"]), + Effect.provideService(ProjectStore.ProjectStoreV2, { + listShells: () => Effect.succeed([]), + } as unknown as ProjectStore.ProjectStoreV2["Service"]), + Effect.provideService( + ProjectService.ProjectService, + {} as ProjectService.ProjectService["Service"], + ), + Effect.provideService(ProjectEnrichment.ProjectEnrichmentService, { + subscribeChanges: PubSub.subscribe(yield* PubSub.unbounded()), + } as ProjectEnrichment.ProjectEnrichmentService["Service"]), + Effect.exit, + Effect.forkChild, + ); + const fiber = yield* subscription; + yield* Deferred.await(readStarted); + // A pass publishes its boundary, then deletes the rows it covers. + yield* sql` + INSERT INTO orchestration_v2_projection_metadata ( + projection_name, schema_version, last_sequence, updated_at + ) VALUES (${OrchestrationEventStore.EVENT_LOG_PRUNED_THROUGH}, 2, ${second!.sequence}, ${now}) + `; + yield* sql`DELETE FROM orchestration_events WHERE sequence = ${second!.sequence}`; + yield* Deferred.succeed(releaseRead, undefined); + const exit = yield* Fiber.join(fiber); + assert.deepEqual(Exit.isFailure(exit) ? Cause.squash(exit.cause) : exit, "snapshot"); + }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)), +); + +// A thread idle since before the boundary missed nothing; resuming must not resend its body. +it.effect("thread resume at its own high-water below the pruned boundary skips the snapshot", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; const now = "2026-10-13T00:00:00.000Z"; - const threadId = ThreadId.make("thread:compacted"); - const insertMessage = (version: number, id: string, text: string) => sql<{ - sequence: number; - }>` + const threadId = ThreadId.make("thread:idle"); + const [last] = yield* sql<{ sequence: number }>` INSERT INTO orchestration_events ( event_id, aggregate_kind, stream_id, stream_version, event_type, occurred_at, actor_kind, payload_json, metadata_json, application_event_version ) VALUES ( - ${`compacted:${version}`}, 'thread', ${threadId}, ${version}, 'message.updated', - ${now}, 'server', ${JSON.stringify({ - createdBy: "user", - creationSource: "web", - id, - threadId, - runId: null, - nodeId: null, - role: "user", - text, - attachments: [], - streaming: false, - createdAt: now, - updatedAt: now, - })}, '{}', 2 + 'idle:0', 'thread', ${threadId}, 0, 'provider-session.detached', + ${now}, 'server', ${JSON.stringify({ providerSessionId: "session:idle", detachedAt: now })}, + '{}', 2 ) RETURNING sequence `; - const [first] = yield* insertMessage(0, "message:a", "a created"); - const [second] = yield* insertMessage(1, "message:b", "b created"); - yield* insertMessage(2, "message:a", "a superseded"); - const [head] = yield* insertMessage(3, "message:a", "a final"); - // thread.deleted is never superseded: clients switch on the event type. - for (const version of [0, 1]) { - yield* sql` - INSERT INTO orchestration_events ( - event_id, aggregate_kind, stream_id, stream_version, event_type, - occurred_at, actor_kind, payload_json, metadata_json, application_event_version - ) VALUES ( - ${`deleted:${version}`}, 'thread', 'thread:deleted-twice', ${version}, 'thread.deleted', - ${now}, 'server', '{}', '{}', 2 - ) - `; - } - assert.equal((yield* maintenance.compactEventStore()).deletedEventCount, 1); - - const items = yield* subscribeOrchestrationV2Thread({ + yield* sql` + INSERT INTO orchestration_v2_projection_metadata ( + projection_name, schema_version, last_sequence, updated_at + ) VALUES (${OrchestrationEventStore.EVENT_LOG_PRUNED_THROUGH}, 2, ${last!.sequence + 10}, ${now}) + `; + const exit = yield* subscribeOrchestrationV2Thread({ threadId, - afterSequence: first!.sequence - 1, - requestCompletionMarker: true, + afterSequence: last!.sequence, }).pipe( - Effect.flatMap((stream) => - stream.pipe( - Stream.takeUntil((item) => item.kind === "synchronized"), - Stream.runCollect, - ), - ), Effect.provideService(ThreadManagement.ThreadManagementService, { ensureLegacyTranscript: () => Effect.void, - streamStoredEventsFrom: () => Stream.never, + getThreadSnapshot: () => Effect.die("snapshot"), + streamStoredEventsFrom: () => Stream.empty, } as unknown as ThreadManagement.ThreadManagementService["Service"]), + Effect.exit, ); - assert.deepEqual( - Array.from(items, (item) => - item.kind === "event" - ? [item.sequence, item.event.type === "message.updated" ? item.event.payload.text : null] - : [item.kind], - ), - [ - [first!.sequence, "a created"], - [second!.sequence, "b created"], - [head!.sequence, "a final"], - ["synchronized"], - ], - ); - }).pipe( - Effect.provide( - ProjectionMaintenance.layer.pipe( - Layer.provide(Layer.merge(EventStore.layer, ProjectionStore.layer)), - Layer.provideMerge(ReplayTestLayer), - ), - ), - ), + // A snapshot load dies with "snapshot", so success means the resume replayed nothing. + assert.isTrue(Exit.isSuccess(exit)); + }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)), ); diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index b56276f6a2141..386d819251656 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -55,6 +55,8 @@ import { OrchestrationGetTurnDiffError, ORCHESTRATION_V2_WS_METHODS, ORCHESTRATION_PROTOCOL_QUERY_PARAM, + CLIENT_CAPABILITIES_QUERY_PARAM, + SETTINGS_SECRET_REFERENCES_CAPABILITY, ORCHESTRATION_PROTOCOL_VERSION, OrchestrationV2DispatchCommandError, OrchestrationV2GetShellSnapshotError, @@ -707,6 +709,11 @@ function hasCompatibleOrchestrationProtocol(url: URL): boolean { ); } +function readSettingsClient(url: URL) { + const capabilities = url.searchParams.get(CLIENT_CAPABILITIES_QUERY_PARAM)?.split(",") ?? []; + return { secretReferences: capabilities.includes(SETTINGS_SECRET_REFERENCES_CAPABILITY) }; +} + function shouldUseBoundedThreadSnapshot(input: { readonly acceptBoundedSnapshot?: boolean; readonly acceptDeferredWork?: boolean; @@ -792,6 +799,7 @@ const canReplayPersistedRange = Effect.fnUntraced(function* ( if (replayGap < 0 || replayGap > maxGap) { return false; } + if (afterSequence < (yield* applicationEvents.prunedThroughSequence)) return false; const stats = yield* applicationEvents.getReplayStats({ afterSequence, throughSequence: headSequence, @@ -980,7 +988,21 @@ export const subscribeOrchestrationV2Thread = Effect.fn("ws.orchestrationV2.subs }), ), ); - if (input.afterSequence > highWater) { + const prunedThrough = yield* applicationEvents.prunedThroughSequence.pipe( + Effect.mapError( + (cause) => + new OrchestrationV2GetThreadProjectionError({ + threadId: input.threadId, + message: `Failed to prepare orchestration V2 thread ${input.threadId} replay`, + cause, + }), + ), + ); + // An idle thread's cursor sits at its own high-water: nothing was missed, so replay nothing. + if ( + input.afterSequence > highWater || + (input.afterSequence < highWater && input.afterSequence < prunedThrough) + ) { return yield* snapshotThenLive(); } const stats = yield* applicationEvents @@ -1021,9 +1043,25 @@ export const subscribeOrchestrationV2Thread = Effect.fn("ws.orchestrationV2.subs // Retained creation events may describe deleted threads; replace replay only if a snapshot exists. if (shell !== null) return yield* snapshotThenLive(); } - const replay = coalesceLiveUpdatedEvents( - yield* loadReplayThrough(input.afterSequence, highWater), - ); + const replayed = yield* loadReplayThrough(input.afterSequence, highWater); + // Pruning publishes its boundary before deleting, so a pass during the read shows up here. + if ( + input.afterSequence < highWater && + input.afterSequence < + (yield* applicationEvents.prunedThroughSequence.pipe( + Effect.mapError( + (cause) => + new OrchestrationV2GetThreadProjectionError({ + threadId: input.threadId, + message: `Failed to prepare orchestration V2 thread ${input.threadId} replay`, + cause, + }), + ), + )) + ) { + return yield* snapshotThenLive(); + } + const replay = coalesceLiveUpdatedEvents(replayed); const plan = decideThreadResume({ afterSequence: input.afterSequence, highWater, @@ -1257,31 +1295,38 @@ export const subscribeOrchestrationV2Shell = Effect.fn("ws.orchestrationV2.subsc }); } - const highWater = yield* applicationEvents.latestApplicationSequence; - if (!(yield* canReplayPersistedRange(input.afterSequence, highWater, SHELL_RESUME_MAX_GAP))) { + const fromSnapshot = Effect.gen(function* () { const loaded = yield* loadSnapshot(); return composeShellStreamWithEnrichment({ initial: initialSnapshotItems(loaded), tail: completionThenLive(loaded.snapshot.snapshotSequence), enrichment: enrichmentRefreshes, }); + }); + const highWater = yield* applicationEvents.latestApplicationSequence; + if (!(yield* canReplayPersistedRange(input.afterSequence, highWater, SHELL_RESUME_MAX_GAP))) { + return yield* fromSnapshot; } subagentThreadIds = new Set(yield* threadManagement.listActiveSubagentThreadIds()); const loaded = yield* loadProjectMetadataSnapshot(highWater); - const replay = toShellStream( - applicationEvents - .readApplicationEvents({ - afterSequence: input.afterSequence, - throughSequence: highWater, - // A row a newer build wrote (e.g. after a downgrade) must not fail - // every shell resubscribe. The live tail below resumes from - // highWater, so a dropped row never hides later events. - skipUnknownEventTypes: true, - }) - .pipe(Stream.filter(filterSubagentTranscript)), - ); + // Read before replying: the range is budgeted above, and a prune pass during the read + // publishes its boundary first, so the recheck below catches deleted rows. + const replayed = yield* applicationEvents + .readApplicationEvents({ + afterSequence: input.afterSequence, + throughSequence: highWater, + // A row a newer build wrote (e.g. after a downgrade) must not fail + // every shell resubscribe. The live tail below resumes from + // highWater, so a dropped row never hides later events. + skipUnknownEventTypes: true, + }) + .pipe(Stream.filter(filterSubagentTranscript), Stream.runCollect); + if (input.afterSequence < (yield* applicationEvents.prunedThroughSequence)) { + return yield* fromSnapshot; + } + const replay = toShellStream(Stream.fromIterable(replayed)); return composeShellStreamWithEnrichment({ initial: initialEnrichmentItems(loaded), tail: Stream.concat(Stream.concat(replay, completionMarker), liveFrom(highWater)), @@ -1315,6 +1360,7 @@ export const layerWsRpc = ( clientAnalyticsProps: Readonly>, serverBrowser: ServerBrowser.ServerBrowser["Service"], reportProfiles: ServerBrowser.ProfileReporter, + settingsClient: { readonly secretReferences: boolean }, ) => Layer.effectContext( Effect.gen(function* () { @@ -1838,6 +1884,7 @@ export const layerWsRpc = ( ); const settings = ServerSettings.redactServerSettingsForClient( yield* serverSettings.getSettings, + settingsClient, ); const environment = yield* serverEnvironment.getDescriptor; const auth = yield* serverAuth.getDescriptor(); @@ -2412,13 +2459,13 @@ export const layerWsRpc = ( fresh: input.fresh, }); } + if (input.reloadSecrets === true && input.instanceId === undefined) { + yield* invalidateSettingsSecrets(secretResolver, serverSettings.getSettings); + } // Await refresh; RPC scope closure would interrupt a fork before the hub answered. if (input.instanceId === undefined) { yield* usageLimitSources.refresh; } - if (input.reloadSecrets === true && input.instanceId === undefined) { - yield* invalidateSettingsSecrets(secretResolver, serverSettings.getSettings); - } let providers = yield* input.cwd !== undefined && input.instanceId !== undefined ? providerRegistry.refreshWorkspaceSnapshot({ instanceId: input.instanceId, @@ -2629,7 +2676,7 @@ export const layerWsRpc = ( const settings = yield* providerInstanceMutation === undefined ? serverSettings.updateSettings(nextPatch) : serverSettings.updateProviderInstance(providerInstanceMutation, nextPatch); - return ServerSettings.redactServerSettingsForClient(settings); + return ServerSettings.redactServerSettingsForClient(settings, settingsClient); }), [WS_METHODS.serverSuggestWorktreeCleanupIgnoredNames]: Effect.fn( "ws.serverSuggestWorktreeCleanupIgnoredNames", @@ -3343,7 +3390,9 @@ export const layerWsRpc = ( ) : Stream.empty; const settingsUpdates = settingsChanges.pipe( - Stream.map((settings) => ServerSettings.redactServerSettingsForClient(settings)), + Stream.map((settings) => + ServerSettings.redactServerSettingsForClient(settings, settingsClient), + ), Stream.map((settings) => ({ version: 1 as const, type: "settingsUpdated" as const, @@ -3530,7 +3579,13 @@ export const layer = Layer.unwrap( return httpEffect; }).pipe( Effect.provide( - layerWsRpc(session, clientAnalyticsProps, serverBrowser, reportProfiles).pipe( + layerWsRpc( + session, + clientAnalyticsProps, + serverBrowser, + reportProfiles, + readSettingsClient(url.value), + ).pipe( Layer.provide( TextGeneration.layer.pipe( Layer.provide( diff --git a/apps/web/public/.well-known/apple-app-site-association.json b/apps/web/public/.well-known/apple-app-site-association.json new file mode 100644 index 0000000000000..b5bcbc50e573b --- /dev/null +++ b/apps/web/public/.well-known/apple-app-site-association.json @@ -0,0 +1,14 @@ +{ + "applinks": { + "details": [ + { + "appIDs": [ + "ARK85ZXQ4Z.com.t3tools.t3code", + "ARK85ZXQ4Z.com.t3tools.t3code.preview", + "ARK85ZXQ4Z.com.t3tools.t3code.dev" + ], + "components": [{ "/": "/threads/*" }] + } + ] + } +} diff --git a/apps/web/public/.well-known/assetlinks.json b/apps/web/public/.well-known/assetlinks.json new file mode 100644 index 0000000000000..317c04cf92d49 --- /dev/null +++ b/apps/web/public/.well-known/assetlinks.json @@ -0,0 +1,32 @@ +[ + { + "relation": ["delegate_permission/common.handle_all_urls"], + "target": { + "namespace": "android_app", + "package_name": "com.t3tools.t3code", + "sha256_cert_fingerprints": [ + "B9:F5:1E:D5:A7:6C:D1:86:C9:51:08:F2:19:91:B7:D5:3A:58:FD:A9:83:12:A9:E3:53:E2:1A:59:B2:42:03:B6" + ] + } + }, + { + "relation": ["delegate_permission/common.handle_all_urls"], + "target": { + "namespace": "android_app", + "package_name": "com.t3tools.t3code.preview", + "sha256_cert_fingerprints": [ + "BE:B8:BD:67:53:2F:12:95:0A:30:62:88:91:C1:2F:D0:CF:20:28:C3:81:5D:6D:31:45:E7:5A:AF:24:A2:8F:18" + ] + } + }, + { + "relation": ["delegate_permission/common.handle_all_urls"], + "target": { + "namespace": "android_app", + "package_name": "com.t3tools.t3code.dev", + "sha256_cert_fingerprints": [ + "FA:C6:17:45:DC:09:03:78:6F:B9:ED:E6:2A:96:2B:39:9F:73:48:F0:BB:6F:89:9B:83:32:66:75:91:03:3B:9C" + ] + } + } +] diff --git a/apps/web/src/components/BranchToolbar.logic.test.ts b/apps/web/src/components/BranchToolbar.logic.test.ts index 08e6f7633dee3..903a20066b39e 100644 --- a/apps/web/src/components/BranchToolbar.logic.test.ts +++ b/apps/web/src/components/BranchToolbar.logic.test.ts @@ -160,6 +160,18 @@ describe("dedupeRemoteBranchesWithLocalMatches", () => { describe("resolveBranchSelectionTarget", () => { it("reuses an existing secondary worktree for the selected ref", () => { + expect( + resolveBranchSelectionTarget({ + activeProjectCwd: "/repo/nested-checkout/packages/app", + activeProjectCheckoutRoot: "/repo/nested-checkout", + activeWorktreePath: null, + refName: { isDefault: true, worktreePath: "/repo" }, + }), + ).toEqual({ + checkoutCwd: "/repo", + nextWorktreePath: "/repo", + reuseExistingWorktree: true, + }); expect( resolveBranchSelectionTarget({ activeProjectCwd: "/repo", @@ -177,20 +189,31 @@ describe("resolveBranchSelectionTarget", () => { }); it("switches back to the main repo when the ref already lives there", () => { - expect( - resolveBranchSelectionTarget({ - activeProjectCwd: "/repo", - activeWorktreePath: "/repo/.t3/worktrees/feature-a", - refName: { - isDefault: true, - worktreePath: "/repo", - }, - }), - ).toEqual({ - checkoutCwd: "/repo", - nextWorktreePath: null, - reuseExistingWorktree: true, - }); + // A subdirectory project's own checkout is reported at the repository root. + // The checkout root is undefined until the project's status loads. + for (const [activeProjectCwd, activeWorktreePath, activeProjectCheckoutRoot] of [ + ["/repo", "/repo/.t3/worktrees/feature-a", "/repo"], + ["/repo/packages/app", "/repo/.t3/worktrees/feature-a", "/repo"], + ["/repo/packages/app", null, "/repo"], + ["/repo/packages/app", "/repo/.t3/worktrees/feature-a", undefined], + ["/repo/packages/app", null, undefined], + ] as const) { + expect( + resolveBranchSelectionTarget({ + activeProjectCwd, + activeProjectCheckoutRoot, + activeWorktreePath, + refName: { + isDefault: true, + worktreePath: "/repo", + }, + }), + ).toEqual({ + checkoutCwd: "/repo", + nextWorktreePath: null, + reuseExistingWorktree: true, + }); + } }); it("keeps checkout in the current worktree for non-default refs", () => { diff --git a/apps/web/src/components/BranchToolbar.logic.ts b/apps/web/src/components/BranchToolbar.logic.ts index e22dcfaddedeb..a967c3171fbd1 100644 --- a/apps/web/src/components/BranchToolbar.logic.ts +++ b/apps/web/src/components/BranchToolbar.logic.ts @@ -7,6 +7,7 @@ import type { } from "@t3tools/contracts"; import * as Schema from "effect/Schema"; import { sanitizeNewRefName } from "@t3tools/shared/git"; +import { workspaceRelativeFilePath } from "@t3tools/shared/path"; import { toSortableTimestamp } from "@t3tools/client-runtime/state/thread-sort"; export { deriveLocalBranchNameFromRemoteRef, sanitizeNewRefName } from "@t3tools/shared/git"; @@ -247,6 +248,7 @@ export function resolveLocalCheckoutBranchMismatch(input: { export function resolveBranchSelectionTarget(input: { activeProjectCwd: string; + activeProjectCheckoutRoot?: string | undefined; activeWorktreePath: string | null; refName: Pick; }): { @@ -257,9 +259,14 @@ export function resolveBranchSelectionTarget(input: { const { activeProjectCwd, activeWorktreePath, refName } = input; if (refName.worktreePath) { + // Until the project's checkout root loads, a checkout containing the project counts as its own. + const isProjectCheckout = + input.activeProjectCheckoutRoot === undefined + ? workspaceRelativeFilePath(activeProjectCwd, refName.worktreePath) !== null + : workspaceRelativeFilePath(input.activeProjectCheckoutRoot, refName.worktreePath) === "."; return { checkoutCwd: refName.worktreePath, - nextWorktreePath: refName.worktreePath === activeProjectCwd ? null : refName.worktreePath, + nextWorktreePath: isProjectCheckout ? null : refName.worktreePath, reuseExistingWorktree: true, }; } diff --git a/apps/web/src/components/BranchToolbarBranchSelector.tsx b/apps/web/src/components/BranchToolbarBranchSelector.tsx index e89b74eb8b565..dbdcac33ece3e 100644 --- a/apps/web/src/components/BranchToolbarBranchSelector.tsx +++ b/apps/web/src/components/BranchToolbarBranchSelector.tsx @@ -224,6 +224,12 @@ export function BranchToolbarBranchSelector({ input: { cwd: branchCwd }, }), ); + const projectStatusQuery = useEnvironmentQuery( + activeProjectCwd === null + ? null + : vcsEnvironment.status({ environmentId, input: { cwd: activeProjectCwd } }), + ); + const activeProjectCheckoutRoot = projectStatusQuery.data?.workingTree.rootPath; const trimmedBranchQuery = branchQuery.trim(); const branchRefState = usePaginatedBranches({ environmentId, @@ -389,6 +395,7 @@ export function BranchToolbarBranchSelector({ const selectionTarget = resolveBranchSelectionTarget({ activeProjectCwd, + activeProjectCheckoutRoot, activeWorktreePath, refName, }); @@ -631,6 +638,7 @@ export function BranchToolbarBranchSelector({ (!activeProjectCwd || !resolveBranchSelectionTarget({ activeProjectCwd, + activeProjectCheckoutRoot, activeWorktreePath, refName, }).reuseExistingWorktree)) diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index c794b3eeea1b4..100470fc8f9f0 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -44,6 +44,7 @@ import { resolveNewThreadModelSelection } from "@t3tools/shared/usageLimits"; import { queuedRunEditBlockedReason } from "@t3tools/client-runtime/queued-run-edit"; import { ScrollToEndButton, WorkspaceFileDropOverlay } from "./chat/TimelineOverlays"; import { usePasteToFocusComposer } from "./chat/usePasteToFocusComposer"; +import { configuredHostedAppUrl } from "../hostedPairing"; import { announcedRun, isDetailBehindShell } from "./chat/threadActivityAnnouncement"; import { deriveAvailableRevertTurnCounts, @@ -216,7 +217,6 @@ import { GuestPendingMessages } from "./chat/GuestShareControls"; import { useDeviceState } from "~/state/device"; import { BranchToolbar, type BranchToolbarHandle } from "./BranchToolbar"; import { ComposerContextPermissions } from "./ComposerContextPermissions"; -import { resolveCompatibleRuntimeMode } from "./chat/runtimeModeConfig"; import { makeWorkspaceFileDropHandlers } from "./chat/workspaceFileDrop"; import { isEditableFocused } from "../lib/editableFocus"; import { DEFAULT_RESOLVED_KEYBINDINGS } from "@t3tools/shared/keybindings"; @@ -1999,9 +1999,6 @@ export default function ChatView(props: ChatViewProps) { }); }; const scheduleComposerFocusStable = useStableCallback(scheduleComposerFocus); - const openComposerPermissionsStable = useStableCallback(() => - composerRef.current?.openControl("composer.mode"), - ); const insertArtifactTemplate = (template: CodexArtifactTemplate) => { const composer = composerRef.current; if (!composer) return; @@ -2533,6 +2530,7 @@ export default function ChatView(props: ChatViewProps) { openPanelPullRequestUrl, pullRequests: activeThread?.pullRequests, linkedPullRequestUrl: linkedThreadPullRequest?.url ?? null, + hostedAppUrl: configuredHostedAppUrl(), }); const copyActiveThreadReference = () => { if (activeThreadReferenceCopyTarget === null) return; @@ -3720,6 +3718,7 @@ export default function ChatView(props: ChatViewProps) { {isWorkspaceFileDragActive ? : null} ), })} diff --git a/apps/web/src/components/CommandPalette.browser.test.tsx b/apps/web/src/components/CommandPalette.browser.test.tsx index 3883a3e2258e9..9faeab83c6ffc 100644 --- a/apps/web/src/components/CommandPalette.browser.test.tsx +++ b/apps/web/src/components/CommandPalette.browser.test.tsx @@ -1,13 +1,28 @@ -import { describe, expect, it } from "vite-plus/test"; +import { EnvironmentId, ProjectId, ThreadId } from "@t3tools/contracts"; +import { describe, expect, it, vi } from "vite-plus/test"; import { page } from "vite-plus/test/browser"; import { render } from "../../test/render"; import { openCommandPalette } from "../commandPaletteBus"; import { useCommandPaletteStore } from "../commandPaletteStore"; +import { makeThreadFixture } from "../test-fixtures"; +import type { Project, Thread } from "../types"; import { CommandPalette } from "./CommandPalette"; -function setup() { +const entities = vi.hoisted(() => ({ + projects: [] as ReadonlyArray, + threads: [] as ReadonlyArray, +})); +vi.mock("../state/entities", async (original) => ({ + ...(await original()), + useProjects: () => entities.projects, + useVisibleThreadShells: () => entities.threads, +})); + +function setup(state: { projects?: ReadonlyArray; threads?: ReadonlyArray } = {}) { useCommandPaletteStore.setState(useCommandPaletteStore.getInitialState(), true); + entities.projects = state.projects ?? []; + entities.threads = state.threads ?? []; } describe("CommandPalette open intents", () => { @@ -30,4 +45,50 @@ describe("CommandPalette open intents", () => { await expect.element(input).toHaveFocus(); await expect.element(input).toHaveValue("acme"); }); + + it("opens a searched project's latest listed thread, not a newer subagent", async () => { + const environmentId = EnvironmentId.make("local"); + const projectId = ProjectId.make("acme"); + const parentId = ThreadId.make("parent"); + setup({ + projects: [ + { + id: projectId, + environmentId, + title: "Acme", + workspaceRoot: "/work/acme", + repositoryIdentity: null, + defaultModelSelection: null, + createdAt: "2026-03-01T00:00:00.000Z", + updatedAt: "2026-03-01T00:00:00.000Z", + scripts: [], + }, + ], + threads: [ + makeThreadFixture({ + id: parentId, + environmentId, + projectId, + latestUserMessageAt: "2026-03-02T00:00:00.000Z", + }), + makeThreadFixture({ + id: ThreadId.make("subagent"), + environmentId, + projectId, + updatedAt: "2026-03-03T00:00:00.000Z", + lineage: { + rootThreadId: parentId, + parentThreadId: parentId, + relationshipToParent: "subagent", + }, + }), + ], + }); + await using screen = await render({null}); + + openCommandPalette({ query: "Acme" }); + await page.getByRole("option", { name: /Acme/ }).first().click(); + + await vi.waitFor(() => expect(screen.router?.state.location.pathname).toBe("/local/parent")); + }); }); diff --git a/apps/web/src/components/CommandPalette.tsx b/apps/web/src/components/CommandPalette.tsx index 78abd15de4112..eebbfec09410a 100644 --- a/apps/web/src/components/CommandPalette.tsx +++ b/apps/web/src/components/CommandPalette.tsx @@ -124,6 +124,7 @@ import { useRestartAgentSession } from "../hooks/useRestartAgentSession"; import { useRunScheduledTask } from "../hooks/useRunThreadScheduledTask"; import { useStableCallback } from "../hooks/useStableCallback"; import { readLocalApi } from "../localApi"; +import { configuredHostedAppUrl } from "../hostedPairing"; import { desktopLocalBackendId } from "../connection/desktopLocal"; import { filesystemEnvironment, useFilesystemReadAccess } from "../state/filesystem"; import { projectEnvironment } from "../state/projects"; @@ -233,6 +234,7 @@ import { type SearchOverlayMode, } from "./CommandPalette.logic"; import { resolveEnvironmentOptionLabel } from "./BranchToolbar.logic"; +import { isSidebarHiddenThread } from "./Sidebar.logic"; import { CommandPaletteContent } from "./CommandPaletteContent"; import { selectCommandPaletteClientSettings, @@ -578,12 +580,14 @@ function OpenCommandPaletteDialog(props: { pullRequests: activeThread?.pullRequests, linkedPullRequestUrl: activeThread?.linkedPullRequest?.url ?? activeThread?.branchPullRequest?.url ?? null, + hostedAppUrl: configuredHostedAppUrl(), }); const copyActiveThreadReference = async () => { if (activeThreadReferenceCopyTarget === null) return; await copyThreadReference(activeThreadReferenceCopyTarget); }; - const threads = useVisibleThreadShells(); + // Side chats and subagents open inside their parent, so opening a project never lands on one. + const threads = useVisibleThreadShells().filter((thread) => !isSidebarHiddenThread(thread)); const keybindings = useAtomValue(primaryServerKeybindingsAtom); const providersByEnvironment = useEnvironmentProviders(); const { changeThemeItem, changeAppearanceItem, themeEditorItem } = useCommandPaletteThemeItems(); diff --git a/apps/web/src/components/ComposerContextPermissions.tsx b/apps/web/src/components/ComposerContextPermissions.tsx index b4ac2a2eb6983..688968f9bca53 100644 --- a/apps/web/src/components/ComposerContextPermissions.tsx +++ b/apps/web/src/components/ComposerContextPermissions.tsx @@ -1,41 +1,67 @@ -import type { RuntimeMode } from "@t3tools/contracts"; +import type { EnvironmentId, RuntimeMode } from "@t3tools/contracts"; import { useAtomValue } from "@effect/atom-react"; import { shortcutLabelForCommand } from "../keybindings"; +import { useManagedPolicy } from "../state/managedPolicy"; import { primaryServerKeybindingsAtom } from "../state/server"; import { ComposerControl } from "./chat/ComposerControl"; -import { runtimeModeConfig } from "./chat/runtimeModeConfig"; +import { ComposerPermissionsSection } from "./chat/ComposerFooterControls"; +import { useComposerMenuProps } from "./chat/composerEventScope"; +import { + resolveCompatibleRuntimeMode, + resolveRuntimeModeOptions, + runtimeModeConfig, +} from "./chat/runtimeModeConfig"; import { ComposerContextLabel } from "./ComposerContextLabel"; +import { Menu, MenuPopup, MenuTrigger } from "./ui/menu"; import { Tooltip, TooltipPopup, TooltipTrigger } from "./ui/tooltip"; -/** The new-thread strip names the permissions the first send will use; clicking opens them. */ export function ComposerContextPermissions(props: { + environmentId: EnvironmentId | null; runtimeMode: RuntimeMode; - onOpen: () => void; + supportedRuntimeModes: ReadonlyArray | undefined; + onRuntimeModeChange: (mode: RuntimeMode) => void; }) { const shortcutLabel = shortcutLabelForCommand( useAtomValue(primaryServerKeybindingsAtom), "composer.mode", ); - const { icon: Icon, label } = runtimeModeConfig[props.runtimeMode]; + const { mode, modes } = resolveCompatibleRuntimeMode( + props.runtimeMode, + props.supportedRuntimeModes, + ); + const options = resolveRuntimeModeOptions(modes, useManagedPolicy(props.environmentId)); + const composerMenuProps = useComposerMenuProps(); + const { icon: Icon, label } = runtimeModeConfig[mode]; return ( - - - } - > - - Permissions{shortcutLabel ? ` · ${shortcutLabel}` : null} - + + + + } + /> + } + > + + Permissions{shortcutLabel ? ` · ${shortcutLabel}` : null} + + + + + ); } diff --git a/apps/web/src/components/Sidebar.browser.test.tsx b/apps/web/src/components/Sidebar.browser.test.tsx new file mode 100644 index 0000000000000..875a2225d70a1 --- /dev/null +++ b/apps/web/src/components/Sidebar.browser.test.tsx @@ -0,0 +1,89 @@ +import { DEFAULT_CLIENT_SETTINGS, EnvironmentId, ProjectId } from "@t3tools/contracts"; +import { describe, expect, it, vi } from "vite-plus/test"; +import { userEvent } from "vite-plus/test/browser"; + +import { render } from "../../test/render"; +import { openCommandPalette } from "../commandPaletteBus"; +import { useHandleNewThread } from "../hooks/useHandleNewThread"; +import { __setClientSettingsForTests, getClientSettings } from "../hooks/useSettings"; +import { useProjects } from "../state/entities"; +import type { Project } from "../types"; +import Sidebar from "./Sidebar"; +import { SidebarProvider } from "./ui/sidebar"; + +vi.mock("../state/entities", { spy: true }); +vi.mock("../hooks/useHandleNewThread", { spy: true }); +vi.mock("../commandPaletteBus", () => ({ + openCommandPalette: vi.fn(), + isCommandPaletteOpen: () => false, +})); +vi.mock("./sidebar/SidebarChrome", () => ({ + SidebarChromeHeader: () => null, + SidebarChromeFooter: () => null, +})); + +function project(id: string): Project { + return { + id: ProjectId.make(id), + environmentId: EnvironmentId.make("local"), + title: id, + workspaceRoot: `/projects/${id}`, + repositoryIdentity: null, + defaultModelSelection: null, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + scripts: [], + }; +} + +describe("sidebar header new thread", () => { + it.each([ + { mode: "normal", count: 2, shift: false, picker: true }, + { mode: "boring", count: 2, shift: false, picker: true }, + { mode: "boring", count: 2, shift: true, picker: false }, + { mode: "boring", count: 1, shift: false, picker: false }, + { mode: "legacy", count: 2, shift: false, picker: false }, + ] as const)("$mode sidebar, $count projects, shift=$shift: picker=$picker", async (scenario) => { + using stack = new DisposableStack(); + const previousSettings = getClientSettings(); + stack.defer(() => __setClientSettingsForTests(previousSettings)); + __setClientSettingsForTests({ + ...DEFAULT_CLIENT_SETTINGS, + boringSidebarEnabled: scenario.mode === "boring", + legacySidebarEnabled: scenario.mode === "legacy", + }); + const projects = [project("active"), project("idle")].slice(0, scenario.count); + const currentProjectRef = { + environmentId: projects[0]!.environmentId, + projectId: projects[0]!.id, + }; + const startThread = vi.fn().mockResolvedValue(null); + vi.mocked(useProjects).mockReturnValue(projects); + vi.mocked(useHandleNewThread).mockReturnValue({ + activeDraftThread: null, + activeThread: null, + defaultProjectRef: currentProjectRef, + handleNewThread: startThread, + routeDraftId: null, + routeThreadRef: null, + }); + vi.mocked(openCommandPalette).mockClear(); + await using screen = await render( + + + , + ); + + await userEvent.click(screen.getByRole("button", { name: "New thread", exact: true }), { + modifiers: scenario.shift ? ["Shift"] : [], + }); + + if (scenario.picker) { + expect(openCommandPalette).toHaveBeenCalledExactlyOnceWith({ open: "new-thread-in" }); + expect(startThread).not.toHaveBeenCalled(); + } else { + expect(openCommandPalette).not.toHaveBeenCalled(); + expect(startThread).toHaveBeenCalledExactlyOnceWith(currentProjectRef); + } + }); +}); diff --git a/apps/web/src/components/Sidebar.logic.test.ts b/apps/web/src/components/Sidebar.logic.test.ts index 14710e4a04d34..ec47d51fab607 100644 --- a/apps/web/src/components/Sidebar.logic.test.ts +++ b/apps/web/src/components/Sidebar.logic.test.ts @@ -1781,6 +1781,7 @@ function makeThread(overrides: ThreadFixtureOverrides = {}): Thread { describe("getFallbackThreadIdAfterDelete", () => { it("returns the top remaining thread in the deleted thread's project sidebar order", () => { const fallbackThreadId = getFallbackThreadIdAfterDelete({ + now: "2026-03-09T10:30:00.000Z", threads: [ makeThread({ id: ThreadId.make("thread-oldest"), @@ -1839,10 +1840,45 @@ describe("getFallbackThreadIdAfterDelete", () => { deletedThreadId: ThreadId.make("thread-active"), deletedThreadIds: new Set([ThreadId.make("thread-active"), ThreadId.make("thread-newest")]), sortOrder: "created_at", + now: "2026-03-09T10:10:00.000Z", }); expect(fallbackThreadId).toBe(ThreadId.make("thread-next")); }); + + it.each([ + [ + "snoozed", + { snoozedUntil: "2099-01-01T00:00:00.000Z", snoozedAt: "2026-03-09T10:06:00.000Z" }, + ], + ["settled", { settledOverride: "settled", settledAt: "2026-03-09T10:06:00.000Z" }], + ] as const)("skips a %s thread parked on its shelf", (_state, parked) => { + const fallbackThreadId = getFallbackThreadIdAfterDelete({ + threads: [ + makeThread({ + id: ThreadId.make("thread-visible"), + createdAt: "2026-03-09T10:01:00.000Z", + updatedAt: "2026-03-09T10:01:00.000Z", + }), + makeThread({ + id: ThreadId.make("thread-parked"), + createdAt: "2026-03-09T10:02:00.000Z", + updatedAt: "2026-03-09T10:06:00.000Z", + ...parked, + }), + makeThread({ + id: ThreadId.make("thread-active"), + createdAt: "2026-03-09T10:03:00.000Z", + updatedAt: "2026-03-09T10:03:00.000Z", + }), + ], + deletedThreadId: ThreadId.make("thread-active"), + sortOrder: "created_at", + now: "2026-03-09T10:10:00.000Z", + }); + + expect(fallbackThreadId).toBe(ThreadId.make("thread-visible")); + }); }); describe("sortScopedProjectsForSidebar", () => { it("keeps identical project ids in different environments separate", () => { diff --git a/apps/web/src/components/Sidebar.logic.ts b/apps/web/src/components/Sidebar.logic.ts index 9c0ce609b7ebe..fe984b96760e1 100644 --- a/apps/web/src/components/Sidebar.logic.ts +++ b/apps/web/src/components/Sidebar.logic.ts @@ -1681,14 +1681,17 @@ export function resolveThreadStatusPill(input: { } export function getFallbackThreadIdAfterDelete< - T extends Pick & ThreadSortInput, + T extends Pick & + ThreadSortInput & + ThreadSnoozeShell, >(input: { threads: readonly T[]; deletedThreadId: T["id"]; sortOrder: SidebarThreadSortOrder; + now: string; deletedThreadIds?: ReadonlySet; }): T["id"] | null { - const { deletedThreadId, deletedThreadIds, sortOrder, threads } = input; + const { deletedThreadId, deletedThreadIds, sortOrder, now, threads } = input; const deletedThread = threads.find((thread) => thread.id === deletedThreadId); if (!deletedThread) { return null; @@ -1700,7 +1703,9 @@ export function getFallbackThreadIdAfterDelete< (thread) => thread.projectId === deletedThread.projectId && thread.id !== deletedThreadId && - !deletedThreadIds?.has(thread.id), + !deletedThreadIds?.has(thread.id) && + thread.settledOverride !== "settled" && + !effectiveSnoozed(thread, { now }), ), sortOrder, )[0]?.id ?? null diff --git a/apps/web/src/components/Sidebar.tsx b/apps/web/src/components/Sidebar.tsx index f017baf0edcf9..1034efe899353 100644 --- a/apps/web/src/components/Sidebar.tsx +++ b/apps/web/src/components/Sidebar.tsx @@ -126,6 +126,7 @@ import { isModelPickerOpen } from "../modelPickerVisibility"; import { selectThreadTerminalUiState, useTerminalUiStateStore } from "../terminalUiStateStore"; import { isMacPlatform, cn } from "~/lib/utils"; import { readLocalApi } from "../localApi"; +import { configuredHostedAppUrl } from "../hostedPairing"; import { useSidebarPendingFileDropStore } from "../sidebarPendingFileDropStore"; import { pickableProjectGroups, @@ -412,6 +413,7 @@ export default function Sidebar({ routeReveal }: { routeReveal: SidebarRouteReve const confirmThreadArchive = useClientSettings((s) => s.confirmThreadArchive); const timestampFormat = useClientSettings((s) => s.timestampFormat); const boringSidebarEnabled = useBoringSidebarEnabled(); + const legacySidebarEnabled = useClientSettings((settings) => settings.legacySidebarEnabled); const workingShelfEnabled = useClientSettings((s) => s.sidebarWorkingShelfEnabled) && !boringSidebarEnabled; const projectGroupingSettings = useClientSettings(selectProjectGroupingSettings); @@ -2915,7 +2917,9 @@ export default function Sidebar({ routeReveal }: { routeReveal: SidebarRouteReve } return; case "copy-thread": - void copyThreadReference(threadChipCopyTarget(threadRef, thread.title)); + void copyThreadReference( + threadChipCopyTarget(threadRef, thread.title, configuredHostedAppUrl()), + ); return; case "copy-thread-id": copyThreadIdToClipboard(thread.id, { threadId: thread.id }); @@ -3095,6 +3099,7 @@ export default function Sidebar({ routeReveal }: { routeReveal: SidebarRouteReve const showJumpHints = useThreadJumpHintsVisible(shouldShowJumpHintsNow); const handleNewThreadClick = (event?: ReactMouseEvent) => { if ( + legacySidebarEnabled || shouldCreateNewThreadInCurrentProject( event?.shiftKey ?? false, pickableProjectGroups( @@ -3156,7 +3161,7 @@ export default function Sidebar({ routeReveal }: { routeReveal: SidebarRouteReve newThreadDisabled={projects.length === 0} newThreadShortcutLabel={newThreadShortcutLabel} newThreadInProjectShortcutLabel={newThreadInProjectShortcutLabel} - showNewThreadInProjectHint={projectChoiceCount > 1} + showNewThreadInProjectHint={!legacySidebarEnabled && projectChoiceCount > 1} searchInputRef={threadSearchInputRef} searchQuery={isThreadSearchApplied ? "" : threadSearchQuery} onSearchFocus={() => setIsThreadSearchApplied(false)} diff --git a/apps/web/src/components/chat/ChatBannerLane.tsx b/apps/web/src/components/chat/ChatBannerLane.tsx index 41ee46353f908..8d8a6cb01b839 100644 --- a/apps/web/src/components/chat/ChatBannerLane.tsx +++ b/apps/web/src/components/chat/ChatBannerLane.tsx @@ -1,4 +1,5 @@ import type { + EnvironmentId, OrchestrationV2ProviderFailureClass, ProviderInstanceId, ServerProvider, @@ -13,6 +14,7 @@ import { dismissThreadErrorBannerForSession, ThreadErrorBanner } from "./ThreadE * values, so a ChatView render that changes neither banner leaves both cached. */ export function ChatBannerLane(props: { + environmentId: EnvironmentId; providerStatus: ServerProvider | null; providerStatusBannerKey: string | null; setDismissedProviderStatusBannerKey: Dispatch>; @@ -41,6 +43,7 @@ export function ChatBannerLane(props: { return (
setDismissedProviderStatusBannerKey(providerStatusBannerKey)} onOpenProviderSetup={onOpenProviderSetup} diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 95139fbb164d6..03dfac9865ed2 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -101,7 +101,7 @@ import { BackgroundThreadContext } from "./backgroundThread"; import { resolveCompatibleRuntimeMode, runtimeModeConfig, - runtimeModeOptions as runtimeModes, + resolveRuntimeModeOptions, } from "./runtimeModeConfig"; import { isLocalEnvironmentDisabled } from "../../localEnvironment"; import { usePrimaryEnvironmentId } from "../../state/environments"; @@ -341,8 +341,6 @@ function consumePasteAsTextShortcut(deadline: { current: number }): boolean { return armed; } -const runtimeModeOptions = runtimeModes.map((mode) => ({ mode, ...runtimeModeConfig[mode] })); - export interface ChatComposerHandle { focusAtEnd: () => void; focusAt: (cursor: number) => void; @@ -1040,12 +1038,10 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) ); // Modes above an administrator's ceiling stay listed, disabled with the reason. const managedPolicy = useManagedPolicy(environmentId); - const compatibleRuntimeModeOptions = runtimeModeOptions - .filter((option) => compatibleRuntimeModes.includes(option.mode)) - .map((option) => { - const disabledReason = managedRuntimeModeDenial(managedPolicy, option.mode); - return disabledReason === undefined ? option : { ...option, disabledReason }; - }); + const compatibleRuntimeModeOptions = resolveRuntimeModeOptions( + compatibleRuntimeModes, + managedPolicy, + ); // A thread or draft already above the ceiling would be refused; offer the broadest mode left. const runtimeModeCeilingDenial = managedRuntimeModeDenial(managedPolicy, compatibleRuntimeMode); const runtimeModeCeilingFallback = diff --git a/apps/web/src/components/chat/ComposerPermissionsSection.browser.test.tsx b/apps/web/src/components/chat/ComposerPermissionsSection.browser.test.tsx index 850256b28904a..0fe4cc14d3869 100644 --- a/apps/web/src/components/chat/ComposerPermissionsSection.browser.test.tsx +++ b/apps/web/src/components/chat/ComposerPermissionsSection.browser.test.tsx @@ -5,6 +5,7 @@ import { page, userEvent } from "vite-plus/test/browser"; import { render } from "../../../test/render"; import { ComposerHandleContext, type ComposerHandleRef } from "../../composerHandleContext"; +import { ComposerContextPermissions } from "../ComposerContextPermissions"; import { Menu, MenuPopup, MenuRadioGroup, MenuRadioItem, MenuTrigger } from "../ui/menu"; import { focusRequestedPermission, @@ -149,3 +150,38 @@ it("returns focus to the editor when picking the default hides the chip", async await vi.waitFor(() => expect(handle.focusAtEnd).toHaveBeenCalled()); await expect.element(screen.getByRole("button", { name: "Model" })).not.toHaveFocus(); }); + +function ContextPermissions() { + const [mode, setMode] = useState("full-access"); + return ( + + ); +} + +it("opens permissions directly from the context strip and applies a supported mode", async () => { + await using screen = await render(, { router: false }); // no routes involved + await userEvent.click(screen.getByRole("button", { name: "Permissions: Full access" })); + await expect + .element(page.getByRole("menuitemradio", { name: "Full access" })) + .toHaveAttribute("aria-checked", "true"); + await expect + .element(page.getByRole("menuitemradio", { name: "Auto", exact: true })) + .not.toBeInTheDocument(); + await userEvent.click(page.getByRole("menuitemradio", { name: "Supervised" })); + const trigger = screen.getByRole("button", { name: "Permissions: Supervised" }); + await expect.element(trigger).toBeVisible(); + await expect + .element(page.getByRole("menuitemradio", { name: "Supervised" })) + .not.toBeInTheDocument(); + await userEvent.click(trigger); + await expect + .element(page.getByRole("menuitemradio", { name: "Supervised" })) + .toHaveAttribute("aria-checked", "true"); + await userEvent.keyboard("{Escape}"); + await expect.element(trigger).toHaveFocus(); +}); diff --git a/apps/web/src/components/chat/MarkdownThreadLink.tsx b/apps/web/src/components/chat/MarkdownThreadLink.tsx index ae67b24ca2a7c..00382ce69a1f0 100644 --- a/apps/web/src/components/chat/MarkdownThreadLink.tsx +++ b/apps/web/src/components/chat/MarkdownThreadLink.tsx @@ -1,7 +1,7 @@ import { scopeProjectRef, scopeThreadRef } from "@t3tools/client-runtime/environment"; import type { EnvironmentId } from "@t3tools/contracts"; import { - formatThreadLink, + formatHostedThreadLink, guessThreadLinkRef, percentDecodedThreadLinkId, type ThreadLink, @@ -11,6 +11,7 @@ import { Link } from "@tanstack/react-router"; import { MessageSquareTextIcon } from "lucide-react"; import { useProject, useThreadRefs, useThreadShell } from "../../state/entities"; +import { configuredHostedAppUrl } from "../../hostedPairing"; import { ProjectFavicon } from "../ProjectFavicon"; import { ContextPreviewHover } from "../contextPreviewRegistry"; @@ -47,7 +48,11 @@ export function MarkdownThreadLink(props: { { { container }, ); const heading = screen.getByRole("button", { name: /^Worked for/ }); - await expect.element(heading).toBeVisible(); + // LegendList keeps rows click-through until its initial scroll settles on animation + // frames, which the fake clock below would freeze. + await expect.element(heading).toHaveStyle({ pointerEvents: "auto" }); const viewport = props.listRef.current!.getScrollableNode()!; await vi.waitFor(() => expect(viewport.scrollHeight - viewport.clientHeight - viewport.scrollTop).toBeLessThan(2), diff --git a/apps/web/src/components/chat/ModelListRow.tsx b/apps/web/src/components/chat/ModelListRow.tsx index a12c7b832aa1e..5a7d6cd858949 100644 --- a/apps/web/src/components/chat/ModelListRow.tsx +++ b/apps/web/src/components/chat/ModelListRow.tsx @@ -77,6 +77,7 @@ export const ModelListRow = memo(function ModelListRow(props: { {/* The logo is decorative, so name the provider for screen readers. */} , {props.providerDisplayName}
+ {props.isSelected ?