fix(server): isolate provider model refresh failures

apps/server/src/provider/ProviderRegistry.test.ts

@@ -76,6 +76,7 @@ import * as ProviderInstanceRegistryMutator from "./ProviderInstanceRegistryMuta
 import * as ProviderRegistry from "./ProviderRegistry.ts";
 import { mergeProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment";
 import { makeManualOnlyProviderMaintenanceCapabilities } from "@t3tools/provider-core/server/maintenanceResolver";
+import { ProviderDriverError } from "@t3tools/provider-core/server/errors";
 vi.mock("@anthropic-ai/claude-agent-sdk", { spy: true });
 
 const decodeServerProvider = Schema.decodeSync(ServerProvider);
@@ -1489,6 +1490,167 @@ it.layer(
       );
     });
 
+    it.effect.each(["all", "single", "disabled", "uninstalled", "non-refreshable"] as const)(
+      "%s model refresh preserves cached models and skips ineligible instances",
+      (target) =>
+        Effect.gen(function* () {
+          const driver = ProviderDriverKind.make("antigravity");
+          const firstId = ProviderInstanceId.make("antigravity_first");
+          const secondId = ProviderInstanceId.make("antigravity_second");
+          const thirdId = ProviderInstanceId.make("antigravity_third");
+          const skippedIds = {
+            disabled: ProviderInstanceId.make("antigravity_disabled"),
+            uninstalled: ProviderInstanceId.make("antigravity_uninstalled"),
+            "non-refreshable": ProviderInstanceId.make("antigravity_static"),
+          };
+          const driverError = new ProviderDriverError({
+            driver,
+            instanceId: firstId,
+            detail: "Model refresh failed.",
+            cause: new Error("native refresh failure"),
+          });
+          const thirdStarted = yield* Deferred.make<void>();
+          const calls = yield* Ref.make<ReadonlyArray<ProviderInstanceId>>([]);
+          const activeRefreshes = yield* Ref.make(0);
+          const maxRefreshes = yield* Ref.make(0);
+          const model = (slug: string) => ({
+            slug,
+            name: slug,
+            isCustom: false,
+            capabilities: null,
+          });
+          const makeInstance = Effect.fn("Test.makeModelRefreshInstance")(function* (
+            instanceId: ProviderInstanceId,
+            options: { enabled?: boolean; installed?: boolean; refreshable?: boolean } = {},
+          ) {
+            const snapshot = yield* Ref.make<ServerProvider>({
+              instanceId,
+              driver,
+              enabled: options.enabled ?? true,
+              installed: options.installed ?? true,
+              version: "1.1.1",
+              status: "ready",
+              auth: { status: "authenticated" },
+              checkedAt: "2026-10-25T00:00:00.000Z",
+              models: [model("cached-model")],
+              slashCommands: [],
+              skills: [],
+            });
+            return {
+              instanceId,
+              driverKind: driver,
+              continuationIdentity: { driverKind: driver, continuationKey: instanceId },
+              displayName: undefined,
+              enabled: options.enabled ?? true,
+              snapshot: {
+                getSnapshot: Ref.get(snapshot),
+                refresh: Ref.get(snapshot),
+                resolveMaintenance: () =>
+                  Effect.succeed(
+                    makeManualOnlyProviderMaintenanceCapabilities({
+                      provider: driver,
+                      packageName: null,
+                    }),
+                  ),
+                streamChanges: Stream.empty,
+                applyUsageLimits: () => Effect.void,
+              },
+              ...(options.refreshable === false
+                ? {}
+                : {
+                    refreshModels: () =>
+                      Effect.gen(function* () {
+                        yield* Ref.update(calls, (ids) => [...ids, instanceId]);
+                        if (instanceId === firstId) {
+                          return yield* driverError;
+                        }
+                        const active = yield* Ref.updateAndGet(
+                          activeRefreshes,
+                          (count) => count + 1,
+                        );
+                        yield* Ref.update(maxRefreshes, (count) => Math.max(count, active));
+                        yield* instanceId === secondId
+                          ? Deferred.await(thirdStarted)
+                          : Deferred.succeed(thirdStarted, undefined);
+                        yield* Ref.update(snapshot, (provider) => ({
+                          ...provider,
+                          models: [model("refreshed-model")],
+                        }));
+                        yield* Ref.update(activeRefreshes, (count) => count - 1);
+                      }),
+                  }),
+              orchestrationAdapter: {} as ProviderInstance["orchestrationAdapter"],
+              textGeneration: {} as ProviderInstance["textGeneration"],
+            } satisfies ProviderInstance;
+          });
+          const instances = [
+            yield* makeInstance(firstId),
+            yield* makeInstance(secondId),
+            yield* makeInstance(thirdId),
+            yield* makeInstance(skippedIds.disabled, { enabled: false }),
+            yield* makeInstance(skippedIds.uninstalled, { installed: false }),
+            yield* makeInstance(skippedIds["non-refreshable"], { refreshable: false }),
+          ];
+          const instanceLayer = Layer.succeed(ProviderInstanceRegistry.ProviderInstanceRegistry, {
+            listInstances: Effect.succeed(instances),
+            getInstance: (id) =>
+              Effect.succeed(instances.find((instance) => instance.instanceId === id)),
+            listUnavailable: Effect.succeed([]),
+            streamChanges: Stream.empty,
+            subscribeChanges: Effect.flatMap(PubSub.unbounded<void>(), PubSub.subscribe),
+          });
+          yield* Effect.gen(function* () {
+            const registry = yield* ProviderRegistry.ProviderRegistry;
+            if (target === "single") {
+              const failure = yield* registry.refreshModels(firstId).pipe(Effect.flip);
+              assert.equal(failure.instanceId, firstId);
+              assert.equal(failure.operation, "refresh-models");
+              assert.equal(failure.detail, driverError.detail);
+              assert.strictEqual(failure.cause, driverError);
+            } else {
+              yield* registry.refreshModels(target === "all" ? undefined : skippedIds[target]);
+            }
+            assert.sameMembers(
+              [...(yield* Ref.get(calls))],
+              target === "all"
+                ? [firstId, secondId, thirdId]
+                : target === "single"
+                  ? [firstId]
+                  : [],
+            );
+            assert.equal(yield* Ref.get(maxRefreshes), target === "all" ? 2 : 0);
+            assert.deepEqual(
+              Object.fromEntries(
+                (yield* registry.getProviders).map((provider) => [
+                  provider.instanceId,
+                  provider.models.map((entry) => entry.slug),
+                ]),
+              ),
+              {
+                [firstId]: ["cached-model"],
+                [secondId]: [target === "all" ? "refreshed-model" : "cached-model"],
+                [thirdId]: [target === "all" ? "refreshed-model" : "cached-model"],
+                [skippedIds.disabled]: ["cached-model"],
+                [skippedIds.uninstalled]: ["cached-model"],
+                [skippedIds["non-refreshable"]]: ["cached-model"],
+              },
+            );
+          }).pipe(
+            Effect.provide(
+              ProviderRegistry.layer.pipe(
+                Layer.provide(instanceLayer),
+                Layer.provide(layerUnusedMutator),
+                Layer.provide(
+                  ServerConfig.layerTest(process.cwd(), {
+                    prefix: "t3-provider-registry-model-refresh-",
+                  }),
+                ),
+              ),
+            ),
+          );
+        }),
+    );
+
     it.effect("invalidates model caches only for an explicit model refresh", () =>
       Effect.gen(function* () {
         const forceRefreshCalls = yield* Ref.make(0);

apps/server/src/provider/ProviderRegistry.ts

@@ -2,6 +2,7 @@ import {
   defaultInstanceIdForDriver,
   isProviderWorkspaceSnapshotCurrent,
   ProviderDriverKind,
+  ProviderSetupError,
   type ProviderInstanceId,
   type ServerProvider,
   type ServerProviderUpdateState,
@@ -64,6 +65,10 @@ export class ProviderRegistry extends Context.Service<
       instanceId: ProviderInstanceId,
     ) => Effect.Effect<ReadonlyArray<ServerProvider>>;
 
+    readonly refreshModels: (
+      instanceId?: ProviderInstanceId,
+    ) => Effect.Effect<ReadonlyArray<ServerProvider>, ProviderSetupError>;
+
     /** Re-reads secret-backed environments first, so a fixed secret applies without a restart. */
     readonly reloadSecretsAndRefresh: (
       instanceId?: ProviderInstanceId,
@@ -685,6 +690,58 @@ export const layer = Layer.effect(
       return yield* refreshOneSource(instance);
     });
 
+    const refreshModels = Effect.fn("ProviderRegistry.refreshModels")(function* (
+      instanceId?: ProviderInstanceId,
+    ) {
+      yield* syncLiveSourcesAndContinue;
+      const providers = yield* Ref.get(providersRef);
+      const instances = yield* instanceRegistry.listInstances;
+      yield* Effect.forEach(
+        instances,
+        (instance) => {
+          if (
+            !instance.refreshModels ||
+            (instanceId !== undefined && instanceId !== instance.instanceId) ||
+            !providers.some(
+              (provider) =>
+                provider.instanceId === instance.instanceId &&
+                provider.enabled &&
+                provider.installed,
+            )
+          ) {
+            return Effect.void;
+          }
+          const refresh = instance.refreshModels().pipe(
+            Effect.mapError(
+              (error) =>
+                new ProviderSetupError({
+                  instanceId: instance.instanceId,
+                  operation: "refresh-models",
+                  detail: error.detail,
+                  cause: error,
+                }),
+            ),
+            Effect.andThen(
+              refreshInstance(instance.instanceId).pipe(Effect.catchCause(recoverRefreshFailure)),
+            ),
+            Effect.asVoid,
+          );
+          return instanceId === undefined
+            ? refresh.pipe(
+                Effect.catch((error) =>
+                  Effect.logWarning("provider model refresh failed", {
+                    instanceId: instance.instanceId,
+                    detail: error.detail,
+                  }),
+                ),
+              )
+            : refresh;
+        },
+        { concurrency: 2, discard: true },
+      );
+      return yield* Ref.get(providersRef);
+    });
+
     const reloadSecretsAndRefresh = Effect.fn("reloadSecretsAndRefresh")(function* (
       instanceId?: ProviderInstanceId,
     ) {
@@ -1080,6 +1137,7 @@ export const layer = Layer.effect(
       refreshWorkspaceSnapshot: (input) =>
         refreshWorkspaceSnapshot(input).pipe(Effect.catchCause(recoverRefreshFailure)),
       refreshAfterUpdate,
+      refreshModels,
       invalidateModelCaches,
       getProviderMaintenanceCapabilitiesForInstance,
       setProviderMaintenanceActionState,

apps/server/src/provider/providerMaintenanceRunner.test.ts

@@ -197,6 +197,7 @@ function makeRegistry(
       getProviders: Ref.get(providersRef),
       refresh: () => Ref.get(providersRef),
       refreshInstance: () => Ref.get(providersRef),
+      refreshModels: () => Ref.get(providersRef),
       reloadSecretsAndRefresh: () => Ref.get(providersRef),
       refreshAfterUpdate: () => {
         const version = options?.installedVersionAfterUpdate;

apps/server/src/provider/testUtils/providerRegistryMock.ts

@@ -11,6 +11,7 @@ const makeProviderRegistryMock = (
   getProviders: Effect.succeed(providers),
   refresh: () => Effect.succeed(providers),
   refreshInstance: () => Effect.succeed(providers),
+  refreshModels: () => Effect.succeed(providers),
   reloadSecretsAndRefresh: () => Effect.succeed(providers),
   refreshAfterUpdate: () => Effect.succeed(providers),
   refreshWorkspaceSnapshot: () => Effect.succeed(providers),