Authorize from the mirror unconditionally; alert on a stale reconciler

Author: RhysSullivan

.changeset/mirror-readiness-removal.md

@@ -0,0 +1,5 @@
+---
+"@executor-js/cloud": patch
+---
+
+`authorizeOrganization` now reads the local membership mirror unconditionally: the per-request readiness check (`MirrorReadiness`) and its live WorkOS `listUserMemberships` fallback are gone from the request path entirely. The backfill is complete and permanent, and an organization that predates the mirror is still covered by the existing on-demand scan (`ensureOrganizationBackfilled`). A stalled reconciler is now an operational alert instead of a per-request fallback: after each run, the cron checks the mirror's `drained_at` heartbeat and, if it has fallen behind the lag budget, logs a structured error and reports it to Sentry. The deploy gate (`scripts/ensure-workos-mirror-ready.ts`) is unchanged — it still refuses to ship while the mirror is unready — and `drained_at` keeps being written by every reconciler run.

.claude/skills/prod-telemetry/SKILL.md

@@ -57,22 +57,33 @@ join the same traces via traceparent).
   `execute`/`execute-action` calls `mcp.execute.code` (the script itself,
   capped at 10k chars — cloud-only content capture; local/self-host
   telemetry never records content).
-- `auth.authorize_organization` — every membership authorization.
-  `mirror.ready` (bool: the local membership mirror answered; `false` =
-  the request fell back to a live WorkOS read) and `mirror.readiness`
-  (why: `ready`, `backfill pending: …`, `reconciler stale: …`). The
-  mirror's write spans are `workos_mirror.<op>`; the reconciler run is
+- `auth.authorize_organization` — every membership authorization. Reads the
+  local membership mirror unconditionally; there is no per-request readiness
+  check and no WorkOS fallback, so this span carries no readiness attribute.
+  The mirror's write spans are `workos_mirror.<op>`; the reconciler run is
   `workos_events.sync`. `workos_sync.drained_at` in the prod DB is the
-  reconciler heartbeat.
+  reconciler heartbeat, and a stalled reconciler now raises its own error
+  from the cron (see below) rather than showing up as a fallback here.
 
-**Recipe — membership-mirror fallback rate (should be ~0 after cutover):**
+**Recipe — reconciler heartbeat (ticks should land roughly every minute; a
+gap wider than the 10-minute lag budget means the cron alert should already
+have fired — see `workos_events: reconciler stale` below):**
 
 ```apl
 ['executor-cloud']
-| where _time > ago(1h) and name == "auth.authorize_organization"
-| extend ready = tobool(['attributes.custom']['mirror.ready'])
-| extend why = tostring(['attributes.custom']['mirror.readiness'])
-| summarize n = count() by ready, why
+| where _time > ago(1h) and name == "workos_events.sync"
+| summarize n = count() by bin(_time, 1m)
+| sort by _time desc
+```
+
+**Recipe — stale-reconciler alerts (should be empty; each row is one paging
+event):**
+
+```apl
+['executor-cloud']
+| where _time > ago(1d) and ['status.message'] contains "workos_events: reconciler stale"
+| project _time, trace_id, msg = tostring(['status.message'])
+| sort by _time desc
 ```
 
 **Recipe — error signatures by class (the daily-digest query):**

apps/cloud/src/account/account-api.ts

@@ -10,7 +10,6 @@ import {
 
 import { ApiKeyService } from "../auth/api-keys";
 import { UserStoreService } from "../auth/context";
-import { MirrorReadiness } from "../auth/mirror-readiness";
 import { WorkOsMirror } from "../auth/workos-mirror";
 import { sessionFromSealed, type Session } from "../auth/middleware";
 import { WorkOSClient } from "../auth/workos";
@@ -51,7 +50,9 @@ import { AccountCaller, workosAccountProvider } from "./workos-account-service";
 // `UserStoreService` / `WorkOsMirror` / `MemberDirectory` are supplied by the
 // combined `rsLive` layer.
 // `ApiKeyService.WorkOS` is built here on top of the boot `WorkOSClient`.
-const AccountProviderMiddleware = HttpRouter.middleware<{ provides: AccountProvider }>()(
+const AccountProviderMiddleware = HttpRouter.middleware<{
+  provides: AccountProvider;
+}>()(
   Effect.gen(function* () {
     // Long-lived services only (built once at boot). `UserStoreService` and
     // `DbService` are NOT grabbed here — they come per request from the combined
@@ -100,15 +101,11 @@ const AccountProviderMiddleware = HttpRouter.middleware<{ provides: AccountProvi
  * (the seat-gate) stays a residual requirement, satisfied by the app `boot`.
  */
 export const workosAccountMiddleware = (
-  rsLive: Layer.Layer<
-    DbService | UserStoreService | WorkOsMirror | MemberDirectory | MirrorReadiness
-  >,
+  rsLive: Layer.Layer<DbService | UserStoreService | WorkOsMirror | MemberDirectory>,
 ) => AccountProviderMiddleware.combine(requestScopedMiddleware(rsLive)).layer;
 
 export const makeAccountApiLive = (
-  rsLive: Layer.Layer<
-    DbService | UserStoreService | WorkOsMirror | MemberDirectory | MirrorReadiness
-  >,
+  rsLive: Layer.Layer<DbService | UserStoreService | WorkOsMirror | MemberDirectory>,
 ) => {
   // Cloud builds the WorkOS `AccountProvider` INSIDE the request body (so it
   // closes over the per-request postgres socket), so it can't be a self-

apps/cloud/src/account/org-api-key-revoke.node.test.ts

@@ -6,7 +6,6 @@ import { AccountError, AccountForbidden } from "@executor-js/api";
 
 import { ApiKeyService, OrgApiKeyNotFound } from "../auth/api-keys";
 import { UserStoreService } from "../auth/context";
-import { MirrorReadiness, MirrorReadinessState } from "../auth/mirror-readiness";
 import { ORG_SELECTOR_HEADER } from "../auth/organization";
 import { WorkOSClient, type WorkOSClientService } from "../auth/workos";
 import { WorkOsMirror } from "../auth/workos-mirror";
@@ -132,12 +131,6 @@ const stubMirror = Layer.succeed(WorkOsMirror)({
 // The mirror as the directory reads it: both are active members of ORG, and
 // only ADMIN carries the `admin` role. Revoke reads the caller's membership
 // (the org check and the admin gate) and nothing else.
-// The mirror is READY in these tests (backfill complete, reconciler caught
-// up), so membership is read from the stubbed directory, never from WorkOS.
-const stubReadiness = Layer.succeed(MirrorReadiness)({
-  state: () => Effect.succeed(MirrorReadinessState.Ready()),
-});
-
 const stubDirectory = Layer.succeed(MemberDirectory)({
   membership: (accountId, organizationId) =>
     Effect.succeed(
@@ -201,7 +194,6 @@ const providerWith = (accountId: string) => {
             stubUsers,
             stubMirror,
             stubDirectory,
-            stubReadiness,
             stubApiKeys,
             stubAutumn,
             Layer.succeed(AccountCaller)({ session: session(accountId) }),

apps/cloud/src/account/workos-account-service.ts

@@ -11,7 +11,6 @@ import {
 import { ApiKeyService } from "../auth/api-keys";
 import { UserStoreService } from "../auth/context";
 import type { Session } from "../auth/middleware";
-import { MirrorReadiness } from "../auth/mirror-readiness";
 import { WorkOSClient } from "../auth/workos";
 import { ensureOrganizationBackfilled, mirrorInvitedMember } from "../auth/mirror-feeders";
 import { WorkOsMirror, mirrorMembershipFromWorkOs } from "../auth/workos-mirror";
@@ -72,7 +71,6 @@ export const workosAccountProvider: Layer.Layer<
   | UserStoreService
   | WorkOsMirror
   | MemberDirectory
-  | MirrorReadiness
   | ApiKeyService
   | AutumnService
   | AccountCaller
@@ -103,12 +101,7 @@ export const workosAccountProvider: Layer.Layer<
     // erased to `R = never`, as the neutral AccountProvider shape requires.
     // Provided per method below.
     const ctx = yield* Effect.context<
-      | WorkOSClient
-      | UserStoreService
-      | AutumnService
-      | MemberDirectory
-      | MirrorReadiness
-      | WorkOsMirror
+      WorkOSClient | UserStoreService | AutumnService | MemberDirectory | WorkOsMirror
     >();
 
     // Unauthenticated (missing/invalid session) => AccountUnauthorized, exactly

apps/cloud/src/admin/admin-users-api.node.test.ts

@@ -6,7 +6,6 @@ import { MemberDirectory, type DirectoryMember } from "@executor-js/api/server";
 
 import { ApiKeyService } from "../auth/api-keys";
 import { UserStoreService } from "../auth/context";
-import { MirrorReadiness, MirrorReadinessState } from "../auth/mirror-readiness";
 import { ORG_SELECTOR_HEADER } from "../auth/organization";
 import { WorkOSClient, type WorkOSClientService } from "../auth/workos";
 import { WorkOsMirror, type WorkOsMirrorShape } from "../auth/workos-mirror";
@@ -49,12 +48,6 @@ const memberships = new Map<string, DirectoryMember>([
   ["user_invited_admin", mirrored("user_invited_admin", { role: "admin", status: "pending" })],
 ]);
 
-// The mirror is READY in these tests (backfill complete, reconciler caught
-// up), so membership is read from the stubbed directory, never from WorkOS.
-const stubReadiness = Layer.succeed(MirrorReadiness)({
-  state: () => Effect.succeed(MirrorReadinessState.Ready()),
-});
-
 const stubDirectory = Layer.succeed(MemberDirectory)({
   membership: (accountId, organizationId) =>
     Effect.succeed(organizationId === ORG ? (memberships.get(accountId) ?? null) : null),
@@ -149,7 +142,11 @@ const stubWorkOS = (userId: string) =>
       get: (_target, prop) => {
         if (prop === "authenticateRequest") {
           return () =>
-            Effect.succeed({ userId, email: `${userId}@placeholder.test`, organizationId: null });
+            Effect.succeed({
+              userId,
+              email: `${userId}@placeholder.test`,
+              organizationId: null,
+            });
         }
         return () => Effect.die(`unexpected WorkOSClient.${String(prop)} call`);
       },
@@ -163,14 +160,7 @@ const authorizeAs = (userId: string) =>
     }),
   ).pipe(
     Effect.provide(
-      Layer.mergeAll(
-        stubDirectory,
-        stubApiKeys,
-        stubUsers,
-        stubWorkOS(userId),
-        stubMirror,
-        stubReadiness,
-      ),
+      Layer.mergeAll(stubDirectory, stubApiKeys, stubUsers, stubWorkOS(userId), stubMirror),
     ),
   );
 

apps/cloud/src/admin/admin-users-api.ts

@@ -56,7 +56,6 @@ import type { Executor } from "@executor-js/sdk";
 
 import { ApiKeyService } from "../auth/api-keys";
 import { UserStoreService } from "../auth/context";
-import { MirrorReadiness } from "../auth/mirror-readiness";
 import { WorkOsMirror } from "../auth/workos-mirror";
 import { isPlatformAuth, resolveBearerAuth } from "../auth/workos-auth-provider";
 import { orgSelectorFromRequest, authorizeOrganizationSelector } from "../auth/organization";
@@ -77,7 +76,7 @@ export const authorizeTenant = (
 ): Effect.Effect<
   string,
   AdminUsersUnauthorized | AdminUsersForbidden,
-  WorkOSClient | ApiKeyService | UserStoreService | MemberDirectory | MirrorReadiness | WorkOsMirror
+  WorkOSClient | ApiKeyService | UserStoreService | MemberDirectory | WorkOsMirror
 > =>
   Effect.gen(function* () {
     // (1) The bearer path. `resolveBearerAuth` (not `resolveApiKeyPrincipal`,
@@ -138,7 +137,6 @@ const withPlatformView = <A, E extends AdminUsersError | AdminUserNotFound = Adm
   | ApiKeyService
   | UserStoreService
   | MemberDirectory
-  | MirrorReadiness
   | WorkOsMirror
   | DbProvider
   | PluginsProvider
@@ -178,7 +176,6 @@ export const workosAdminUsersProvider: Layer.Layer<
   | ApiKeyService
   | UserStoreService
   | MemberDirectory
-  | MirrorReadiness
   | WorkOsMirror
   | DbProvider
   | PluginsProvider
@@ -190,7 +187,6 @@ export const workosAdminUsersProvider: Layer.Layer<
       | ApiKeyService
       | UserStoreService
       | MemberDirectory
-      | MirrorReadiness
       | WorkOsMirror
       | DbProvider
       | PluginsProvider
@@ -241,7 +237,9 @@ export const workosAdminUsersProvider: Layer.Layer<
 // per-request `DbService`/`UserStoreService`/`MemberDirectory` (and the
 // execution seams built over them) are supplied by the combined
 // `requestScopedMiddleware`.
-const AdminUsersProviderMiddleware = HttpRouter.middleware<{ provides: AdminUsersProvider }>()(
+const AdminUsersProviderMiddleware = HttpRouter.middleware<{
+  provides: AdminUsersProvider;
+}>()(
   Effect.gen(function* () {
     const longLived = yield* Effect.context<WorkOSClient | ApiKeyService>();
     return (httpEffect) =>
@@ -265,9 +263,7 @@ const AdminUsersProviderMiddleware = HttpRouter.middleware<{ provides: AdminUser
  * `/api` prefix as the rest of the cloud router.
  */
 export const makeCloudAdminUsersRoutes = (
-  rsLive: Layer.Layer<
-    DbService | UserStoreService | MemberDirectory | MirrorReadiness | WorkOsMirror
-  >,
+  rsLive: Layer.Layer<DbService | UserStoreService | MemberDirectory | WorkOsMirror>,
   options: Parameters<typeof makeAdminUsersApiLayer>[1] = {},
 ) =>
   makeAdminUsersApiLayer(

apps/cloud/src/api/layers.ts

@@ -11,7 +11,6 @@ import {
 import { SessionAuthLive } from "../auth/middleware-live";
 import { UserStoreService } from "../auth/context";
 import { cloudMemberDirectoryLayer } from "../auth/member-directory";
-import { MirrorReadiness } from "../auth/mirror-readiness";
 import { WorkOsMirror } from "../auth/workos-mirror";
 import {
   CloudAuthPublicHandlers,
@@ -36,23 +35,14 @@ const WorkOsMirrorLive = WorkOsMirror.Live.pipe(Layer.provide(DbLive));
 // The shared `MemberDirectory` read seam over the membership mirror — the
 // same per-request socket the mirror writes through.
 const MemberDirectoryLive = cloudMemberDirectoryLayer.pipe(Layer.provide(DbLive));
-// Whether the mirror may authorize this request at all (backfill complete,
-// reconciler caught up) — read on the same socket before the membership row.
-const MirrorReadinessLive = MirrorReadiness.Live.pipe(Layer.provide(DbLive));
 
 // Per-request layer. Anything that opens an I/O object (postgres.js socket,
 // fetch stream readers, anything backed by a `Writable`) MUST live here —
 // `provideRequestScoped` rebuilds it per request so Cloudflare Workers'
 // I/O isolation is satisfied. See `api.request-scope.test.ts`.
 export const RequestScopedServicesLive: Layer.Layer<
-  DbService | UserStoreService | WorkOsMirror | MemberDirectory | MirrorReadiness
-> = Layer.mergeAll(
-  DbLive,
-  UserStoreLive,
-  WorkOsMirrorLive,
-  MemberDirectoryLive,
-  MirrorReadinessLive,
-);
+  DbService | UserStoreService | WorkOsMirror | MemberDirectory
+> = Layer.mergeAll(DbLive, UserStoreLive, WorkOsMirrorLive, MemberDirectoryLive);
 
 // Boot-scoped layer. Built once at worker boot, reused across requests.
 // Safe for config, in-memory caches, the global tracer provider, and
@@ -77,9 +67,7 @@ export const BootSharedServices = Layer.mergeAll(
 // handler reads it for the free-organizations-per-user limit gate — one of the
 // few app-only billing touchpoints. (It is NOT on the neutral boot core.)
 export const makeNonProtectedApiLive = (
-  rsLive: Layer.Layer<
-    DbService | UserStoreService | WorkOsMirror | MemberDirectory | MirrorReadiness
-  >,
+  rsLive: Layer.Layer<DbService | UserStoreService | WorkOsMirror | MemberDirectory>,
 ) =>
   HttpApiBuilder.layer(NonProtectedApi).pipe(
     Layer.provide(Layer.mergeAll(CloudAuthPublicHandlers, CloudSessionAuthHandlers)),
@@ -95,9 +83,7 @@ export const makeNonProtectedApiLive = (
 // gates on billing, so `AutumnService.Default` is provided here, not on the
 // neutral boot core.
 export const makeOrgApiLive = (
-  rsLive: Layer.Layer<
-    DbService | UserStoreService | MemberDirectory | MirrorReadiness | WorkOsMirror
-  >,
+  rsLive: Layer.Layer<DbService | UserStoreService | MemberDirectory | WorkOsMirror>,
 ) =>
   HttpApiBuilder.layer(OrgHttpApi).pipe(
     Layer.provide(OrgHandlers),
@@ -143,7 +129,9 @@ export const OrgApiLive = makeOrgApiLive(RequestScopedServicesLive);
 // folded into `.layer` here; the rest of the router (`makeApiLive` in
 // `./router.ts`, `./protected.ts`, the test harness) re-provides the same
 // shared `RouterConfigLive` directly.
-const protectedApi = makeProtectedApiLayer(cloudPlugins, { errorCapture: ErrorCaptureLive });
+const protectedApi = makeProtectedApiLayer(cloudPlugins, {
+  errorCapture: ErrorCaptureLive,
+});
 
 export const ProtectedCloudApi = protectedApi.api;
 export const ProtectedCloudApiHandlers = protectedApi.handlers;

apps/cloud/src/api/protected-api-key-auth.node.test.ts

@@ -5,7 +5,6 @@ import { MemberDirectory } from "@executor-js/api/server";
 
 import { ApiKeyService } from "../auth/api-keys";
 import { UserStoreService } from "../auth/context";
-import { MirrorReadiness, MirrorReadinessState } from "../auth/mirror-readiness";
 import { WorkOSClient, type WorkOSClientService } from "../auth/workos";
 import { WorkOsMirror, type WorkOsMirrorShape } from "../auth/workos-mirror";
 import { resolveProtectedPrincipal } from "./protected";
@@ -55,13 +54,8 @@ const stubWorkOS = Layer.succeed(
 );
 
 // The mirror as the directory reads it: user_123 holds an active membership in
-// org_123 and nothing else. Membership is never read from WorkOS.
-// The mirror is READY in these tests (backfill complete, reconciler caught
-// up), so membership is read from the stubbed directory, never from WorkOS.
-const stubReadiness = Layer.succeed(MirrorReadiness)({
-  state: () => Effect.succeed(MirrorReadinessState.Ready()),
-});
-
+// org_123 and nothing else. Membership is always read from the mirror, never
+// from WorkOS.
 const stubDirectory = Layer.succeed(MemberDirectory)({
   membership: (accountId, organizationId) =>
     Effect.succeed(
@@ -136,9 +130,7 @@ const stubMirror = Layer.succeed(
 
 const run = (request: Request) =>
   resolveProtectedPrincipal(request).pipe(
-    Effect.provide(
-      Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers, stubDirectory, stubMirror, stubReadiness),
-    ),
+    Effect.provide(Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers, stubDirectory, stubMirror)),
   );
 
 describe("protected API key auth", () => {

apps/cloud/src/api/protected-jwt-auth.node.test.ts

@@ -6,7 +6,6 @@ import { MemberDirectory } from "@executor-js/api/server";
 
 import { ApiKeyService } from "../auth/api-keys";
 import { UserStoreService } from "../auth/context";
-import { MirrorReadiness, MirrorReadinessState } from "../auth/mirror-readiness";
 import type { JwtBearerConfig } from "../auth/workos-auth-provider";
 import { WorkOSClient, type WorkOSClientService } from "../auth/workos";
 import { WorkOsMirror, type WorkOsMirrorShape } from "../auth/workos-mirror";
@@ -72,13 +71,8 @@ const stubWorkOS = Layer.succeed(
 );
 
 // The mirror as the directory reads it: user_123 holds an active membership in
-// org_123 and nothing else. Membership is never read from WorkOS.
-// The mirror is READY in these tests (backfill complete, reconciler caught
-// up), so membership is read from the stubbed directory, never from WorkOS.
-const stubReadiness = Layer.succeed(MirrorReadiness)({
-  state: () => Effect.succeed(MirrorReadinessState.Ready()),
-});
-
+// org_123 and nothing else. Membership is always read from the mirror, never
+// from WorkOS.
 const stubDirectory = Layer.succeed(MemberDirectory)({
   membership: (accountId, organizationId) =>
     Effect.succeed(
@@ -153,9 +147,7 @@ const stubMirror = Layer.succeed(
 
 const run = (request: Request, jwt: JwtBearerConfig) =>
   resolveProtectedPrincipal(request, jwt).pipe(
-    Effect.provide(
-      Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers, stubDirectory, stubMirror, stubReadiness),
-    ),
+    Effect.provide(Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers, stubDirectory, stubMirror)),
   );
 
 const request = (token: string) =>