From f112f1f305f3715262c01484787179b531513cbf Mon Sep 17 00:00:00 2001 From: Jason-jo17 Date: Mon, 13 Jul 2026 16:42:06 +0530 Subject: [PATCH 1/2] =?UTF-8?q?feat(drive):=20Google=20Drive=20folder=20sy?= =?UTF-8?q?nc=20=E2=80=94=20backend=20(Phase=2010)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bring-your-own-OAuth Google Drive integration, modelled on the existing Google Calendar link (lib/google-calendar.ts, secret-box, the workspace_members refresh-token column) but wired to core tables (folders/docs/attachments) — no enterprise coupling, so verify-core-only stays green. - schema + migration 0073: workspace_members.drive_refresh_token (encrypted) plus drive_folder_links (Mnema⇄Drive folder pairing, direction, accepted types, conflict policy) and drive_file_mappings (per-file idempotency + conflict tracking). Idempotent SQL, applied by the psql migrate service. - lib/google-drive.ts: OAuth (offline refresh token) + Drive REST via the googleapis SDK; configurable scope (drive.file default, opt-in drive). - lib/drive-sync.ts: pull (Drive→Mnema: text→docs via markdownToYjsState, binary→attachments+R2) and push (Mnema→Drive: docs→.md), keyed on md5/content-hash so re-runs never duplicate; manual conflict policy never clobbers un-pushed local edits. - routes/drive.ts: connect/callback/status, folder picker, links CRUD, "sync now", conflicts, and a Google push webhook (auth-exempted, matched by stored channel id). - queue/drive-sync.ts + workers/drive-sync worker (registered in workers/server.ts); route registered in server.ts. - env: GOOGLE_DRIVE_CLIENT_ID/SECRET/REDIRECT_URI, GOOGLE_DRIVE_SCOPE, DRIVE_SYNC_MAX_FILE_MB, DRIVE_WEBHOOK_SECRET — all optional; routes 503 until configured. - dep: googleapis (discussed — Drive changes/resumable-upload surface). @boppl/api typechecks clean. Live end-to-end needs an operator's Google Cloud OAuth app + running stack (see docs/connect/drive.md). CI: registers migration 0073 in the drizzle journal so CI's `db:migrate` applies it (the psql migrate service already applies it on self-host) — without this the new `workspace_members.drive_refresh_token` column is absent in the CI DB and core inserts fail. Adds a `url-template -> BSD-3-Clause` license-gate override (transitive dep of googleapis; bundled LICENSE is 3-clause BSD, but package.json reports the legacy non-SPDX "BSD"). Co-Authored-By: Claude Opus 4.8 Signed-off-by: Jason-jo17 --- .../drizzle/migrations/0073_drive_sync.sql | 54 +++ .../api/drizzle/migrations/meta/_journal.json | 7 + apps/api/package.json | 11 +- apps/api/src/config/env.ts | 18 + apps/api/src/db/schema.ts | 74 ++++ apps/api/src/lib/drive-sync.ts | 289 ++++++++++++++ apps/api/src/lib/google-drive.ts | 249 +++++++++++++ apps/api/src/plugins/auth.ts | 3 + apps/api/src/queue/drive-sync.ts | 31 ++ apps/api/src/routes/drive.ts | 285 ++++++++++++++ apps/api/src/server.ts | 2 + apps/api/src/workers/drive-sync/worker.ts | 41 ++ apps/api/src/workers/server.ts | 6 + pnpm-lock.yaml | 352 +++++++++++++++++- scripts/check-licenses.mjs | 4 + 15 files changed, 1419 insertions(+), 7 deletions(-) create mode 100644 apps/api/drizzle/migrations/0073_drive_sync.sql create mode 100644 apps/api/src/lib/drive-sync.ts create mode 100644 apps/api/src/lib/google-drive.ts create mode 100644 apps/api/src/queue/drive-sync.ts create mode 100644 apps/api/src/routes/drive.ts create mode 100644 apps/api/src/workers/drive-sync/worker.ts diff --git a/apps/api/drizzle/migrations/0073_drive_sync.sql b/apps/api/drizzle/migrations/0073_drive_sync.sql new file mode 100644 index 000000000..e8b22f2e7 --- /dev/null +++ b/apps/api/drizzle/migrations/0073_drive_sync.sql @@ -0,0 +1,54 @@ +-- 0073 — Google Drive folder sync (Phase 10, core). +-- Adds a per-member encrypted Drive refresh token and two tables: +-- drive_folder_links — a Mnema folder ⇄ Google Drive folder pairing + settings +-- drive_file_mappings — per-file idempotency + conflict tracking +-- Core migration (folders/docs/attachments are core tables) — applies on self-host. +-- Idempotent (IF NOT EXISTS everywhere); a re-run on restart is safe. + +-- Per-member encrypted Google Drive refresh token (mirrors calendar_refresh_token). +ALTER TABLE "workspace_members" ADD COLUMN IF NOT EXISTS "drive_refresh_token" text; + +CREATE TABLE IF NOT EXISTS "drive_folder_links" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid(), + "workspace_id" uuid NOT NULL REFERENCES "workspaces"("id") ON DELETE CASCADE, + "folder_id" uuid NOT NULL REFERENCES "folders"("id") ON DELETE CASCADE, + "drive_folder_id" text NOT NULL, + "drive_folder_name" text, + "connected_by" uuid NOT NULL REFERENCES "users"("id") ON DELETE CASCADE, + "direction" text NOT NULL DEFAULT 'both', + "accepted_types" text[] NOT NULL DEFAULT ARRAY[]::text[], + "conflict_policy" text NOT NULL DEFAULT 'manual', + "drive_page_token" text, + "drive_channel_id" text, + "drive_channel_expires_at" timestamptz, + "last_synced_at" timestamptz, + "status" text NOT NULL DEFAULT 'active', + "error_message" text, + "created_at" timestamptz NOT NULL DEFAULT now(), + "updated_at" timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS "drive_links_workspace_idx" ON "drive_folder_links" ("workspace_id"); +CREATE UNIQUE INDEX IF NOT EXISTS "drive_links_folder_unique" ON "drive_folder_links" ("folder_id"); +CREATE UNIQUE INDEX IF NOT EXISTS "drive_links_ws_drivefolder_unique" ON "drive_folder_links" ("workspace_id", "drive_folder_id"); + +CREATE TABLE IF NOT EXISTS "drive_file_mappings" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid(), + "link_id" uuid NOT NULL REFERENCES "drive_folder_links"("id") ON DELETE CASCADE, + "drive_file_id" text NOT NULL, + "drive_name" text, + "doc_id" uuid REFERENCES "docs"("id") ON DELETE SET NULL, + "attachment_id" uuid REFERENCES "attachments"("id") ON DELETE SET NULL, + "drive_md5" text, + "content_hash" text, + "drive_modified_at" timestamptz, + "mnema_modified_at" timestamptz, + "sync_state" text NOT NULL DEFAULT 'synced', + "created_at" timestamptz NOT NULL DEFAULT now(), + "updated_at" timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS "drive_map_link_idx" ON "drive_file_mappings" ("link_id"); +CREATE UNIQUE INDEX IF NOT EXISTS "drive_map_link_file_unique" ON "drive_file_mappings" ("link_id", "drive_file_id"); +CREATE INDEX IF NOT EXISTS "drive_map_doc_idx" ON "drive_file_mappings" ("doc_id"); +CREATE INDEX IF NOT EXISTS "drive_map_attachment_idx" ON "drive_file_mappings" ("attachment_id"); diff --git a/apps/api/drizzle/migrations/meta/_journal.json b/apps/api/drizzle/migrations/meta/_journal.json index 993e7cd03..3eadcd6a3 100644 --- a/apps/api/drizzle/migrations/meta/_journal.json +++ b/apps/api/drizzle/migrations/meta/_journal.json @@ -505,6 +505,13 @@ "when": 1784900000000, "tag": "0070_task_comments", "breakpoints": true + }, + { + "idx": 72, + "version": "7", + "when": 1785000000000, + "tag": "0073_drive_sync", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/package.json b/apps/api/package.json index 7c703ac62..092aad4fc 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -50,12 +50,14 @@ "@workos-inc/node": "^10.7.0", "ai": "^6.0.221", "bullmq": "5.76.8", + "chart.js": "^4.4.7", "diff": "^9.0.0", "dompurify": "3.4.11", "dotenv": "^17.4.2", "drizzle-orm": "^0.45.2", "fastify": "^5.10.0", "fastify-plugin": "^5.0.0", + "googleapis": "^173.0.0", "gpt-tokenizer": "^3.4.0", "graphology": "^0.26.0", "graphology-communities-louvain": "^2.0.2", @@ -68,6 +70,7 @@ "juice": "^11.1.1", "mammoth": "^1.12.0", "marked": "^18.0.5", + "mdast2docx": "1.6.1", "mermaid": "11.16.0", "mime-types": "^3.0.2", "nanoid": "^5.1.16", @@ -77,16 +80,14 @@ "pino-pretty": "^13.1.3", "playwright": "^1.61.1", "postgres": "^3.4.0", + "remark-parse": "^11.0.0", "resend": "^6.17.1", "turndown": "^7.2.4", "turndown-plugin-gfm": "^1.0.2", + "unified": "^11.0.5", "voyageai": "^0.4.0", "yjs": "^13.6.31", - "zod": "^3.23.0", - "mdast2docx": "1.6.1", - "unified": "^11.0.5", - "remark-parse": "^11.0.0", - "chart.js": "^4.4.7" + "zod": "^3.23.0" }, "devDependencies": { "@types/node": "^22.20.1", diff --git a/apps/api/src/config/env.ts b/apps/api/src/config/env.ts index cb561fd7e..eb7eabb10 100644 --- a/apps/api/src/config/env.ts +++ b/apps/api/src/config/env.ts @@ -224,6 +224,24 @@ const envSchema = z.object({ GOOGLE_CALENDAR_CLIENT_SECRET: z.string().min(1).optional(), GOOGLE_CALENDAR_REDIRECT_URI: z.string().url().optional(), + // Phase 10 — Google Drive folder sync (optional; drive routes return 503 when + // unset). Bring-your-own OAuth client, same shape as calendar above. The + // redirect URI must equal the /api/drive/callback URL and be registered as an + // authorised redirect URI in Google Cloud Console. + GOOGLE_DRIVE_CLIENT_ID: z.string().min(1).optional(), + GOOGLE_DRIVE_CLIENT_SECRET: z.string().min(1).optional(), + GOOGLE_DRIVE_REDIRECT_URI: z.string().url().optional(), + // 'drive.file' (least privilege — Mnema only sees files it created or the user + // explicitly opens via the picker) or 'drive' (full — lets users pull + // pre-existing Drive folders; requires Google's restricted-scope review to ship + // to real users in production). + GOOGLE_DRIVE_SCOPE: z.enum(['drive.file', 'drive']).default('drive.file'), + // Skip Drive files larger than this many megabytes when syncing. + DRIVE_SYNC_MAX_FILE_MB: z.coerce.number().int().positive().default(50), + // Signs the per-link Drive push-notification channel tokens. Falls back to + // SECRETBOX_MASTER_KEY when unset (see lib/google-drive.ts). + DRIVE_WEBHOOK_SECRET: z.string().min(1).optional(), + // Phase C — internal meeting-bot controller (the express service that asks // Recall to send the bot). Defaults to the Docker-network address; the API // POSTs /join here when an admitted meeting is starting soon. diff --git a/apps/api/src/db/schema.ts b/apps/api/src/db/schema.ts index afdb55881..8c98d48e5 100644 --- a/apps/api/src/db/schema.ts +++ b/apps/api/src/db/schema.ts @@ -105,6 +105,8 @@ export const workspaceMembers = pgTable( joinedAt: timestamp('joined_at', { withTimezone: true }).notNull().defaultNow(), // Phase C — encrypted Google Calendar refresh token (per member). calendarRefreshToken: text('calendar_refresh_token'), + // Phase 10 — encrypted Google Drive refresh token (per member). + driveRefreshToken: text('drive_refresh_token'), }, (table) => ({ pk: primaryKey({ columns: [table.workspaceId, table.userId] }), @@ -1681,6 +1683,78 @@ export const attachments = pgTable( }), ); +// ─── GOOGLE DRIVE SYNC ──────────────────────────────────────────────────────── +// Phase 10 — links a Mnema folder to a Google Drive folder and tracks per-file +// sync state. Core-only: references folders/docs/attachments/users/workspaces, +// no enterprise coupling. The encrypted refresh token lives on workspace_members +// (drive_refresh_token, above); these two tables hold the links + file mappings. + +export const driveFolderLinks = pgTable( + 'drive_folder_links', + { + id: uuid('id').primaryKey().default(sql`gen_random_uuid()`), + workspaceId: uuid('workspace_id').notNull().references(() => workspaces.id, { onDelete: 'cascade' }), + // The Mnema folder that mirrors a Drive folder. + folderId: uuid('folder_id').notNull().references(() => folders.id, { onDelete: 'cascade' }), + // The Google Drive folder id + display name (for the UI). + driveFolderId: text('drive_folder_id').notNull(), + driveFolderName: text('drive_folder_name'), + // Whose encrypted Drive token drives the sync for this link. + connectedBy: uuid('connected_by').notNull().references(() => users.id, { onDelete: 'cascade' }), + // 'pull' (Drive→Mnema), 'push' (Mnema→Drive), or 'both'. + direction: text('direction').notNull().default('both'), + // Allow-listed extensions synced (lowercase, no dot). Empty = built-in defaults. + acceptedTypes: text('accepted_types').array().notNull().default(sql`ARRAY[]::text[]`), + // 'lww' (last-writer-wins by modified time) or 'manual' (flag conflicts, never overwrite). + conflictPolicy: text('conflict_policy').notNull().default('manual'), + // Drive changes-API cursor for incremental pulls. + drivePageToken: text('drive_page_token'), + // Drive push-notification channel (id + expiry) so we can stop/renew it. + driveChannelId: text('drive_channel_id'), + driveChannelExpiresAt: timestamp('drive_channel_expires_at', { withTimezone: true }), + lastSyncedAt: timestamp('last_synced_at', { withTimezone: true }), + // 'active' | 'paused' | 'error' + status: text('status').notNull().default('active'), + errorMessage: text('error_message'), + createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(), + }, + (table) => ({ + workspaceIdx: index('drive_links_workspace_idx').on(table.workspaceId), + folderUnique: uniqueIndex('drive_links_folder_unique').on(table.folderId), + driveFolderUnique: uniqueIndex('drive_links_ws_drivefolder_unique').on(table.workspaceId, table.driveFolderId), + }), +); + +export const driveFileMappings = pgTable( + 'drive_file_mappings', + { + id: uuid('id').primaryKey().default(sql`gen_random_uuid()`), + linkId: uuid('link_id').notNull().references(() => driveFolderLinks.id, { onDelete: 'cascade' }), + // The Google Drive file id. + driveFileId: text('drive_file_id').notNull(), + driveName: text('drive_name'), + // Where it landed in Mnema — a doc (md/txt) or an attachment (binary). One is set. + docId: uuid('doc_id').references(() => docs.id, { onDelete: 'set null' }), + attachmentId: uuid('attachment_id').references(() => attachments.id, { onDelete: 'set null' }), + // Drive checksum (md5Checksum) + the Mnema-side content hash, for change detection. + driveMd5: text('drive_md5'), + contentHash: text('content_hash'), + driveModifiedAt: timestamp('drive_modified_at', { withTimezone: true }), + mnemaModifiedAt: timestamp('mnema_modified_at', { withTimezone: true }), + // 'synced' | 'pending_push' | 'pending_pull' | 'conflict' + syncState: text('sync_state').notNull().default('synced'), + createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(), + }, + (table) => ({ + linkIdx: index('drive_map_link_idx').on(table.linkId), + driveFileUnique: uniqueIndex('drive_map_link_file_unique').on(table.linkId, table.driveFileId), + docIdx: index('drive_map_doc_idx').on(table.docId), + attachmentIdx: index('drive_map_attachment_idx').on(table.attachmentId), + }), +); + // ─── KNOWLEDGE GRAPH ────────────────────────────────────────────────────────── export const graphNodes = pgTable( diff --git a/apps/api/src/lib/drive-sync.ts b/apps/api/src/lib/drive-sync.ts new file mode 100644 index 000000000..a9f68b3d6 --- /dev/null +++ b/apps/api/src/lib/drive-sync.ts @@ -0,0 +1,289 @@ +/** + * Phase 10 — Google Drive ⇄ Mnema sync engine (shared by routes/drive.ts and the + * drive-sync worker). + * + * pullLink(link) — Drive → Mnema: create/update docs (text) + attachments (binary) + * pushLink(link) — Mnema → Drive: export the linked folder's docs as files + * syncLink(link) — run both directions according to link.direction + * + * All writes go through withSystemPrivilege (server-side, RLS-bypassing) and are + * explicitly scoped by workspace_id. Sync is idempotent, keyed on + * drive_file_mappings (link_id, drive_file_id) and the file md5 checksum. + */ +import { randomUUID } from 'node:crypto'; +import { PutObjectCommand } from '@aws-sdk/client-s3'; +import { markdownToYjsState } from '@boppl/schema/node'; +import { and, eq, isNotNull } from 'drizzle-orm'; +import { config } from '../config/env.js'; +import { withSystemPrivilege } from '../db/with-system-privilege.js'; +import { + attachments, docs, driveFileMappings, driveFolderLinks, workspaceMembers, +} from '../db/schema.js'; +import { + downloadDriveFile, driveClientFromRefresh, extOf, isAcceptedType, isTextExt, + listFilesInFolder, mimeForExt, updateDriveFile, uploadToFolder, + type DriveClient, type DriveFile, +} from './google-drive.js'; +import { decryptSecret } from './secret-box.js'; +import { R2_BUCKET, isR2Configured, r2 } from './storage/r2-client.js'; +import { contentHash } from './yjs.js'; + +export type DriveLink = typeof driveFolderLinks.$inferSelect; +type DriveMapping = typeof driveFileMappings.$inferSelect; + +export interface SyncResult { + created: number; + updated: number; + skipped: number; + pushed: number; + conflicts: number; + error?: string; +} + +const emptyResult = (): SyncResult => ({ created: 0, updated: 0, skipped: 0, pushed: 0, conflicts: 0 }); + +/** The Drive client for a link, from the connecting member's encrypted token. */ +export async function getLinkDrive(link: DriveLink): Promise { + const rows = await withSystemPrivilege((tx) => + tx.select({ tok: workspaceMembers.driveRefreshToken }) + .from(workspaceMembers) + .where(and(eq(workspaceMembers.userId, link.connectedBy), isNotNull(workspaceMembers.driveRefreshToken))) + .limit(1), + ); + const enc = rows[0]?.tok; + if (!enc) return null; + return driveClientFromRefresh(decryptSecret(enc)); +} + +function tooLarge(f: DriveFile): boolean { + return Boolean(f.size && f.size > config.DRIVE_SYNC_MAX_FILE_MB * 1024 * 1024); +} + +async function findMapping(linkId: string, driveFileId: string): Promise { + const rows = await withSystemPrivilege((tx) => + tx.select().from(driveFileMappings) + .where(and(eq(driveFileMappings.linkId, linkId), eq(driveFileMappings.driveFileId, driveFileId))) + .limit(1), + ); + return rows[0]; +} + +async function touchLink(linkId: string, patch: Partial): Promise { + await withSystemPrivilege((tx) => + tx.update(driveFolderLinks).set({ ...patch, updatedAt: new Date() }).where(eq(driveFolderLinks.id, linkId)), + ); +} + +// ── Pull (Drive → Mnema) ────────────────────────────────────────────────────── + +async function upsertDocFromDrive( + link: DriveLink, f: DriveFile, markdown: string, mapping: DriveMapping | undefined, +): Promise { + const hash = contentHash(markdown); + const title = f.name.replace(/\.[^.]+$/, ''); + const yjsState = Buffer.from(await markdownToYjsState(markdown)); + const now = new Date(); + + await withSystemPrivilege(async (tx) => { + let docId = mapping?.docId ?? null; + if (docId) { + await tx.update(docs) + .set({ title, markdown, yjsState, contentHash: hash, updatedAt: now }) + .where(and(eq(docs.id, docId), eq(docs.workspaceId, link.workspaceId))); + } else { + const [doc] = await tx.insert(docs).values({ + workspaceId: link.workspaceId, + folderId: link.folderId, + path: `drive-${f.id}`, + title, + markdown, + yjsState, + contentHash: hash, + createdBy: link.connectedBy, + updatedBy: link.connectedBy, + }).returning({ id: docs.id }); + docId = doc!.id; + } + await upsertMapping(tx, link.id, f, { docId, contentHash: hash }); + }); +} + +async function upsertAttachmentFromDrive( + link: DriveLink, f: DriveFile, ext: string, bytes: Buffer, mapping: DriveMapping | undefined, +): Promise { + const r2Key = `attachments/${link.workspaceId}/${randomUUID()}.${ext}`; + const mime = f.mimeType || mimeForExt(ext); + await r2().send(new PutObjectCommand({ + Bucket: R2_BUCKET(), Key: r2Key, Body: bytes, ContentType: mime, + ContentDisposition: `attachment; filename="${f.name}"`, + })); + + await withSystemPrivilege(async (tx) => { + let attachmentId = mapping?.attachmentId ?? null; + if (attachmentId) { + await tx.update(attachments) + .set({ r2Key, format: ext, originalName: f.name, mimeType: mime, sizeBytes: bytes.length, status: 'ready', updatedAt: new Date() }) + .where(and(eq(attachments.id, attachmentId), eq(attachments.workspaceId, link.workspaceId))); + } else { + const [att] = await tx.insert(attachments).values({ + workspaceId: link.workspaceId, + type: 'source', + format: ext, + originalName: f.name, + r2Key, + mimeType: mime, + sizeBytes: bytes.length, + status: 'ready', + }).returning({ id: attachments.id }); + attachmentId = att!.id; + } + await upsertMapping(tx, link.id, f, { attachmentId }); + }); +} + +async function upsertMapping( + tx: Parameters[0]>[0], + linkId: string, f: DriveFile, extra: { docId?: string; attachmentId?: string; contentHash?: string }, +): Promise { + const now = new Date(); + await tx.insert(driveFileMappings).values({ + linkId, + driveFileId: f.id, + driveName: f.name, + docId: extra.docId ?? null, + attachmentId: extra.attachmentId ?? null, + driveMd5: f.md5Checksum, + contentHash: extra.contentHash ?? null, + driveModifiedAt: f.modifiedTime ? new Date(f.modifiedTime) : null, + mnemaModifiedAt: now, + syncState: 'synced', + }).onConflictDoUpdate({ + target: [driveFileMappings.linkId, driveFileMappings.driveFileId], + set: { + driveName: f.name, + ...(extra.docId ? { docId: extra.docId } : {}), + ...(extra.attachmentId ? { attachmentId: extra.attachmentId } : {}), + driveMd5: f.md5Checksum, + ...(extra.contentHash ? { contentHash: extra.contentHash } : {}), + driveModifiedAt: f.modifiedTime ? new Date(f.modifiedTime) : null, + mnemaModifiedAt: now, + syncState: 'synced', + updatedAt: now, + }, + }); +} + +/** Drive → Mnema. Downloads accepted, changed files and writes docs/attachments. */ +export async function pullLink(link: DriveLink): Promise { + const res = emptyResult(); + const drive = await getLinkDrive(link); + if (!drive) return { ...res, error: 'not_connected' }; + + let files: DriveFile[]; + try { + files = await listFilesInFolder(drive, link.driveFolderId); + } catch (err) { + await touchLink(link.id, { status: 'error', errorMessage: err instanceof Error ? err.message : 'list_failed' }); + return { ...res, error: 'list_failed' }; + } + + for (const f of files) { + const ext = extOf(f.name); + if (!isAcceptedType(ext, link.acceptedTypes) || tooLarge(f)) { res.skipped++; continue; } + + const mapping = await findMapping(link.id, f.id); + if (mapping?.driveMd5 && mapping.driveMd5 === f.md5Checksum) continue; // unchanged on Drive + + // Manual conflict policy: if Mnema has an un-pushed local edit, don't clobber it. + if (mapping?.syncState === 'pending_push' && link.conflictPolicy === 'manual') { + await withSystemPrivilege((tx) => + tx.update(driveFileMappings).set({ syncState: 'conflict', updatedAt: new Date() }).where(eq(driveFileMappings.id, mapping.id)), + ); + res.conflicts++; + continue; + } + + try { + const bytes = await downloadDriveFile(drive, f.id); + if (isTextExt(ext)) { + await upsertDocFromDrive(link, f, bytes.toString('utf8'), mapping); + } else { + if (!isR2Configured()) { res.skipped++; continue; } // no blob store → can't hold binaries + await upsertAttachmentFromDrive(link, f, ext, bytes, mapping); + } + if (mapping) res.updated++; else res.created++; + } catch { + res.skipped++; + } + } + + await touchLink(link.id, { lastSyncedAt: new Date(), status: 'active', errorMessage: null }); + return res; +} + +// ── Push (Mnema → Drive) ────────────────────────────────────────────────────── + +/** Mnema → Drive. Exports the linked folder's docs to `.md` files in Drive. */ +export async function pushLink(link: DriveLink): Promise { + const res = emptyResult(); + const drive = await getLinkDrive(link); + if (!drive) return { ...res, error: 'not_connected' }; + + const folderDocs = await withSystemPrivilege((tx) => + tx.select({ id: docs.id, title: docs.title, markdown: docs.markdown, contentHash: docs.contentHash }) + .from(docs) + .where(and(eq(docs.workspaceId, link.workspaceId), eq(docs.folderId, link.folderId))), + ); + + for (const d of folderDocs) { + try { + const existing = await withSystemPrivilege((tx) => + tx.select().from(driveFileMappings) + .where(and(eq(driveFileMappings.linkId, link.id), eq(driveFileMappings.docId, d.id))) + .limit(1), + ); + const map = existing[0]; + // Skip if the doc hasn't changed since it was last synced. + if (map && map.contentHash && map.contentHash === d.contentHash) continue; + + const body = Buffer.from(d.markdown, 'utf8'); + const name = `${d.title || 'untitled'}.md`; + if (map?.driveFileId) { + const { md5Checksum } = await updateDriveFile(drive, map.driveFileId, 'text/markdown', body); + await withSystemPrivilege((tx) => + tx.update(driveFileMappings) + .set({ driveMd5: md5Checksum, contentHash: d.contentHash, syncState: 'synced', mnemaModifiedAt: new Date(), updatedAt: new Date() }) + .where(eq(driveFileMappings.id, map.id)), + ); + } else { + const { id: driveFileId, md5Checksum } = await uploadToFolder(drive, link.driveFolderId, name, 'text/markdown', body); + await withSystemPrivilege((tx) => + tx.insert(driveFileMappings).values({ + linkId: link.id, driveFileId, driveName: name, docId: d.id, + driveMd5: md5Checksum, contentHash: d.contentHash, mnemaModifiedAt: new Date(), syncState: 'synced', + }), + ); + } + res.pushed++; + } catch { + res.skipped++; + } + } + + await touchLink(link.id, { lastSyncedAt: new Date() }); + return res; +} + +/** Run the configured direction(s) for a link and return a combined result. */ +export async function syncLink(link: DriveLink): Promise { + if (link.status === 'paused') return { ...emptyResult(), error: 'paused' }; + const combined = emptyResult(); + const add = (r: SyncResult) => { + combined.created += r.created; combined.updated += r.updated; combined.skipped += r.skipped; + combined.pushed += r.pushed; combined.conflicts += r.conflicts; + if (r.error && !combined.error) combined.error = r.error; + }; + if (link.direction === 'pull' || link.direction === 'both') add(await pullLink(link)); + if (link.direction === 'push' || link.direction === 'both') add(await pushLink(link)); + return combined; +} diff --git a/apps/api/src/lib/google-drive.ts b/apps/api/src/lib/google-drive.ts new file mode 100644 index 000000000..3bb1644d0 --- /dev/null +++ b/apps/api/src/lib/google-drive.ts @@ -0,0 +1,249 @@ +/** + * Phase 10 — Google Drive client (googleapis SDK). + * + * Bring-your-own OAuth app, same shape as lib/google-calendar.ts: the OAuth + * client credentials come from env (optional) and callers must check + * `driveConfigured()` first and 503 when unset. The refresh token is stored + * encrypted (secret-box) on workspace_members.drive_refresh_token. + * + * Scope is configurable (GOOGLE_DRIVE_SCOPE): + * - 'drive.file' (default): least privilege — Mnema only sees files it created + * or that the user explicitly opened with it. + * - 'drive': full access — lets users pull pre-existing Drive folders; needs + * Google's restricted-scope review to ship to real users in production. + */ +import { Readable } from 'node:stream'; +import { google, type drive_v3 } from 'googleapis'; +import { config } from '../config/env.js'; + +export type DriveClient = drive_v3.Drive; + +const SCOPE_URLS: Record<'drive.file' | 'drive', string> = { + 'drive.file': 'https://www.googleapis.com/auth/drive.file', + drive: 'https://www.googleapis.com/auth/drive', +}; + +/** File extensions synced by default when a link doesn't set its own allow-list. */ +export const DEFAULT_ACCEPTED_TYPES = [ + 'md', 'markdown', 'txt', 'pdf', 'doc', 'docx', 'csv', 'png', 'jpg', 'jpeg', +]; + +/** Extensions that become Mnema docs (text). Everything else becomes an attachment. */ +const TEXT_EXTS = new Set(['md', 'markdown', 'txt']); + +const MIME_BY_EXT: Record = { + md: 'text/markdown', markdown: 'text/markdown', txt: 'text/plain', + pdf: 'application/pdf', csv: 'text/csv', + doc: 'application/msword', + docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', +}; + +export function driveConfigured(): boolean { + return Boolean( + config.GOOGLE_DRIVE_CLIENT_ID && + config.GOOGLE_DRIVE_CLIENT_SECRET && + config.GOOGLE_DRIVE_REDIRECT_URI, + ); +} + +export function driveScopeUrl(): string { + return SCOPE_URLS[config.GOOGLE_DRIVE_SCOPE]; +} + +/** Secret used to sign Drive push-notification channel tokens. */ +export function driveWebhookSecret(): string { + return config.DRIVE_WEBHOOK_SECRET ?? config.SECRETBOX_MASTER_KEY; +} + +export function extOf(name: string): string { + const i = name.lastIndexOf('.'); + return i >= 0 ? name.slice(i + 1).toLowerCase() : ''; +} + +export function isTextExt(ext: string): boolean { + return TEXT_EXTS.has(ext.toLowerCase()); +} + +export function mimeForExt(ext: string): string { + return MIME_BY_EXT[ext.toLowerCase()] ?? 'application/octet-stream'; +} + +/** True if `ext` is allowed for this link (empty allow-list = built-in defaults). */ +export function isAcceptedType(ext: string, acceptedTypes: string[]): boolean { + const list = acceptedTypes.length > 0 ? acceptedTypes : DEFAULT_ACCEPTED_TYPES; + return list.includes(ext.toLowerCase()); +} + +function oauthClient() { + return new google.auth.OAuth2( + config.GOOGLE_DRIVE_CLIENT_ID, + config.GOOGLE_DRIVE_CLIENT_SECRET, + config.GOOGLE_DRIVE_REDIRECT_URI, + ); +} + +/** Google consent URL to start the Drive-link flow (offline → refresh token). */ +export function consentUrl(state: string): string { + return oauthClient().generateAuthUrl({ + access_type: 'offline', + prompt: 'select_account consent', // chooser → consent, and force a refresh_token + scope: [driveScopeUrl()], + state, + include_granted_scopes: true, + }); +} + +/** Exchange an auth code for tokens. Returns the offline refresh token. */ +export async function exchangeCode(code: string): Promise<{ refresh_token?: string; access_token?: string }> { + const { tokens } = await oauthClient().getToken(code); + return { refresh_token: tokens.refresh_token ?? undefined, access_token: tokens.access_token ?? undefined }; +} + +/** An authenticated Drive v3 client from a stored refresh token (auto-refreshes access). */ +export function driveClientFromRefresh(refreshToken: string): DriveClient { + const auth = oauthClient(); + auth.setCredentials({ refresh_token: refreshToken }); + return google.drive({ version: 'v3', auth }); +} + +export interface DriveFolder { id: string; name: string } + +/** List the user's Drive folders under `parentId` (or 'root') — for the picker. */ +export async function listDriveFolders(drive: DriveClient, parentId = 'root'): Promise { + const res = await drive.files.list({ + q: `mimeType = 'application/vnd.google-apps.folder' and trashed = false and '${parentId}' in parents`, + fields: 'files(id, name)', + orderBy: 'name', + pageSize: 200, + spaces: 'drive', + }); + return (res.data.files ?? []) + .filter((f): f is { id: string; name: string } => Boolean(f.id && f.name)) + .map((f) => ({ id: f.id, name: f.name })); +} + +export interface DriveFile { + id: string; + name: string; + mimeType: string; + md5Checksum: string | null; + modifiedTime: string | null; + size: number | null; + trashed: boolean; +} + +const FILE_FIELDS = 'id, name, mimeType, md5Checksum, modifiedTime, size, trashed'; + +function toDriveFile(f: drive_v3.Schema$File): DriveFile { + return { + id: f.id!, + name: f.name ?? 'untitled', + mimeType: f.mimeType ?? 'application/octet-stream', + md5Checksum: f.md5Checksum ?? null, + modifiedTime: f.modifiedTime ?? null, + size: f.size ? Number(f.size) : null, + trashed: Boolean(f.trashed), + }; +} + +/** List non-folder files directly under a Drive folder (paginates fully). */ +export async function listFilesInFolder(drive: DriveClient, driveFolderId: string): Promise { + const out: DriveFile[] = []; + let pageToken: string | undefined; + do { + const res = await drive.files.list({ + q: `'${driveFolderId}' in parents and mimeType != 'application/vnd.google-apps.folder' and trashed = false`, + fields: `nextPageToken, files(${FILE_FIELDS})`, + pageSize: 200, + spaces: 'drive', + pageToken, + }); + for (const f of res.data.files ?? []) if (f.id) out.push(toDriveFile(f)); + pageToken = res.data.nextPageToken ?? undefined; + } while (pageToken); + return out; +} + +/** The current changes-API start token (cursor) for incremental pulls. */ +export async function getStartPageToken(drive: DriveClient): Promise { + const res = await drive.changes.getStartPageToken({}); + return res.data.startPageToken!; +} + +export interface DriveChangePage { files: DriveFile[]; removedIds: string[]; newPageToken: string } + +/** Fetch the batch of changes since `pageToken`. Caller filters to linked folders. */ +export async function listChanges(drive: DriveClient, pageToken: string): Promise { + const files: DriveFile[] = []; + const removedIds: string[] = []; + let token: string | undefined = pageToken; + let newPageToken = pageToken; + do { + const { data }: { data: drive_v3.Schema$ChangeList } = await drive.changes.list({ + pageToken: token, + fields: `nextPageToken, newStartPageToken, changes(removed, fileId, file(${FILE_FIELDS}, parents))`, + pageSize: 200, + spaces: 'drive', + }); + for (const c of data.changes ?? []) { + if (c.removed || c.file?.trashed) { if (c.fileId) removedIds.push(c.fileId); continue; } + if (c.file?.id && c.file.mimeType !== 'application/vnd.google-apps.folder') files.push(toDriveFile(c.file)); + } + if (data.newStartPageToken) newPageToken = data.newStartPageToken; + token = data.nextPageToken ?? undefined; + } while (token); + return { files, removedIds, newPageToken }; +} + +/** The parent folder ids of a Drive file (to test membership of a linked folder). */ +export async function getFileParents(drive: DriveClient, fileId: string): Promise { + const res = await drive.files.get({ fileId, fields: 'parents' }); + return res.data.parents ?? []; +} + +/** Download a Drive file's bytes. */ +export async function downloadDriveFile(drive: DriveClient, fileId: string): Promise { + const res = await drive.files.get( + { fileId, alt: 'media' }, + { responseType: 'arraybuffer' }, + ); + return Buffer.from(res.data as ArrayBuffer); +} + +/** Create a Drive folder (optionally under `parentId`). Returns its id. */ +export async function createDriveFolder(drive: DriveClient, name: string, parentId?: string): Promise { + const res = await drive.files.create({ + requestBody: { + name, + mimeType: 'application/vnd.google-apps.folder', + ...(parentId ? { parents: [parentId] } : {}), + }, + fields: 'id', + }); + return res.data.id!; +} + +/** Upload a new file into a Drive folder. Returns the new file id + md5. */ +export async function uploadToFolder( + drive: DriveClient, driveFolderId: string, name: string, mimeType: string, body: Buffer, +): Promise<{ id: string; md5Checksum: string | null }> { + const res = await drive.files.create({ + requestBody: { name, parents: [driveFolderId] }, + media: { mimeType, body: Readable.from(body) }, + fields: 'id, md5Checksum', + }); + return { id: res.data.id!, md5Checksum: res.data.md5Checksum ?? null }; +} + +/** Overwrite an existing Drive file's content. Returns its new md5. */ +export async function updateDriveFile( + drive: DriveClient, fileId: string, mimeType: string, body: Buffer, +): Promise<{ md5Checksum: string | null }> { + const res = await drive.files.update({ + fileId, + media: { mimeType, body: Readable.from(body) }, + fields: 'id, md5Checksum', + }); + return { md5Checksum: res.data.md5Checksum ?? null }; +} diff --git a/apps/api/src/plugins/auth.ts b/apps/api/src/plugins/auth.ts index 2b109d680..9d47b7df6 100644 --- a/apps/api/src/plugins/auth.ts +++ b/apps/api/src/plugins/auth.ts @@ -95,6 +95,9 @@ export const authPlugin: FastifyPluginAsync = fp(async (app) => { // /callback auth: OnlyOffice JWT signature. /file auth: tenantId query param. if (url === '/api/onlyoffice/callback') return; if (url.startsWith('/api/onlyoffice/') && url.endsWith('/file')) return; + // Google Drive push notifications — server-to-server POST from Google with no + // cookie. Authenticated in-handler by matching the stored channel id. + if (url === '/api/drive/webhook') return; const token = req.cookies[JWT_COOKIE_NAME] ?? extractBearer(req.headers.authorization); if (!token) { diff --git a/apps/api/src/queue/drive-sync.ts b/apps/api/src/queue/drive-sync.ts new file mode 100644 index 000000000..5c050401f --- /dev/null +++ b/apps/api/src/queue/drive-sync.ts @@ -0,0 +1,31 @@ +/** + * BullMQ queue for Google Drive folder sync (Phase 10). + * + * Enqueued from routes/drive.ts (manual "Sync now" + the push webhook) and the + * reconcile cron. The drive-sync worker runs syncLink() for the link. A stable + * jobId per link debounces concurrent syncs of the same folder. + */ +import { Queue } from 'bullmq'; +import { redisConnection } from '../lib/redis.js'; + +export const DRIVE_SYNC_QUEUE_NAME = 'drive-sync'; + +export interface DriveSyncJobData { + linkId: string; + reason: 'manual' | 'webhook' | 'cron' | 'push'; +} + +export const driveSyncQueue = new Queue(DRIVE_SYNC_QUEUE_NAME, { + connection: redisConnection, + defaultJobOptions: { + attempts: 3, + backoff: { type: 'exponential', delay: 5000 }, + removeOnComplete: 100, + removeOnFail: 200, + }, +}); + +export async function enqueueDriveSync(data: DriveSyncJobData): Promise { + // jobId dedupes: a queued/active sync for the same link absorbs new triggers. + await driveSyncQueue.add('sync', data, { jobId: `drive-sync:${data.linkId}` }); +} diff --git a/apps/api/src/routes/drive.ts b/apps/api/src/routes/drive.ts new file mode 100644 index 000000000..12df33310 --- /dev/null +++ b/apps/api/src/routes/drive.ts @@ -0,0 +1,285 @@ +/** + * Phase 10 — Google Drive folder linking + two-way sync. + * + * GET /api/drive/connect → redirect to Google consent (offline, configurable scope) + * GET /api/drive/callback → exchange code, store encrypted refresh token + * GET /api/drive/status → { connected, configured, scope } + * GET /api/drive/folders?parent= → list the user's Drive folders (picker) + * GET /api/drive/links → list this workspace's folder links + * POST /api/drive/links → link a Mnema folder ⇄ Drive folder + * PATCH /api/drive/links/:id → edit types / direction / conflict policy / pause + * DELETE /api/drive/links/:id → unlink + * POST /api/drive/links/:id/sync → enqueue a sync now + * POST /api/drive/webhook → Google push-notification receiver (public + token) + * + * The refresh token is user-scoped (their Google Drive); links + mappings are + * workspace-scoped and enforced app-layer (filter by workspace_id + requireRole). + */ +import { and, eq, isNotNull } from 'drizzle-orm'; +import type { FastifyPluginAsync, FastifyReply } from 'fastify'; +import { config } from '../config/env.js'; +import { db } from '../db/index.js'; +import { driveFileMappings, driveFolderLinks, folders, workspaceMembers } from '../db/schema.js'; +import { + consentUrl, createDriveFolder, DEFAULT_ACCEPTED_TYPES, driveClientFromRefresh, + driveConfigured, driveScopeUrl, exchangeCode, listDriveFolders, type DriveClient, +} from '../lib/google-drive.js'; +import { enqueueDriveSync } from '../queue/drive-sync.js'; +import { requireRole, RoleError } from '../lib/role.js'; +import { decryptSecret, encryptSecret, signState, verifySignedState } from '../lib/secret-box.js'; + +const DIRECTIONS = ['pull', 'push', 'both'] as const; +const CONFLICT_POLICIES = ['manual', 'lww'] as const; + +export const driveRoutes: FastifyPluginAsync = async (app) => { + function guard(err: unknown, reply: FastifyReply): boolean { + if (err instanceof RoleError) { reply.code(err.status).send({ error: err.reason }); return true; } + return false; + } + + /** The requesting user's Drive client (their token), or null if not connected. */ + async function userDrive(userId: string): Promise { + const rows = await db.select({ tok: workspaceMembers.driveRefreshToken }) + .from(workspaceMembers) + .where(and(eq(workspaceMembers.userId, userId), isNotNull(workspaceMembers.driveRefreshToken))) + .limit(1); + const enc = rows[0]?.tok; + return enc ? driveClientFromRefresh(decryptSecret(enc)) : null; + } + + function normalizeTypes(input: unknown): string[] { + if (!Array.isArray(input)) return []; + return [...new Set(input + .filter((x): x is string => typeof x === 'string') + .map((x) => x.trim().replace(/^\./, '').toLowerCase()) + .filter(Boolean))]; + } + + // ── Start the link flow ──────────────────────────────────────────────────── + app.get('/api/drive/connect', async (req, reply) => { + if (!req.auth) return reply.code(401).send({ error: 'unauthorized' }); + if (!driveConfigured()) return reply.code(503).send({ error: 'drive_not_configured' }); + try { await requireRole(req, 'editor'); } catch (e) { if (guard(e, reply)) return; throw e; } + const state = signState({ sub: req.auth.sub, tenant: req.auth.tenant_id }); + const url = consentUrl(state); + // Navigate to Google WITHOUT a referrer (same reasoning as calendar connect): + // a referrer-less navigation resolves the active session and shows the account + // chooser instead of Google's sign-in form (which 400s without a session). + return reply + .header('Referrer-Policy', 'no-referrer') + .type('text/html') + .send( + `` + + `` + + ``, + ); + }); + + // ── Google redirects back here ───────────────────────────────────────────── + app.get('/api/drive/callback', async (req, reply) => { + const q = req.query as { code?: string; state?: string; error?: string }; + const back = `${config.WEB_BASE_URL}/app/settings/integrations`; + if (q.error || !q.code || !q.state) return reply.redirect(`${back}?drive=error`); + const st = verifySignedState(q.state); + if (!st || !driveConfigured()) return reply.redirect(`${back}?drive=error`); + try { + const tokens = await exchangeCode(q.code); + if (!tokens.refresh_token) { + req.log.warn('drive callback: no refresh_token returned'); + return reply.redirect(`${back}?drive=error`); + } + await db.update(workspaceMembers) + .set({ driveRefreshToken: encryptSecret(tokens.refresh_token) }) + .where(and(eq(workspaceMembers.workspaceId, st.tenant), eq(workspaceMembers.userId, st.sub))); + return reply.redirect(`${back}?drive=connected`); + } catch (err) { + req.log.error({ err }, 'drive callback failed'); + return reply.redirect(`${back}?drive=error`); + } + }); + + // ── Connection status ────────────────────────────────────────────────────── + app.get('/api/drive/status', async (req, reply) => { + if (!req.auth) return reply.code(401).send({ error: 'unauthorized' }); + const rows = await db.select({ tok: workspaceMembers.driveRefreshToken }) + .from(workspaceMembers) + .where(and(eq(workspaceMembers.userId, req.auth.sub), isNotNull(workspaceMembers.driveRefreshToken))) + .limit(1); + return reply.send({ + connected: rows.length > 0, + configured: driveConfigured(), + scope: driveConfigured() ? driveScopeUrl() : null, + defaultTypes: DEFAULT_ACCEPTED_TYPES, + }); + }); + + // ── List the user's Drive folders (for the picker) ───────────────────────── + app.get('/api/drive/folders', async (req, reply) => { + if (!req.auth) return reply.code(401).send({ error: 'unauthorized' }); + if (!driveConfigured()) return reply.code(503).send({ error: 'drive_not_configured' }); + const drive = await userDrive(req.auth.sub); + if (!drive) return reply.code(400).send({ error: 'drive_not_connected' }); + const parent = (req.query as { parent?: string }).parent || 'root'; + try { + const folderList = await listDriveFolders(drive, parent); + return reply.send({ folders: folderList }); + } catch (err) { + req.log.error({ err }, 'drive folders list failed'); + return reply.code(502).send({ error: 'drive_list_failed' }); + } + }); + + // ── List links for the active workspace ──────────────────────────────────── + app.get('/api/drive/links', async (req, reply) => { + if (!req.auth) return reply.code(401).send({ error: 'unauthorized' }); + const ws = req.auth.tenant_id; + const rows = await db.select().from(driveFolderLinks) + .where(eq(driveFolderLinks.workspaceId, ws)); + return reply.send({ links: rows }); + }); + + // ── Create a link ────────────────────────────────────────────────────────── + app.post('/api/drive/links', async (req, reply) => { + if (!req.auth) return reply.code(401).send({ error: 'unauthorized' }); + if (!driveConfigured()) return reply.code(503).send({ error: 'drive_not_configured' }); + try { await requireRole(req, 'editor'); } catch (e) { if (guard(e, reply)) return; throw e; } + + const ws = req.auth.tenant_id; + const body = req.body as { + folderId?: string; + driveFolderId?: string; + createInDrive?: boolean; + driveParentId?: string; + driveFolderName?: string; + acceptedTypes?: string[]; + direction?: string; + conflictPolicy?: string; + }; + if (!body.folderId) return reply.code(400).send({ error: 'folderId_required' }); + + // The Mnema folder must belong to the active workspace. + const [folder] = await db.select({ id: folders.id, name: folders.name }) + .from(folders) + .where(and(eq(folders.id, body.folderId), eq(folders.workspaceId, ws))) + .limit(1); + if (!folder) return reply.code(404).send({ error: 'folder_not_found' }); + + const drive = await userDrive(req.auth.sub); + if (!drive) return reply.code(400).send({ error: 'drive_not_connected' }); + + // Resolve the Drive folder: either an existing one, or create it from the Mnema folder. + let driveFolderId = body.driveFolderId; + let driveFolderName = body.driveFolderName ?? null; + try { + if (body.createInDrive) { + driveFolderId = await createDriveFolder(drive, folder.name, body.driveParentId); + driveFolderName = folder.name; + } + } catch (err) { + req.log.error({ err }, 'drive folder create failed'); + return reply.code(502).send({ error: 'drive_create_failed' }); + } + if (!driveFolderId) return reply.code(400).send({ error: 'driveFolderId_or_createInDrive_required' }); + + const direction = DIRECTIONS.includes(body.direction as never) ? body.direction! : 'both'; + const conflictPolicy = CONFLICT_POLICIES.includes(body.conflictPolicy as never) ? body.conflictPolicy! : 'manual'; + + try { + const [link] = await db.insert(driveFolderLinks).values({ + workspaceId: ws, + folderId: folder.id, + driveFolderId, + driveFolderName, + connectedBy: req.auth.sub, + direction, + conflictPolicy, + acceptedTypes: normalizeTypes(body.acceptedTypes), + }).returning(); + // Kick off an initial sync. + await enqueueDriveSync({ linkId: link!.id, reason: 'manual' }); + return reply.send({ link }); + } catch (err) { + // Unique violation → folder already linked. + req.log.warn({ err }, 'drive link create failed'); + return reply.code(409).send({ error: 'already_linked' }); + } + }); + + // ── Edit a link ──────────────────────────────────────────────────────────── + app.patch('/api/drive/links/:id', async (req, reply) => { + if (!req.auth) return reply.code(401).send({ error: 'unauthorized' }); + try { await requireRole(req, 'editor'); } catch (e) { if (guard(e, reply)) return; throw e; } + const ws = req.auth.tenant_id; + const { id } = req.params as { id: string }; + const body = req.body as { acceptedTypes?: string[]; direction?: string; conflictPolicy?: string; status?: string }; + + const patch: Partial = { updatedAt: new Date() }; + if (body.acceptedTypes !== undefined) patch.acceptedTypes = normalizeTypes(body.acceptedTypes); + if (body.direction && DIRECTIONS.includes(body.direction as never)) patch.direction = body.direction; + if (body.conflictPolicy && CONFLICT_POLICIES.includes(body.conflictPolicy as never)) patch.conflictPolicy = body.conflictPolicy; + if (body.status === 'active' || body.status === 'paused') patch.status = body.status; + + const [link] = await db.update(driveFolderLinks).set(patch) + .where(and(eq(driveFolderLinks.id, id), eq(driveFolderLinks.workspaceId, ws))) + .returning(); + if (!link) return reply.code(404).send({ error: 'link_not_found' }); + return reply.send({ link }); + }); + + // ── Unlink ───────────────────────────────────────────────────────────────── + app.delete('/api/drive/links/:id', async (req, reply) => { + if (!req.auth) return reply.code(401).send({ error: 'unauthorized' }); + try { await requireRole(req, 'editor'); } catch (e) { if (guard(e, reply)) return; throw e; } + const ws = req.auth.tenant_id; + const { id } = req.params as { id: string }; + const [gone] = await db.delete(driveFolderLinks) + .where(and(eq(driveFolderLinks.id, id), eq(driveFolderLinks.workspaceId, ws))) + .returning({ id: driveFolderLinks.id }); + if (!gone) return reply.code(404).send({ error: 'link_not_found' }); + return reply.send({ ok: true }); + }); + + // ── Sync now ─────────────────────────────────────────────────────────────── + app.post('/api/drive/links/:id/sync', async (req, reply) => { + if (!req.auth) return reply.code(401).send({ error: 'unauthorized' }); + try { await requireRole(req, 'editor'); } catch (e) { if (guard(e, reply)) return; throw e; } + const ws = req.auth.tenant_id; + const { id } = req.params as { id: string }; + const [link] = await db.select({ id: driveFolderLinks.id }).from(driveFolderLinks) + .where(and(eq(driveFolderLinks.id, id), eq(driveFolderLinks.workspaceId, ws))) + .limit(1); + if (!link) return reply.code(404).send({ error: 'link_not_found' }); + await enqueueDriveSync({ linkId: id, reason: 'manual' }); + return reply.send({ ok: true, queued: true }); + }); + + // ── Google push-notification receiver ────────────────────────────────────── + // Public route (no auth cookie survives Google's POST). We authenticate by + // matching the channel id we stored on the link when registering the watch. + app.post('/api/drive/webhook', async (req, reply) => { + const channelId = req.headers['x-goog-channel-id']; + const state = req.headers['x-goog-resource-state']; + if (typeof channelId !== 'string') return reply.code(204).send(); + // 'sync' is Google's initial handshake ping — ack and ignore. + if (state === 'sync') return reply.code(204).send(); + const [link] = await db.select({ id: driveFolderLinks.id }).from(driveFolderLinks) + .where(eq(driveFolderLinks.driveChannelId, channelId)) + .limit(1); + if (link) await enqueueDriveSync({ linkId: link.id, reason: 'webhook' }); + return reply.code(204).send(); + }); + + // ── Conflicts for a link (for the resolver UI) ───────────────────────────── + app.get('/api/drive/links/:id/conflicts', async (req, reply) => { + if (!req.auth) return reply.code(401).send({ error: 'unauthorized' }); + const ws = req.auth.tenant_id; + const { id } = req.params as { id: string }; + const [link] = await db.select({ id: driveFolderLinks.id }).from(driveFolderLinks) + .where(and(eq(driveFolderLinks.id, id), eq(driveFolderLinks.workspaceId, ws))) + .limit(1); + if (!link) return reply.code(404).send({ error: 'link_not_found' }); + const rows = await db.select().from(driveFileMappings) + .where(and(eq(driveFileMappings.linkId, id), eq(driveFileMappings.syncState, 'conflict'))); + return reply.send({ conflicts: rows }); + }); +}; diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index e881cc241..cf0c642ae 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -30,6 +30,7 @@ import { healthRoutes } from './routes/health.js'; import { invitationsRoutes } from './routes/invitations.js'; import { membersRoutes } from './routes/members.js'; import { calendarRoutes } from './routes/calendar.js'; +import { driveRoutes } from './routes/drive.js'; import { notificationsRoutes } from './routes/notifications.js'; import { workspacesRoutes } from './routes/workspaces.js'; import { byokRoutes } from './routes/byok.js'; @@ -147,6 +148,7 @@ await app.register(communityRoutes); await app.register(invitationsRoutes); await app.register(membersRoutes); await app.register(calendarRoutes); +await app.register(driveRoutes); await app.register(notificationsRoutes); await app.register(workspacesRoutes); await app.register(byokRoutes); diff --git a/apps/api/src/workers/drive-sync/worker.ts b/apps/api/src/workers/drive-sync/worker.ts new file mode 100644 index 000000000..57bc4b4ab --- /dev/null +++ b/apps/api/src/workers/drive-sync/worker.ts @@ -0,0 +1,41 @@ +/** + * Google Drive sync worker (Phase 10). + * + * Consumes drive-sync jobs (manual "Sync now", push webhook, reconcile cron) and + * runs syncLink() for the link — pulling Drive→Mnema and/or pushing Mnema→Drive + * per the link's direction. Idempotent; safe to re-run. + */ +import { Worker } from 'bullmq'; +import { eq } from 'drizzle-orm'; +import IORedis from 'ioredis'; +import { config } from '../../config/env.js'; +import { db } from '../../db/index.js'; +import { driveFolderLinks } from '../../db/schema.js'; +import { syncLink } from '../../lib/drive-sync.js'; +import { DRIVE_SYNC_QUEUE_NAME, type DriveSyncJobData } from '../../queue/drive-sync.js'; + +export function startDriveSyncWorker(): Worker { + // Dedicated connection for the worker (separate from queue connection). + const connection = new IORedis(config.REDIS_URL, { maxRetriesPerRequest: null }); + + const worker = new Worker( + DRIVE_SYNC_QUEUE_NAME, + async (job) => { + const rows = await db.select().from(driveFolderLinks) + .where(eq(driveFolderLinks.id, job.data.linkId)).limit(1); + const link = rows[0]; + if (!link) return; + const res = await syncLink(link); + // eslint-disable-next-line no-console + console.log(`[drive-sync] ${job.data.linkId} reason=${job.data.reason}`, res); + }, + { connection, concurrency: 4 }, + ); + + worker.on('failed', (job, err) => { + console.error(`[drive-sync] ${job?.id ?? '?'} failed (attempt ${job?.attemptsMade ?? '?'}):`, err.message); + }); + worker.on('error', (err) => { console.error('[drive-sync] worker error:', err); }); + + return worker; +} diff --git a/apps/api/src/workers/server.ts b/apps/api/src/workers/server.ts index 6569ac127..8926fea9b 100644 --- a/apps/api/src/workers/server.ts +++ b/apps/api/src/workers/server.ts @@ -12,6 +12,7 @@ import { startEmbeddingsWorker } from './embeddings/worker.js'; import { startEmailWorker } from './email/worker.js'; import { startHookEventsWorker } from './hook-events/worker.js'; +import { startDriveSyncWorker } from './drive-sync/worker.js'; import { startRetryWorker } from './retry/worker.js'; import { startCronWorkers } from './cron.js'; import { startPdfGenerationWorker } from './pdf-generation/worker.js'; @@ -29,6 +30,10 @@ const hookEventsWorker = startHookEventsWorker(); // eslint-disable-next-line no-console console.log('[workers] hook-events worker started'); +const driveSyncWorker = startDriveSyncWorker(); +// eslint-disable-next-line no-console +console.log('[workers] drive-sync worker started'); + const retryWorker = startRetryWorker(); // eslint-disable-next-line no-console console.log('[workers] retry worker started'); @@ -58,6 +63,7 @@ const shutdown = async (signal: string): Promise => { await embeddings.close(); await emailWorker.close(); await hookEventsWorker.close(); + await driveSyncWorker.close(); await retryWorker.close(); await cronWorkers.close(); if (pdfWorker) await pdfWorker.close(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 51129081d..6fc27b533 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -141,6 +141,9 @@ importers: fastify-plugin: specifier: ^5.0.0 version: 5.1.0 + googleapis: + specifier: ^173.0.0 + version: 173.0.0 gpt-tokenizer: specifier: ^3.4.0 version: 3.4.0 @@ -929,11 +932,11 @@ packages: '@esbuild-kit/core-utils@3.3.2': resolution: {integrity: sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ==} - deprecated: 'Merged into tsx: https://tsx.is' + deprecated: 'Merged into tsx: https://tsx.hirok.io' '@esbuild-kit/esm-loader@2.6.5': resolution: {integrity: sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA==} - deprecated: 'Merged into tsx: https://tsx.is' + deprecated: 'Merged into tsx: https://tsx.hirok.io' '@esbuild/aix-ppc64@0.25.12': resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} @@ -1827,6 +1830,10 @@ packages: '@ioredis/commands@1.10.0': resolution: {integrity: sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==} + '@isaacs/cliui@8.0.2': + resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} + engines: {node: '>=12'} + '@isaacs/fs-minipass@4.0.1': resolution: {integrity: sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==} engines: {node: '>=18.0.0'} @@ -2240,6 +2247,10 @@ packages: '@pinojs/redact@0.4.0': resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} + '@pkgjs/parseargs@0.11.0': + resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} + engines: {node: '>=14'} + '@prosemirror-adapter/core@0.5.3': resolution: {integrity: sha512-jGcI/eq3USFqf1fCaejgCmifmXYDyFco6/5KjyyNBnuEhLqyLjD+020CtcduEOsnN4xPZr5ikclgHIR6kEFkQg==} @@ -3318,6 +3329,9 @@ packages: bidi-js@1.0.3: resolution: {integrity: sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==} + bignumber.js@9.3.1: + resolution: {integrity: sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==} + bindings@1.5.0: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} @@ -3337,6 +3351,9 @@ packages: brace-expansion@1.1.14: resolution: {integrity: sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==} + brace-expansion@2.1.2: + resolution: {integrity: sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==} + brace-expansion@5.0.7: resolution: {integrity: sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==} engines: {node: 18 || 20 || >=22} @@ -3346,6 +3363,9 @@ packages: engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true + buffer-equal-constant-time@1.0.1: + resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} @@ -3762,6 +3782,10 @@ packages: resolution: {integrity: sha512-QmU3lyEnbENQPo0M1F9BMu4s6cqNNp8iJA+b/HP2sSb7pf3dxwF3+EP1eO69rwBfH9kFJ1apmzrtogAmVt2/Xw==} engines: {node: '>=12'} + data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + data-urls@7.0.0: resolution: {integrity: sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==} engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0} @@ -3974,6 +3998,12 @@ packages: resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} engines: {node: '>= 0.4'} + eastasianwidth@0.2.0: + resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + + ecdsa-sig-formatter@1.0.11: + resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} + ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} @@ -3989,6 +4019,9 @@ packages: emoji-regex@8.0.0: resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + emoji-regex@9.2.2: + resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + encodeurl@2.0.0: resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} engines: {node: '>= 0.8'} @@ -4237,6 +4270,10 @@ packages: picomatch: optional: true + fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + fflate@0.8.3: resolution: {integrity: sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==} @@ -4285,10 +4322,18 @@ packages: resolution: {integrity: sha512-TdJ2KbkoiDQ7NIRx8IPGD0mAXXpLhamS7c+b7W98b0MHG7lphnda1VOQX/98UDTsttIAdH4TcP0l0MauSnLK8w==} engines: {node: '>=12'} + foreground-child@3.3.1: + resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} + engines: {node: '>=14'} + format@0.2.2: resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==} engines: {node: '>=0.4.x'} + formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + forwarded@0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -4310,6 +4355,22 @@ packages: function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + gaxios@7.1.3: + resolution: {integrity: sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==} + engines: {node: '>=18'} + + gaxios@7.2.0: + resolution: {integrity: sha512-CUVb4wcYe+771XevyH6HtGmXFAGGKkIC3kswAP8Z1JCe0j80JMaTPZH930DWFrvo0atjh18Arc0pEyUCWa5bfg==} + engines: {node: '>=18'} + + gcp-metadata@8.1.2: + resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==} + engines: {node: '>=18'} + + gcp-metadata@8.1.3: + resolution: {integrity: sha512-ziTrzUhhpL9Zk5k0HHzgP/KIpWDJT0VMBC/ynt/QIBvTW+UUcSivQRl6VlwTf/EilDxtSWklHoRsKy1c4k+59w==} + engines: {node: '>=18'} + gensync@1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} engines: {node: '>=6.9.0'} @@ -4344,6 +4405,11 @@ packages: resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} engines: {node: '>=10.13.0'} + glob@10.5.0: + resolution: {integrity: sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + hasBin: true + glob@13.0.6: resolution: {integrity: sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==} engines: {node: 18 || 20 || >=22} @@ -4356,6 +4422,26 @@ packages: resolution: {integrity: sha512-Czmyns5dUsq4seFBR/Kdydhmo8y9kC79hiSkPn0YcGtNnYWnrgt0vjrSjx9tspoDGWm2CMarffRuLjM4xUz8xg==} engines: {node: '>=18'} + google-auth-library@10.5.0: + resolution: {integrity: sha512-7ABviyMOlX5hIVD60YOfHw4/CxOfBhyduaYB+wbFWCWoni4N7SLcV46hrVRktuBbZjFC9ONyqamZITN7q3n32w==} + engines: {node: '>=18'} + + google-auth-library@10.9.0: + resolution: {integrity: sha512-xtvUqvINPhTaBm7nXqlYPcrMHJPm1lCNdSovxnKKhTm+4JsvQ+KGVYJViLoH9Yxu8w+T0Qv5HubzYT9BLrppJg==} + engines: {node: '>=18'} + + google-logging-utils@1.1.3: + resolution: {integrity: sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==} + engines: {node: '>=14'} + + googleapis-common@8.0.2: + resolution: {integrity: sha512-5MXeQzIZaqCH7B+HJWqhQm946VARpZep6acbWSr/fcgF2cQANq7allgX+i/G0EqF0WyUxB277gtWMzRYHMl9tg==} + engines: {node: '>=18.0.0'} + + googleapis@173.0.0: + resolution: {integrity: sha512-xEJJYLZ4qeenVyfzispNfRjCe9bsv7CzBv5zYFLvScOze9snJ8S9W6hjQ729CWPQt5mvn/JrcRaCHzQiukt0ng==} + engines: {node: '>=18'} + gopd@1.2.0: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} @@ -4404,6 +4490,10 @@ packages: peerDependencies: graphology-types: '>=0.24.0' + gtoken@8.0.0: + resolution: {integrity: sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==} + engines: {node: '>=18'} + h3@1.15.11: resolution: {integrity: sha512-L3THSe2MPeBwgIZVSH5zLdBBU90TOxarvhK9d04IDY2AmVS8j2Jz2LIWtwsGOU3lu2I5jCN7FNvVfY2+XyF+mg==} @@ -4617,6 +4707,9 @@ packages: isomorphic.js@0.2.5: resolution: {integrity: sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==} + jackspeak@3.4.3: + resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + jerrypick@1.1.2: resolution: {integrity: sha512-YKnxXEekXKzhpf7CLYA0A+oDP8V0OhICNCr5lv96FvSsDEmrb0GKM776JgQvHTMjr7DTTPEVv/1Ciaw0uEWzBA==} engines: {node: '>=12'} @@ -4659,6 +4752,9 @@ packages: engines: {node: '>=6'} hasBin: true + json-bigint@1.0.0: + resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} + json-buffer@3.0.1: resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} @@ -4703,6 +4799,12 @@ packages: engines: {node: '>=18.17'} hasBin: true + jwa@2.0.1: + resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} + + jws@4.0.1: + resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} + kapsule@1.16.3: resolution: {integrity: sha512-4+5mNNf4vZDSwPhKprKwz3330iisPrb08JyMgbsdFrimBCKNHecua/WBwvVg3n7vwx0C1ARjfhwIpbrbd9n5wg==} engines: {node: '>=12'} @@ -4842,6 +4944,9 @@ packages: loupe@3.2.1: resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} + lru-cache@10.4.3: + resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + lru-cache@11.3.6: resolution: {integrity: sha512-Gf/KoL3C/MlI7Bt0PGI9I+TeTC/I6r/csU58N4BSNc4lppLBeKsOdFYkK+dX0ABDUMJNfCHTyPpzwwO21Awd3A==} engines: {node: 20 || >=22} @@ -5078,6 +5183,10 @@ packages: minimatch@3.1.5: resolution: {integrity: sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==} + minimatch@9.0.9: + resolution: {integrity: sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==} + engines: {node: '>=16 || 14 >=14.17'} + minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} @@ -5154,6 +5263,11 @@ packages: node-abort-controller@3.1.1: resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + node-fetch-native@1.6.7: resolution: {integrity: sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==} @@ -5166,6 +5280,10 @@ packages: encoding: optional: true + node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + node-gyp-build-optional-packages@5.2.2: resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==} hasBin: true @@ -5281,6 +5399,9 @@ packages: resolution: {integrity: sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg==} engines: {node: '>=20'} + package-json-from-dist@1.0.1: + resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} + package-manager-detector@1.7.0: resolution: {integrity: sha512-xg1eHpwYL/D/HEdWw2goFZP6vV0FH7W+PZ5rFkGjdIDLtxq7EkzBUeT3m+lndYCt8wKbmofUu1MUdMCXkCk9ZQ==} @@ -5331,6 +5452,10 @@ packages: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} + path-scurry@1.11.1: + resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} + engines: {node: '>=16 || 14 >=14.18'} + path-scurry@2.0.2: resolution: {integrity: sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==} engines: {node: 18 || 20 || >=22} @@ -5769,6 +5894,10 @@ packages: rfdc@1.4.1: resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==} + rimraf@5.0.10: + resolution: {integrity: sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==} + hasBin: true + robust-predicates@3.0.3: resolution: {integrity: sha512-NS3levdsRIUOmiJ8FZWCP7LG3QpJyrs/TE0Zpf1yvZu8cAJJ6QMW92H1c7kWpdIHo8RvmLxN/o2JXTKHp74lUA==} @@ -5897,6 +6026,10 @@ packages: siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + simple-icons@16.25.0: resolution: {integrity: sha512-ReYPfbqjkTBovxjerHw2tf6E1rQdaCqNJKaJbd78Mui8w+78xQlE1aDO/2ekID++n8qa+1t+oaueYLeiSoQsIA==} engines: {node: '>=0.12.18'} @@ -5963,6 +6096,10 @@ packages: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} + string-width@5.1.2: + resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} + engines: {node: '>=12'} + string-width@7.2.0: resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} engines: {node: '>=18'} @@ -6316,6 +6453,9 @@ packages: uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + url-template@2.0.8: + resolution: {integrity: sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==} + use-sync-external-store@1.6.0: resolution: {integrity: sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==} peerDependencies: @@ -6555,6 +6695,10 @@ packages: resolution: {integrity: sha512-Ezr98sqXW/+OCGoUEXuOKVR+oVFlSdn1tIySEEJdiSAw4IjrW8hQkwARSSBJTSB5Us5dnytDgL0ZDliAYBhaNA==} engines: {node: '>=10.0.0'} + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -6604,6 +6748,10 @@ packages: resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} engines: {node: '>=10'} + wrap-ansi@8.1.0: + resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} + engines: {node: '>=12'} + wrap-ansi@9.0.2: resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} engines: {node: '>=18'} @@ -8208,6 +8356,15 @@ snapshots: '@ioredis/commands@1.10.0': {} + '@isaacs/cliui@8.0.2': + dependencies: + string-width: 5.1.2 + string-width-cjs: string-width@4.2.3 + strip-ansi: 7.2.0 + strip-ansi-cjs: strip-ansi@6.0.1 + wrap-ansi: 8.1.0 + wrap-ansi-cjs: wrap-ansi@7.0.0 + '@isaacs/fs-minipass@4.0.1': dependencies: minipass: 7.1.3 @@ -8881,6 +9038,9 @@ snapshots: '@pinojs/redact@0.4.0': {} + '@pkgjs/parseargs@0.11.0': + optional: true + '@prosemirror-adapter/core@0.5.3': dependencies: '@ocavue/utils': 1.6.0 @@ -10127,6 +10287,8 @@ snapshots: dependencies: require-from-string: 2.0.2 + bignumber.js@9.3.1: {} + bindings@1.5.0: dependencies: file-uri-to-path: 1.0.0 @@ -10156,6 +10318,10 @@ snapshots: balanced-match: 1.0.2 concat-map: 0.0.1 + brace-expansion@2.1.2: + dependencies: + balanced-match: 1.0.2 + brace-expansion@5.0.7: dependencies: balanced-match: 4.0.4 @@ -10168,6 +10334,8 @@ snapshots: node-releases: 2.0.50 update-browserslist-db: 1.2.3(browserslist@4.28.5) + buffer-equal-constant-time@1.0.1: {} + buffer-from@1.1.2: {} bullmq@5.76.8: @@ -10605,6 +10773,8 @@ snapshots: dependencies: accessor-fn: 1.5.3 + data-uri-to-buffer@4.0.1: {} + data-urls@7.0.0: dependencies: whatwg-mimetype: 5.0.0 @@ -10720,6 +10890,12 @@ snapshots: es-errors: 1.3.0 gopd: 1.2.0 + eastasianwidth@0.2.0: {} + + ecdsa-sig-formatter@1.0.11: + dependencies: + safe-buffer: 5.1.2 + ee-first@1.1.1: {} electron-to-chromium@1.5.389: {} @@ -10733,6 +10909,8 @@ snapshots: emoji-regex@8.0.0: {} + emoji-regex@9.2.2: {} + encodeurl@2.0.0: {} encoding-sniffer@0.2.1: @@ -11100,6 +11278,11 @@ snapshots: optionalDependencies: picomatch: 4.0.5 + fetch-blob@3.2.0: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + fflate@0.8.3: {} file-entry-cache@8.0.0: @@ -11171,8 +11354,17 @@ snapshots: kapsule: 1.16.3 lodash-es: 4.18.1 + foreground-child@3.3.1: + dependencies: + cross-spawn: 7.0.6 + signal-exit: 4.1.0 + format@0.2.2: {} + formdata-polyfill@4.0.10: + dependencies: + fetch-blob: 3.2.0 + forwarded@0.2.0: {} fresh@2.0.0: {} @@ -11185,6 +11377,39 @@ snapshots: function-bind@1.1.2: {} + gaxios@7.1.3: + dependencies: + extend: 3.0.2 + https-proxy-agent: 7.0.6 + node-fetch: 3.3.2 + rimraf: 5.0.10 + transitivePeerDependencies: + - supports-color + + gaxios@7.2.0: + dependencies: + extend: 3.0.2 + https-proxy-agent: 7.0.6 + node-fetch: 3.3.2 + transitivePeerDependencies: + - supports-color + + gcp-metadata@8.1.2: + dependencies: + gaxios: 7.2.0 + google-logging-utils: 1.1.3 + json-bigint: 1.0.0 + transitivePeerDependencies: + - supports-color + + gcp-metadata@8.1.3: + dependencies: + gaxios: 7.1.3 + google-logging-utils: 1.1.3 + json-bigint: 1.0.0 + transitivePeerDependencies: + - supports-color + gensync@1.0.0-beta.2: {} get-caller-file@2.0.5: {} @@ -11223,6 +11448,15 @@ snapshots: dependencies: is-glob: 4.0.3 + glob@10.5.0: + dependencies: + foreground-child: 3.3.1 + jackspeak: 3.4.3 + minimatch: 9.0.9 + minipass: 7.1.3 + package-json-from-dist: 1.0.1 + path-scurry: 1.11.1 + glob@13.0.6: dependencies: minimatch: 10.2.5 @@ -11233,6 +11467,49 @@ snapshots: globals@17.7.0: {} + google-auth-library@10.5.0: + dependencies: + base64-js: 1.5.1 + ecdsa-sig-formatter: 1.0.11 + gaxios: 7.1.3 + gcp-metadata: 8.1.3 + google-logging-utils: 1.1.3 + gtoken: 8.0.0 + jws: 4.0.1 + transitivePeerDependencies: + - supports-color + + google-auth-library@10.9.0: + dependencies: + base64-js: 1.5.1 + ecdsa-sig-formatter: 1.0.11 + gaxios: 7.2.0 + gcp-metadata: 8.1.2 + google-logging-utils: 1.1.3 + jws: 4.0.1 + transitivePeerDependencies: + - supports-color + + google-logging-utils@1.1.3: {} + + googleapis-common@8.0.2: + dependencies: + extend: 3.0.2 + gaxios: 7.1.3 + google-auth-library: 10.5.0 + google-logging-utils: 1.1.3 + qs: 6.15.2 + url-template: 2.0.8 + transitivePeerDependencies: + - supports-color + + googleapis@173.0.0: + dependencies: + google-auth-library: 10.9.0 + googleapis-common: 8.0.2 + transitivePeerDependencies: + - supports-color + gopd@1.2.0: {} gpt-tokenizer@3.4.0: {} @@ -11287,6 +11564,13 @@ snapshots: events: 3.3.0 graphology-types: 0.24.8 + gtoken@8.0.0: + dependencies: + gaxios: 7.1.3 + jws: 4.0.1 + transitivePeerDependencies: + - supports-color + h3@1.15.11: dependencies: cookie-es: 1.2.3 @@ -11545,6 +11829,12 @@ snapshots: isomorphic.js@0.2.5: {} + jackspeak@3.4.3: + dependencies: + '@isaacs/cliui': 8.0.2 + optionalDependencies: + '@pkgjs/parseargs': 0.11.0 + jerrypick@1.1.2: {} jiti@2.7.0: {} @@ -11591,6 +11881,10 @@ snapshots: jsesc@3.1.0: {} + json-bigint@1.0.0: + dependencies: + bignumber.js: 9.3.1 + json-buffer@3.0.1: {} json-schema-ref-resolver@3.0.0: @@ -11634,6 +11928,17 @@ snapshots: slick: 1.12.2 web-resource-inliner: 8.0.0 + jwa@2.0.1: + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.1.2 + + jws@4.0.1: + dependencies: + jwa: 2.0.1 + safe-buffer: 5.1.2 + kapsule@1.16.3: dependencies: lodash-es: 4.18.1 @@ -11752,6 +12057,8 @@ snapshots: loupe@3.2.1: {} + lru-cache@10.4.3: {} + lru-cache@11.3.6: {} lru-cache@11.5.2: {} @@ -12220,6 +12527,10 @@ snapshots: dependencies: brace-expansion: 1.1.14 + minimatch@9.0.9: + dependencies: + brace-expansion: 2.1.2 + minimist@1.2.8: {} minipass@7.1.3: {} @@ -12288,12 +12599,20 @@ snapshots: node-abort-controller@3.1.1: {} + node-domexception@1.0.0: {} + node-fetch-native@1.6.7: {} node-fetch@2.7.0: dependencies: whatwg-url: 5.0.0 + node-fetch@3.3.2: + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + node-gyp-build-optional-packages@5.2.2: dependencies: detect-libc: 2.1.2 @@ -12388,6 +12707,8 @@ snapshots: p-timeout@7.0.1: {} + package-json-from-dist@1.0.1: {} + package-manager-detector@1.7.0: {} pako@1.0.11: {} @@ -12438,6 +12759,11 @@ snapshots: path-key@3.1.1: {} + path-scurry@1.11.1: + dependencies: + lru-cache: 10.4.3 + minipass: 7.1.3 + path-scurry@2.0.2: dependencies: lru-cache: 11.5.2 @@ -12963,6 +13289,10 @@ snapshots: rfdc@1.4.1: {} + rimraf@5.0.10: + dependencies: + glob: 10.5.0 + robust-predicates@3.0.3: {} rollup@4.62.2: @@ -13163,6 +13493,8 @@ snapshots: siginfo@2.0.0: {} + signal-exit@4.1.0: {} + simple-icons@16.25.0: {} sisteransi@1.0.5: {} @@ -13218,6 +13550,12 @@ snapshots: is-fullwidth-code-point: 3.0.0 strip-ansi: 6.0.1 + string-width@5.1.2: + dependencies: + eastasianwidth: 0.2.0 + emoji-regex: 9.2.2 + strip-ansi: 7.2.0 + string-width@7.2.0: dependencies: emoji-regex: 10.6.0 @@ -13521,6 +13859,8 @@ snapshots: dependencies: punycode: 2.3.1 + url-template@2.0.8: {} + use-sync-external-store@1.6.0(react@19.2.7): dependencies: react: 19.2.7 @@ -13858,6 +14198,8 @@ snapshots: mime: 2.6.0 valid-data-url: 3.0.1 + web-streams-polyfill@3.3.3: {} + webidl-conversions@3.0.1: {} webidl-conversions@8.0.1: {} @@ -13902,6 +14244,12 @@ snapshots: string-width: 4.2.3 strip-ansi: 6.0.1 + wrap-ansi@8.1.0: + dependencies: + ansi-styles: 6.2.3 + string-width: 5.1.2 + strip-ansi: 7.2.0 + wrap-ansi@9.0.2: dependencies: ansi-styles: 6.2.3 diff --git a/scripts/check-licenses.mjs b/scripts/check-licenses.mjs index d93c941a6..effe18aed 100755 --- a/scripts/check-licenses.mjs +++ b/scripts/check-licenses.mjs @@ -79,6 +79,10 @@ const OVERRIDES = { '@sentry/cli-win32-arm64': 'MIT', '@sentry/cli-win32-i686': 'MIT', '@sentry/cli-win32-x64': 'MIT', + // url-template -> BSD-3-Clause (its bundled LICENSE is the 3-clause BSD text; + // package.json reports the legacy non-SPDX "BSD"). Transitive dep pulled in by + // googleapis via googleapis-common. + 'url-template': 'BSD-3-Clause', }; function isAllowedAtom(atom) { From 03bc143562643c55b9525dfc1d731a8050393d4e Mon Sep 17 00:00:00 2001 From: Jason-jo17 Date: Mon, 13 Jul 2026 20:10:24 +0530 Subject: [PATCH 2/2] refactor(drive): drop unfinished push webhook + normalize per review Addresses review on #96: - Remove the /api/drive/webhook route, its public auth-plugin exemption, the DRIVE_WEBHOOK_SECRET env var + driveWebhookSecret() helper, and the drive_channel_id / drive_channel_expires_at / drive_page_token columns (schema + migration 0073). None of it was wired (no changes.watch was ever registered), so it was dead, public, unauthenticated code. The working manual/poll sync path stays; Drive push-sync can be a follow-up. Also drops the now-unused changes-API helpers (listChanges / getStartPageToken / getFileParents). - Normalize attachments.format: binaries are limited to docx/pdf (the formats the attachment pipeline models); other accepted types are skipped with a log rather than stored with a foreign extension. Default accepted types are now text + docx/pdf. - Log swallowed per-file sync errors (console.warn) for debuggability. @boppl/api typechecks clean; schema <-> migration 0073 match column-for-column. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Jason-jo17 --- .../drizzle/migrations/0073_drive_sync.sql | 3 -- apps/api/src/config/env.ts | 3 -- apps/api/src/db/schema.ts | 5 -- apps/api/src/lib/drive-sync.ts | 20 ++++++-- apps/api/src/lib/google-drive.ts | 50 ++----------------- apps/api/src/plugins/auth.ts | 3 -- apps/api/src/queue/drive-sync.ts | 8 +-- apps/api/src/routes/drive.ts | 17 ------- 8 files changed, 24 insertions(+), 85 deletions(-) diff --git a/apps/api/drizzle/migrations/0073_drive_sync.sql b/apps/api/drizzle/migrations/0073_drive_sync.sql index e8b22f2e7..98d89113c 100644 --- a/apps/api/drizzle/migrations/0073_drive_sync.sql +++ b/apps/api/drizzle/migrations/0073_drive_sync.sql @@ -18,9 +18,6 @@ CREATE TABLE IF NOT EXISTS "drive_folder_links" ( "direction" text NOT NULL DEFAULT 'both', "accepted_types" text[] NOT NULL DEFAULT ARRAY[]::text[], "conflict_policy" text NOT NULL DEFAULT 'manual', - "drive_page_token" text, - "drive_channel_id" text, - "drive_channel_expires_at" timestamptz, "last_synced_at" timestamptz, "status" text NOT NULL DEFAULT 'active', "error_message" text, diff --git a/apps/api/src/config/env.ts b/apps/api/src/config/env.ts index eb7eabb10..593167b6c 100644 --- a/apps/api/src/config/env.ts +++ b/apps/api/src/config/env.ts @@ -238,9 +238,6 @@ const envSchema = z.object({ GOOGLE_DRIVE_SCOPE: z.enum(['drive.file', 'drive']).default('drive.file'), // Skip Drive files larger than this many megabytes when syncing. DRIVE_SYNC_MAX_FILE_MB: z.coerce.number().int().positive().default(50), - // Signs the per-link Drive push-notification channel tokens. Falls back to - // SECRETBOX_MASTER_KEY when unset (see lib/google-drive.ts). - DRIVE_WEBHOOK_SECRET: z.string().min(1).optional(), // Phase C — internal meeting-bot controller (the express service that asks // Recall to send the bot). Defaults to the Docker-network address; the API diff --git a/apps/api/src/db/schema.ts b/apps/api/src/db/schema.ts index 8c98d48e5..bbfc26a1e 100644 --- a/apps/api/src/db/schema.ts +++ b/apps/api/src/db/schema.ts @@ -1707,11 +1707,6 @@ export const driveFolderLinks = pgTable( acceptedTypes: text('accepted_types').array().notNull().default(sql`ARRAY[]::text[]`), // 'lww' (last-writer-wins by modified time) or 'manual' (flag conflicts, never overwrite). conflictPolicy: text('conflict_policy').notNull().default('manual'), - // Drive changes-API cursor for incremental pulls. - drivePageToken: text('drive_page_token'), - // Drive push-notification channel (id + expiry) so we can stop/renew it. - driveChannelId: text('drive_channel_id'), - driveChannelExpiresAt: timestamp('drive_channel_expires_at', { withTimezone: true }), lastSyncedAt: timestamp('last_synced_at', { withTimezone: true }), // 'active' | 'paused' | 'error' status: text('status').notNull().default('active'), diff --git a/apps/api/src/lib/drive-sync.ts b/apps/api/src/lib/drive-sync.ts index a9f68b3d6..288f54ccd 100644 --- a/apps/api/src/lib/drive-sync.ts +++ b/apps/api/src/lib/drive-sync.ts @@ -204,15 +204,25 @@ export async function pullLink(link: DriveLink): Promise { } try { - const bytes = await downloadDriveFile(drive, f.id); if (isTextExt(ext)) { + const bytes = await downloadDriveFile(drive, f.id); await upsertDocFromDrive(link, f, bytes.toString('utf8'), mapping); - } else { + } else if (ext === 'pdf' || ext === 'docx') { + // Binaries land in `attachments`, whose `format` is docx|pdf (v1). Other + // binary types are skipped rather than stored with a foreign format. if (!isR2Configured()) { res.skipped++; continue; } // no blob store → can't hold binaries + const bytes = await downloadDriveFile(drive, f.id); await upsertAttachmentFromDrive(link, f, ext, bytes, mapping); + } else { + // eslint-disable-next-line no-console + console.warn(`[drive-sync] link=${link.id} skipping unsupported type "${ext}" (${f.name})`); + res.skipped++; + continue; } if (mapping) res.updated++; else res.created++; - } catch { + } catch (err) { + // eslint-disable-next-line no-console + console.warn(`[drive-sync] link=${link.id} pull failed for "${f.name}":`, err instanceof Error ? err.message : err); res.skipped++; } } @@ -265,7 +275,9 @@ export async function pushLink(link: DriveLink): Promise { ); } res.pushed++; - } catch { + } catch (err) { + // eslint-disable-next-line no-console + console.warn(`[drive-sync] link=${link.id} push failed for doc "${d.title}":`, err instanceof Error ? err.message : err); res.skipped++; } } diff --git a/apps/api/src/lib/google-drive.ts b/apps/api/src/lib/google-drive.ts index 3bb1644d0..bffd2476d 100644 --- a/apps/api/src/lib/google-drive.ts +++ b/apps/api/src/lib/google-drive.ts @@ -23,10 +23,10 @@ const SCOPE_URLS: Record<'drive.file' | 'drive', string> = { drive: 'https://www.googleapis.com/auth/drive', }; -/** File extensions synced by default when a link doesn't set its own allow-list. */ -export const DEFAULT_ACCEPTED_TYPES = [ - 'md', 'markdown', 'txt', 'pdf', 'doc', 'docx', 'csv', 'png', 'jpg', 'jpeg', -]; +// File extensions synced by default when a link doesn't set its own allow-list. +// v1 syncs text (→ Mnema docs) and docx/pdf (→ attachments, the formats the +// attachment pipeline models). More binary types can follow as a separate change. +export const DEFAULT_ACCEPTED_TYPES = ['md', 'markdown', 'txt', 'pdf', 'docx']; /** Extensions that become Mnema docs (text). Everything else becomes an attachment. */ const TEXT_EXTS = new Set(['md', 'markdown', 'txt']); @@ -51,11 +51,6 @@ export function driveScopeUrl(): string { return SCOPE_URLS[config.GOOGLE_DRIVE_SCOPE]; } -/** Secret used to sign Drive push-notification channel tokens. */ -export function driveWebhookSecret(): string { - return config.DRIVE_WEBHOOK_SECRET ?? config.SECRETBOX_MASTER_KEY; -} - export function extOf(name: string): string { const i = name.lastIndexOf('.'); return i >= 0 ? name.slice(i + 1).toLowerCase() : ''; @@ -165,43 +160,6 @@ export async function listFilesInFolder(drive: DriveClient, driveFolderId: strin return out; } -/** The current changes-API start token (cursor) for incremental pulls. */ -export async function getStartPageToken(drive: DriveClient): Promise { - const res = await drive.changes.getStartPageToken({}); - return res.data.startPageToken!; -} - -export interface DriveChangePage { files: DriveFile[]; removedIds: string[]; newPageToken: string } - -/** Fetch the batch of changes since `pageToken`. Caller filters to linked folders. */ -export async function listChanges(drive: DriveClient, pageToken: string): Promise { - const files: DriveFile[] = []; - const removedIds: string[] = []; - let token: string | undefined = pageToken; - let newPageToken = pageToken; - do { - const { data }: { data: drive_v3.Schema$ChangeList } = await drive.changes.list({ - pageToken: token, - fields: `nextPageToken, newStartPageToken, changes(removed, fileId, file(${FILE_FIELDS}, parents))`, - pageSize: 200, - spaces: 'drive', - }); - for (const c of data.changes ?? []) { - if (c.removed || c.file?.trashed) { if (c.fileId) removedIds.push(c.fileId); continue; } - if (c.file?.id && c.file.mimeType !== 'application/vnd.google-apps.folder') files.push(toDriveFile(c.file)); - } - if (data.newStartPageToken) newPageToken = data.newStartPageToken; - token = data.nextPageToken ?? undefined; - } while (token); - return { files, removedIds, newPageToken }; -} - -/** The parent folder ids of a Drive file (to test membership of a linked folder). */ -export async function getFileParents(drive: DriveClient, fileId: string): Promise { - const res = await drive.files.get({ fileId, fields: 'parents' }); - return res.data.parents ?? []; -} - /** Download a Drive file's bytes. */ export async function downloadDriveFile(drive: DriveClient, fileId: string): Promise { const res = await drive.files.get( diff --git a/apps/api/src/plugins/auth.ts b/apps/api/src/plugins/auth.ts index 9d47b7df6..2b109d680 100644 --- a/apps/api/src/plugins/auth.ts +++ b/apps/api/src/plugins/auth.ts @@ -95,9 +95,6 @@ export const authPlugin: FastifyPluginAsync = fp(async (app) => { // /callback auth: OnlyOffice JWT signature. /file auth: tenantId query param. if (url === '/api/onlyoffice/callback') return; if (url.startsWith('/api/onlyoffice/') && url.endsWith('/file')) return; - // Google Drive push notifications — server-to-server POST from Google with no - // cookie. Authenticated in-handler by matching the stored channel id. - if (url === '/api/drive/webhook') return; const token = req.cookies[JWT_COOKIE_NAME] ?? extractBearer(req.headers.authorization); if (!token) { diff --git a/apps/api/src/queue/drive-sync.ts b/apps/api/src/queue/drive-sync.ts index 5c050401f..8bc637436 100644 --- a/apps/api/src/queue/drive-sync.ts +++ b/apps/api/src/queue/drive-sync.ts @@ -1,9 +1,9 @@ /** * BullMQ queue for Google Drive folder sync (Phase 10). * - * Enqueued from routes/drive.ts (manual "Sync now" + the push webhook) and the - * reconcile cron. The drive-sync worker runs syncLink() for the link. A stable - * jobId per link debounces concurrent syncs of the same folder. + * Enqueued from routes/drive.ts (manual "Sync now"); a reconcile cron can enqueue + * with reason 'cron' as a follow-up. The drive-sync worker runs syncLink() for the + * link. A stable jobId per link debounces concurrent syncs of the same folder. */ import { Queue } from 'bullmq'; import { redisConnection } from '../lib/redis.js'; @@ -12,7 +12,7 @@ export const DRIVE_SYNC_QUEUE_NAME = 'drive-sync'; export interface DriveSyncJobData { linkId: string; - reason: 'manual' | 'webhook' | 'cron' | 'push'; + reason: 'manual' | 'cron'; } export const driveSyncQueue = new Queue(DRIVE_SYNC_QUEUE_NAME, { diff --git a/apps/api/src/routes/drive.ts b/apps/api/src/routes/drive.ts index 12df33310..ee3a1d620 100644 --- a/apps/api/src/routes/drive.ts +++ b/apps/api/src/routes/drive.ts @@ -10,7 +10,6 @@ * PATCH /api/drive/links/:id → edit types / direction / conflict policy / pause * DELETE /api/drive/links/:id → unlink * POST /api/drive/links/:id/sync → enqueue a sync now - * POST /api/drive/webhook → Google push-notification receiver (public + token) * * The refresh token is user-scoped (their Google Drive); links + mappings are * workspace-scoped and enforced app-layer (filter by workspace_id + requireRole). @@ -253,22 +252,6 @@ export const driveRoutes: FastifyPluginAsync = async (app) => { return reply.send({ ok: true, queued: true }); }); - // ── Google push-notification receiver ────────────────────────────────────── - // Public route (no auth cookie survives Google's POST). We authenticate by - // matching the channel id we stored on the link when registering the watch. - app.post('/api/drive/webhook', async (req, reply) => { - const channelId = req.headers['x-goog-channel-id']; - const state = req.headers['x-goog-resource-state']; - if (typeof channelId !== 'string') return reply.code(204).send(); - // 'sync' is Google's initial handshake ping — ack and ignore. - if (state === 'sync') return reply.code(204).send(); - const [link] = await db.select({ id: driveFolderLinks.id }).from(driveFolderLinks) - .where(eq(driveFolderLinks.driveChannelId, channelId)) - .limit(1); - if (link) await enqueueDriveSync({ linkId: link.id, reason: 'webhook' }); - return reply.code(204).send(); - }); - // ── Conflicts for a link (for the resolver UI) ───────────────────────────── app.get('/api/drive/links/:id/conflicts', async (req, reply) => { if (!req.auth) return reply.code(401).send({ error: 'unauthorized' });