diff --git a/apps/api/drizzle/migrations/0073_drive_sync.sql b/apps/api/drizzle/migrations/0073_drive_sync.sql new file mode 100644 index 000000000..98d89113c --- /dev/null +++ b/apps/api/drizzle/migrations/0073_drive_sync.sql @@ -0,0 +1,51 @@ +-- 0073 — Google Drive folder sync (Phase 10, core). +-- Adds a per-member encrypted Drive refresh token and two tables: +-- drive_folder_links — a Mnema folder ⇄ Google Drive folder pairing + settings +-- drive_file_mappings — per-file idempotency + conflict tracking +-- Core migration (folders/docs/attachments are core tables) — applies on self-host. +-- Idempotent (IF NOT EXISTS everywhere); a re-run on restart is safe. + +-- Per-member encrypted Google Drive refresh token (mirrors calendar_refresh_token). +ALTER TABLE "workspace_members" ADD COLUMN IF NOT EXISTS "drive_refresh_token" text; + +CREATE TABLE IF NOT EXISTS "drive_folder_links" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid(), + "workspace_id" uuid NOT NULL REFERENCES "workspaces"("id") ON DELETE CASCADE, + "folder_id" uuid NOT NULL REFERENCES "folders"("id") ON DELETE CASCADE, + "drive_folder_id" text NOT NULL, + "drive_folder_name" text, + "connected_by" uuid NOT NULL REFERENCES "users"("id") ON DELETE CASCADE, + "direction" text NOT NULL DEFAULT 'both', + "accepted_types" text[] NOT NULL DEFAULT ARRAY[]::text[], + "conflict_policy" text NOT NULL DEFAULT 'manual', + "last_synced_at" timestamptz, + "status" text NOT NULL DEFAULT 'active', + "error_message" text, + "created_at" timestamptz NOT NULL DEFAULT now(), + "updated_at" timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS "drive_links_workspace_idx" ON "drive_folder_links" ("workspace_id"); +CREATE UNIQUE INDEX IF NOT EXISTS "drive_links_folder_unique" ON "drive_folder_links" ("folder_id"); +CREATE UNIQUE INDEX IF NOT EXISTS "drive_links_ws_drivefolder_unique" ON "drive_folder_links" ("workspace_id", "drive_folder_id"); + +CREATE TABLE IF NOT EXISTS "drive_file_mappings" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid(), + "link_id" uuid NOT NULL REFERENCES "drive_folder_links"("id") ON DELETE CASCADE, + "drive_file_id" text NOT NULL, + "drive_name" text, + "doc_id" uuid REFERENCES "docs"("id") ON DELETE SET NULL, + "attachment_id" uuid REFERENCES "attachments"("id") ON DELETE SET NULL, + "drive_md5" text, + "content_hash" text, + "drive_modified_at" timestamptz, + "mnema_modified_at" timestamptz, + "sync_state" text NOT NULL DEFAULT 'synced', + "created_at" timestamptz NOT NULL DEFAULT now(), + "updated_at" timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS "drive_map_link_idx" ON "drive_file_mappings" ("link_id"); +CREATE UNIQUE INDEX IF NOT EXISTS "drive_map_link_file_unique" ON "drive_file_mappings" ("link_id", "drive_file_id"); +CREATE INDEX IF NOT EXISTS "drive_map_doc_idx" ON "drive_file_mappings" ("doc_id"); +CREATE INDEX IF NOT EXISTS "drive_map_attachment_idx" ON "drive_file_mappings" ("attachment_id"); diff --git a/apps/api/drizzle/migrations/meta/_journal.json b/apps/api/drizzle/migrations/meta/_journal.json index 993e7cd03..3eadcd6a3 100644 --- a/apps/api/drizzle/migrations/meta/_journal.json +++ b/apps/api/drizzle/migrations/meta/_journal.json @@ -505,6 +505,13 @@ "when": 1784900000000, "tag": "0070_task_comments", "breakpoints": true + }, + { + "idx": 72, + "version": "7", + "when": 1785000000000, + "tag": "0073_drive_sync", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/package.json b/apps/api/package.json index 7c703ac62..092aad4fc 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -50,12 +50,14 @@ "@workos-inc/node": "^10.7.0", "ai": "^6.0.221", "bullmq": "5.76.8", + "chart.js": "^4.4.7", "diff": "^9.0.0", "dompurify": "3.4.11", "dotenv": "^17.4.2", "drizzle-orm": "^0.45.2", "fastify": "^5.10.0", "fastify-plugin": "^5.0.0", + "googleapis": "^173.0.0", "gpt-tokenizer": "^3.4.0", "graphology": "^0.26.0", "graphology-communities-louvain": "^2.0.2", @@ -68,6 +70,7 @@ "juice": "^11.1.1", "mammoth": "^1.12.0", "marked": "^18.0.5", + "mdast2docx": "1.6.1", "mermaid": "11.16.0", "mime-types": "^3.0.2", "nanoid": "^5.1.16", @@ -77,16 +80,14 @@ "pino-pretty": "^13.1.3", "playwright": "^1.61.1", "postgres": "^3.4.0", + "remark-parse": "^11.0.0", "resend": "^6.17.1", "turndown": "^7.2.4", "turndown-plugin-gfm": "^1.0.2", + "unified": "^11.0.5", "voyageai": "^0.4.0", "yjs": "^13.6.31", - "zod": "^3.23.0", - "mdast2docx": "1.6.1", - "unified": "^11.0.5", - "remark-parse": "^11.0.0", - "chart.js": "^4.4.7" + "zod": "^3.23.0" }, "devDependencies": { "@types/node": "^22.20.1", diff --git a/apps/api/src/config/env.ts b/apps/api/src/config/env.ts index cb561fd7e..593167b6c 100644 --- a/apps/api/src/config/env.ts +++ b/apps/api/src/config/env.ts @@ -224,6 +224,21 @@ const envSchema = z.object({ GOOGLE_CALENDAR_CLIENT_SECRET: z.string().min(1).optional(), GOOGLE_CALENDAR_REDIRECT_URI: z.string().url().optional(), + // Phase 10 — Google Drive folder sync (optional; drive routes return 503 when + // unset). Bring-your-own OAuth client, same shape as calendar above. The + // redirect URI must equal the /api/drive/callback URL and be registered as an + // authorised redirect URI in Google Cloud Console. + GOOGLE_DRIVE_CLIENT_ID: z.string().min(1).optional(), + GOOGLE_DRIVE_CLIENT_SECRET: z.string().min(1).optional(), + GOOGLE_DRIVE_REDIRECT_URI: z.string().url().optional(), + // 'drive.file' (least privilege — Mnema only sees files it created or the user + // explicitly opens via the picker) or 'drive' (full — lets users pull + // pre-existing Drive folders; requires Google's restricted-scope review to ship + // to real users in production). + GOOGLE_DRIVE_SCOPE: z.enum(['drive.file', 'drive']).default('drive.file'), + // Skip Drive files larger than this many megabytes when syncing. + DRIVE_SYNC_MAX_FILE_MB: z.coerce.number().int().positive().default(50), + // Phase C — internal meeting-bot controller (the express service that asks // Recall to send the bot). Defaults to the Docker-network address; the API // POSTs /join here when an admitted meeting is starting soon. diff --git a/apps/api/src/db/schema.ts b/apps/api/src/db/schema.ts index afdb55881..bbfc26a1e 100644 --- a/apps/api/src/db/schema.ts +++ b/apps/api/src/db/schema.ts @@ -105,6 +105,8 @@ export const workspaceMembers = pgTable( joinedAt: timestamp('joined_at', { withTimezone: true }).notNull().defaultNow(), // Phase C — encrypted Google Calendar refresh token (per member). calendarRefreshToken: text('calendar_refresh_token'), + // Phase 10 — encrypted Google Drive refresh token (per member). + driveRefreshToken: text('drive_refresh_token'), }, (table) => ({ pk: primaryKey({ columns: [table.workspaceId, table.userId] }), @@ -1681,6 +1683,73 @@ export const attachments = pgTable( }), ); +// ─── GOOGLE DRIVE SYNC ──────────────────────────────────────────────────────── +// Phase 10 — links a Mnema folder to a Google Drive folder and tracks per-file +// sync state. Core-only: references folders/docs/attachments/users/workspaces, +// no enterprise coupling. The encrypted refresh token lives on workspace_members +// (drive_refresh_token, above); these two tables hold the links + file mappings. + +export const driveFolderLinks = pgTable( + 'drive_folder_links', + { + id: uuid('id').primaryKey().default(sql`gen_random_uuid()`), + workspaceId: uuid('workspace_id').notNull().references(() => workspaces.id, { onDelete: 'cascade' }), + // The Mnema folder that mirrors a Drive folder. + folderId: uuid('folder_id').notNull().references(() => folders.id, { onDelete: 'cascade' }), + // The Google Drive folder id + display name (for the UI). + driveFolderId: text('drive_folder_id').notNull(), + driveFolderName: text('drive_folder_name'), + // Whose encrypted Drive token drives the sync for this link. + connectedBy: uuid('connected_by').notNull().references(() => users.id, { onDelete: 'cascade' }), + // 'pull' (Drive→Mnema), 'push' (Mnema→Drive), or 'both'. + direction: text('direction').notNull().default('both'), + // Allow-listed extensions synced (lowercase, no dot). Empty = built-in defaults. + acceptedTypes: text('accepted_types').array().notNull().default(sql`ARRAY[]::text[]`), + // 'lww' (last-writer-wins by modified time) or 'manual' (flag conflicts, never overwrite). + conflictPolicy: text('conflict_policy').notNull().default('manual'), + lastSyncedAt: timestamp('last_synced_at', { withTimezone: true }), + // 'active' | 'paused' | 'error' + status: text('status').notNull().default('active'), + errorMessage: text('error_message'), + createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(), + }, + (table) => ({ + workspaceIdx: index('drive_links_workspace_idx').on(table.workspaceId), + folderUnique: uniqueIndex('drive_links_folder_unique').on(table.folderId), + driveFolderUnique: uniqueIndex('drive_links_ws_drivefolder_unique').on(table.workspaceId, table.driveFolderId), + }), +); + +export const driveFileMappings = pgTable( + 'drive_file_mappings', + { + id: uuid('id').primaryKey().default(sql`gen_random_uuid()`), + linkId: uuid('link_id').notNull().references(() => driveFolderLinks.id, { onDelete: 'cascade' }), + // The Google Drive file id. + driveFileId: text('drive_file_id').notNull(), + driveName: text('drive_name'), + // Where it landed in Mnema — a doc (md/txt) or an attachment (binary). One is set. + docId: uuid('doc_id').references(() => docs.id, { onDelete: 'set null' }), + attachmentId: uuid('attachment_id').references(() => attachments.id, { onDelete: 'set null' }), + // Drive checksum (md5Checksum) + the Mnema-side content hash, for change detection. + driveMd5: text('drive_md5'), + contentHash: text('content_hash'), + driveModifiedAt: timestamp('drive_modified_at', { withTimezone: true }), + mnemaModifiedAt: timestamp('mnema_modified_at', { withTimezone: true }), + // 'synced' | 'pending_push' | 'pending_pull' | 'conflict' + syncState: text('sync_state').notNull().default('synced'), + createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(), + }, + (table) => ({ + linkIdx: index('drive_map_link_idx').on(table.linkId), + driveFileUnique: uniqueIndex('drive_map_link_file_unique').on(table.linkId, table.driveFileId), + docIdx: index('drive_map_doc_idx').on(table.docId), + attachmentIdx: index('drive_map_attachment_idx').on(table.attachmentId), + }), +); + // ─── KNOWLEDGE GRAPH ────────────────────────────────────────────────────────── export const graphNodes = pgTable( diff --git a/apps/api/src/lib/drive-sync.ts b/apps/api/src/lib/drive-sync.ts new file mode 100644 index 000000000..288f54ccd --- /dev/null +++ b/apps/api/src/lib/drive-sync.ts @@ -0,0 +1,301 @@ +/** + * Phase 10 — Google Drive ⇄ Mnema sync engine (shared by routes/drive.ts and the + * drive-sync worker). + * + * pullLink(link) — Drive → Mnema: create/update docs (text) + attachments (binary) + * pushLink(link) — Mnema → Drive: export the linked folder's docs as files + * syncLink(link) — run both directions according to link.direction + * + * All writes go through withSystemPrivilege (server-side, RLS-bypassing) and are + * explicitly scoped by workspace_id. Sync is idempotent, keyed on + * drive_file_mappings (link_id, drive_file_id) and the file md5 checksum. + */ +import { randomUUID } from 'node:crypto'; +import { PutObjectCommand } from '@aws-sdk/client-s3'; +import { markdownToYjsState } from '@boppl/schema/node'; +import { and, eq, isNotNull } from 'drizzle-orm'; +import { config } from '../config/env.js'; +import { withSystemPrivilege } from '../db/with-system-privilege.js'; +import { + attachments, docs, driveFileMappings, driveFolderLinks, workspaceMembers, +} from '../db/schema.js'; +import { + downloadDriveFile, driveClientFromRefresh, extOf, isAcceptedType, isTextExt, + listFilesInFolder, mimeForExt, updateDriveFile, uploadToFolder, + type DriveClient, type DriveFile, +} from './google-drive.js'; +import { decryptSecret } from './secret-box.js'; +import { R2_BUCKET, isR2Configured, r2 } from './storage/r2-client.js'; +import { contentHash } from './yjs.js'; + +export type DriveLink = typeof driveFolderLinks.$inferSelect; +type DriveMapping = typeof driveFileMappings.$inferSelect; + +export interface SyncResult { + created: number; + updated: number; + skipped: number; + pushed: number; + conflicts: number; + error?: string; +} + +const emptyResult = (): SyncResult => ({ created: 0, updated: 0, skipped: 0, pushed: 0, conflicts: 0 }); + +/** The Drive client for a link, from the connecting member's encrypted token. */ +export async function getLinkDrive(link: DriveLink): Promise { + const rows = await withSystemPrivilege((tx) => + tx.select({ tok: workspaceMembers.driveRefreshToken }) + .from(workspaceMembers) + .where(and(eq(workspaceMembers.userId, link.connectedBy), isNotNull(workspaceMembers.driveRefreshToken))) + .limit(1), + ); + const enc = rows[0]?.tok; + if (!enc) return null; + return driveClientFromRefresh(decryptSecret(enc)); +} + +function tooLarge(f: DriveFile): boolean { + return Boolean(f.size && f.size > config.DRIVE_SYNC_MAX_FILE_MB * 1024 * 1024); +} + +async function findMapping(linkId: string, driveFileId: string): Promise { + const rows = await withSystemPrivilege((tx) => + tx.select().from(driveFileMappings) + .where(and(eq(driveFileMappings.linkId, linkId), eq(driveFileMappings.driveFileId, driveFileId))) + .limit(1), + ); + return rows[0]; +} + +async function touchLink(linkId: string, patch: Partial): Promise { + await withSystemPrivilege((tx) => + tx.update(driveFolderLinks).set({ ...patch, updatedAt: new Date() }).where(eq(driveFolderLinks.id, linkId)), + ); +} + +// ── Pull (Drive → Mnema) ────────────────────────────────────────────────────── + +async function upsertDocFromDrive( + link: DriveLink, f: DriveFile, markdown: string, mapping: DriveMapping | undefined, +): Promise { + const hash = contentHash(markdown); + const title = f.name.replace(/\.[^.]+$/, ''); + const yjsState = Buffer.from(await markdownToYjsState(markdown)); + const now = new Date(); + + await withSystemPrivilege(async (tx) => { + let docId = mapping?.docId ?? null; + if (docId) { + await tx.update(docs) + .set({ title, markdown, yjsState, contentHash: hash, updatedAt: now }) + .where(and(eq(docs.id, docId), eq(docs.workspaceId, link.workspaceId))); + } else { + const [doc] = await tx.insert(docs).values({ + workspaceId: link.workspaceId, + folderId: link.folderId, + path: `drive-${f.id}`, + title, + markdown, + yjsState, + contentHash: hash, + createdBy: link.connectedBy, + updatedBy: link.connectedBy, + }).returning({ id: docs.id }); + docId = doc!.id; + } + await upsertMapping(tx, link.id, f, { docId, contentHash: hash }); + }); +} + +async function upsertAttachmentFromDrive( + link: DriveLink, f: DriveFile, ext: string, bytes: Buffer, mapping: DriveMapping | undefined, +): Promise { + const r2Key = `attachments/${link.workspaceId}/${randomUUID()}.${ext}`; + const mime = f.mimeType || mimeForExt(ext); + await r2().send(new PutObjectCommand({ + Bucket: R2_BUCKET(), Key: r2Key, Body: bytes, ContentType: mime, + ContentDisposition: `attachment; filename="${f.name}"`, + })); + + await withSystemPrivilege(async (tx) => { + let attachmentId = mapping?.attachmentId ?? null; + if (attachmentId) { + await tx.update(attachments) + .set({ r2Key, format: ext, originalName: f.name, mimeType: mime, sizeBytes: bytes.length, status: 'ready', updatedAt: new Date() }) + .where(and(eq(attachments.id, attachmentId), eq(attachments.workspaceId, link.workspaceId))); + } else { + const [att] = await tx.insert(attachments).values({ + workspaceId: link.workspaceId, + type: 'source', + format: ext, + originalName: f.name, + r2Key, + mimeType: mime, + sizeBytes: bytes.length, + status: 'ready', + }).returning({ id: attachments.id }); + attachmentId = att!.id; + } + await upsertMapping(tx, link.id, f, { attachmentId }); + }); +} + +async function upsertMapping( + tx: Parameters[0]>[0], + linkId: string, f: DriveFile, extra: { docId?: string; attachmentId?: string; contentHash?: string }, +): Promise { + const now = new Date(); + await tx.insert(driveFileMappings).values({ + linkId, + driveFileId: f.id, + driveName: f.name, + docId: extra.docId ?? null, + attachmentId: extra.attachmentId ?? null, + driveMd5: f.md5Checksum, + contentHash: extra.contentHash ?? null, + driveModifiedAt: f.modifiedTime ? new Date(f.modifiedTime) : null, + mnemaModifiedAt: now, + syncState: 'synced', + }).onConflictDoUpdate({ + target: [driveFileMappings.linkId, driveFileMappings.driveFileId], + set: { + driveName: f.name, + ...(extra.docId ? { docId: extra.docId } : {}), + ...(extra.attachmentId ? { attachmentId: extra.attachmentId } : {}), + driveMd5: f.md5Checksum, + ...(extra.contentHash ? { contentHash: extra.contentHash } : {}), + driveModifiedAt: f.modifiedTime ? new Date(f.modifiedTime) : null, + mnemaModifiedAt: now, + syncState: 'synced', + updatedAt: now, + }, + }); +} + +/** Drive → Mnema. Downloads accepted, changed files and writes docs/attachments. */ +export async function pullLink(link: DriveLink): Promise { + const res = emptyResult(); + const drive = await getLinkDrive(link); + if (!drive) return { ...res, error: 'not_connected' }; + + let files: DriveFile[]; + try { + files = await listFilesInFolder(drive, link.driveFolderId); + } catch (err) { + await touchLink(link.id, { status: 'error', errorMessage: err instanceof Error ? err.message : 'list_failed' }); + return { ...res, error: 'list_failed' }; + } + + for (const f of files) { + const ext = extOf(f.name); + if (!isAcceptedType(ext, link.acceptedTypes) || tooLarge(f)) { res.skipped++; continue; } + + const mapping = await findMapping(link.id, f.id); + if (mapping?.driveMd5 && mapping.driveMd5 === f.md5Checksum) continue; // unchanged on Drive + + // Manual conflict policy: if Mnema has an un-pushed local edit, don't clobber it. + if (mapping?.syncState === 'pending_push' && link.conflictPolicy === 'manual') { + await withSystemPrivilege((tx) => + tx.update(driveFileMappings).set({ syncState: 'conflict', updatedAt: new Date() }).where(eq(driveFileMappings.id, mapping.id)), + ); + res.conflicts++; + continue; + } + + try { + if (isTextExt(ext)) { + const bytes = await downloadDriveFile(drive, f.id); + await upsertDocFromDrive(link, f, bytes.toString('utf8'), mapping); + } else if (ext === 'pdf' || ext === 'docx') { + // Binaries land in `attachments`, whose `format` is docx|pdf (v1). Other + // binary types are skipped rather than stored with a foreign format. + if (!isR2Configured()) { res.skipped++; continue; } // no blob store → can't hold binaries + const bytes = await downloadDriveFile(drive, f.id); + await upsertAttachmentFromDrive(link, f, ext, bytes, mapping); + } else { + // eslint-disable-next-line no-console + console.warn(`[drive-sync] link=${link.id} skipping unsupported type "${ext}" (${f.name})`); + res.skipped++; + continue; + } + if (mapping) res.updated++; else res.created++; + } catch (err) { + // eslint-disable-next-line no-console + console.warn(`[drive-sync] link=${link.id} pull failed for "${f.name}":`, err instanceof Error ? err.message : err); + res.skipped++; + } + } + + await touchLink(link.id, { lastSyncedAt: new Date(), status: 'active', errorMessage: null }); + return res; +} + +// ── Push (Mnema → Drive) ────────────────────────────────────────────────────── + +/** Mnema → Drive. Exports the linked folder's docs to `.md` files in Drive. */ +export async function pushLink(link: DriveLink): Promise { + const res = emptyResult(); + const drive = await getLinkDrive(link); + if (!drive) return { ...res, error: 'not_connected' }; + + const folderDocs = await withSystemPrivilege((tx) => + tx.select({ id: docs.id, title: docs.title, markdown: docs.markdown, contentHash: docs.contentHash }) + .from(docs) + .where(and(eq(docs.workspaceId, link.workspaceId), eq(docs.folderId, link.folderId))), + ); + + for (const d of folderDocs) { + try { + const existing = await withSystemPrivilege((tx) => + tx.select().from(driveFileMappings) + .where(and(eq(driveFileMappings.linkId, link.id), eq(driveFileMappings.docId, d.id))) + .limit(1), + ); + const map = existing[0]; + // Skip if the doc hasn't changed since it was last synced. + if (map && map.contentHash && map.contentHash === d.contentHash) continue; + + const body = Buffer.from(d.markdown, 'utf8'); + const name = `${d.title || 'untitled'}.md`; + if (map?.driveFileId) { + const { md5Checksum } = await updateDriveFile(drive, map.driveFileId, 'text/markdown', body); + await withSystemPrivilege((tx) => + tx.update(driveFileMappings) + .set({ driveMd5: md5Checksum, contentHash: d.contentHash, syncState: 'synced', mnemaModifiedAt: new Date(), updatedAt: new Date() }) + .where(eq(driveFileMappings.id, map.id)), + ); + } else { + const { id: driveFileId, md5Checksum } = await uploadToFolder(drive, link.driveFolderId, name, 'text/markdown', body); + await withSystemPrivilege((tx) => + tx.insert(driveFileMappings).values({ + linkId: link.id, driveFileId, driveName: name, docId: d.id, + driveMd5: md5Checksum, contentHash: d.contentHash, mnemaModifiedAt: new Date(), syncState: 'synced', + }), + ); + } + res.pushed++; + } catch (err) { + // eslint-disable-next-line no-console + console.warn(`[drive-sync] link=${link.id} push failed for doc "${d.title}":`, err instanceof Error ? err.message : err); + res.skipped++; + } + } + + await touchLink(link.id, { lastSyncedAt: new Date() }); + return res; +} + +/** Run the configured direction(s) for a link and return a combined result. */ +export async function syncLink(link: DriveLink): Promise { + if (link.status === 'paused') return { ...emptyResult(), error: 'paused' }; + const combined = emptyResult(); + const add = (r: SyncResult) => { + combined.created += r.created; combined.updated += r.updated; combined.skipped += r.skipped; + combined.pushed += r.pushed; combined.conflicts += r.conflicts; + if (r.error && !combined.error) combined.error = r.error; + }; + if (link.direction === 'pull' || link.direction === 'both') add(await pullLink(link)); + if (link.direction === 'push' || link.direction === 'both') add(await pushLink(link)); + return combined; +} diff --git a/apps/api/src/lib/google-drive.ts b/apps/api/src/lib/google-drive.ts new file mode 100644 index 000000000..bffd2476d --- /dev/null +++ b/apps/api/src/lib/google-drive.ts @@ -0,0 +1,207 @@ +/** + * Phase 10 — Google Drive client (googleapis SDK). + * + * Bring-your-own OAuth app, same shape as lib/google-calendar.ts: the OAuth + * client credentials come from env (optional) and callers must check + * `driveConfigured()` first and 503 when unset. The refresh token is stored + * encrypted (secret-box) on workspace_members.drive_refresh_token. + * + * Scope is configurable (GOOGLE_DRIVE_SCOPE): + * - 'drive.file' (default): least privilege — Mnema only sees files it created + * or that the user explicitly opened with it. + * - 'drive': full access — lets users pull pre-existing Drive folders; needs + * Google's restricted-scope review to ship to real users in production. + */ +import { Readable } from 'node:stream'; +import { google, type drive_v3 } from 'googleapis'; +import { config } from '../config/env.js'; + +export type DriveClient = drive_v3.Drive; + +const SCOPE_URLS: Record<'drive.file' | 'drive', string> = { + 'drive.file': 'https://www.googleapis.com/auth/drive.file', + drive: 'https://www.googleapis.com/auth/drive', +}; + +// File extensions synced by default when a link doesn't set its own allow-list. +// v1 syncs text (→ Mnema docs) and docx/pdf (→ attachments, the formats the +// attachment pipeline models). More binary types can follow as a separate change. +export const DEFAULT_ACCEPTED_TYPES = ['md', 'markdown', 'txt', 'pdf', 'docx']; + +/** Extensions that become Mnema docs (text). Everything else becomes an attachment. */ +const TEXT_EXTS = new Set(['md', 'markdown', 'txt']); + +const MIME_BY_EXT: Record = { + md: 'text/markdown', markdown: 'text/markdown', txt: 'text/plain', + pdf: 'application/pdf', csv: 'text/csv', + doc: 'application/msword', + docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', +}; + +export function driveConfigured(): boolean { + return Boolean( + config.GOOGLE_DRIVE_CLIENT_ID && + config.GOOGLE_DRIVE_CLIENT_SECRET && + config.GOOGLE_DRIVE_REDIRECT_URI, + ); +} + +export function driveScopeUrl(): string { + return SCOPE_URLS[config.GOOGLE_DRIVE_SCOPE]; +} + +export function extOf(name: string): string { + const i = name.lastIndexOf('.'); + return i >= 0 ? name.slice(i + 1).toLowerCase() : ''; +} + +export function isTextExt(ext: string): boolean { + return TEXT_EXTS.has(ext.toLowerCase()); +} + +export function mimeForExt(ext: string): string { + return MIME_BY_EXT[ext.toLowerCase()] ?? 'application/octet-stream'; +} + +/** True if `ext` is allowed for this link (empty allow-list = built-in defaults). */ +export function isAcceptedType(ext: string, acceptedTypes: string[]): boolean { + const list = acceptedTypes.length > 0 ? acceptedTypes : DEFAULT_ACCEPTED_TYPES; + return list.includes(ext.toLowerCase()); +} + +function oauthClient() { + return new google.auth.OAuth2( + config.GOOGLE_DRIVE_CLIENT_ID, + config.GOOGLE_DRIVE_CLIENT_SECRET, + config.GOOGLE_DRIVE_REDIRECT_URI, + ); +} + +/** Google consent URL to start the Drive-link flow (offline → refresh token). */ +export function consentUrl(state: string): string { + return oauthClient().generateAuthUrl({ + access_type: 'offline', + prompt: 'select_account consent', // chooser → consent, and force a refresh_token + scope: [driveScopeUrl()], + state, + include_granted_scopes: true, + }); +} + +/** Exchange an auth code for tokens. Returns the offline refresh token. */ +export async function exchangeCode(code: string): Promise<{ refresh_token?: string; access_token?: string }> { + const { tokens } = await oauthClient().getToken(code); + return { refresh_token: tokens.refresh_token ?? undefined, access_token: tokens.access_token ?? undefined }; +} + +/** An authenticated Drive v3 client from a stored refresh token (auto-refreshes access). */ +export function driveClientFromRefresh(refreshToken: string): DriveClient { + const auth = oauthClient(); + auth.setCredentials({ refresh_token: refreshToken }); + return google.drive({ version: 'v3', auth }); +} + +export interface DriveFolder { id: string; name: string } + +/** List the user's Drive folders under `parentId` (or 'root') — for the picker. */ +export async function listDriveFolders(drive: DriveClient, parentId = 'root'): Promise { + const res = await drive.files.list({ + q: `mimeType = 'application/vnd.google-apps.folder' and trashed = false and '${parentId}' in parents`, + fields: 'files(id, name)', + orderBy: 'name', + pageSize: 200, + spaces: 'drive', + }); + return (res.data.files ?? []) + .filter((f): f is { id: string; name: string } => Boolean(f.id && f.name)) + .map((f) => ({ id: f.id, name: f.name })); +} + +export interface DriveFile { + id: string; + name: string; + mimeType: string; + md5Checksum: string | null; + modifiedTime: string | null; + size: number | null; + trashed: boolean; +} + +const FILE_FIELDS = 'id, name, mimeType, md5Checksum, modifiedTime, size, trashed'; + +function toDriveFile(f: drive_v3.Schema$File): DriveFile { + return { + id: f.id!, + name: f.name ?? 'untitled', + mimeType: f.mimeType ?? 'application/octet-stream', + md5Checksum: f.md5Checksum ?? null, + modifiedTime: f.modifiedTime ?? null, + size: f.size ? Number(f.size) : null, + trashed: Boolean(f.trashed), + }; +} + +/** List non-folder files directly under a Drive folder (paginates fully). */ +export async function listFilesInFolder(drive: DriveClient, driveFolderId: string): Promise { + const out: DriveFile[] = []; + let pageToken: string | undefined; + do { + const res = await drive.files.list({ + q: `'${driveFolderId}' in parents and mimeType != 'application/vnd.google-apps.folder' and trashed = false`, + fields: `nextPageToken, files(${FILE_FIELDS})`, + pageSize: 200, + spaces: 'drive', + pageToken, + }); + for (const f of res.data.files ?? []) if (f.id) out.push(toDriveFile(f)); + pageToken = res.data.nextPageToken ?? undefined; + } while (pageToken); + return out; +} + +/** Download a Drive file's bytes. */ +export async function downloadDriveFile(drive: DriveClient, fileId: string): Promise { + const res = await drive.files.get( + { fileId, alt: 'media' }, + { responseType: 'arraybuffer' }, + ); + return Buffer.from(res.data as ArrayBuffer); +} + +/** Create a Drive folder (optionally under `parentId`). Returns its id. */ +export async function createDriveFolder(drive: DriveClient, name: string, parentId?: string): Promise { + const res = await drive.files.create({ + requestBody: { + name, + mimeType: 'application/vnd.google-apps.folder', + ...(parentId ? { parents: [parentId] } : {}), + }, + fields: 'id', + }); + return res.data.id!; +} + +/** Upload a new file into a Drive folder. Returns the new file id + md5. */ +export async function uploadToFolder( + drive: DriveClient, driveFolderId: string, name: string, mimeType: string, body: Buffer, +): Promise<{ id: string; md5Checksum: string | null }> { + const res = await drive.files.create({ + requestBody: { name, parents: [driveFolderId] }, + media: { mimeType, body: Readable.from(body) }, + fields: 'id, md5Checksum', + }); + return { id: res.data.id!, md5Checksum: res.data.md5Checksum ?? null }; +} + +/** Overwrite an existing Drive file's content. Returns its new md5. */ +export async function updateDriveFile( + drive: DriveClient, fileId: string, mimeType: string, body: Buffer, +): Promise<{ md5Checksum: string | null }> { + const res = await drive.files.update({ + fileId, + media: { mimeType, body: Readable.from(body) }, + fields: 'id, md5Checksum', + }); + return { md5Checksum: res.data.md5Checksum ?? null }; +} diff --git a/apps/api/src/queue/drive-sync.ts b/apps/api/src/queue/drive-sync.ts new file mode 100644 index 000000000..8bc637436 --- /dev/null +++ b/apps/api/src/queue/drive-sync.ts @@ -0,0 +1,31 @@ +/** + * BullMQ queue for Google Drive folder sync (Phase 10). + * + * Enqueued from routes/drive.ts (manual "Sync now"); a reconcile cron can enqueue + * with reason 'cron' as a follow-up. The drive-sync worker runs syncLink() for the + * link. A stable jobId per link debounces concurrent syncs of the same folder. + */ +import { Queue } from 'bullmq'; +import { redisConnection } from '../lib/redis.js'; + +export const DRIVE_SYNC_QUEUE_NAME = 'drive-sync'; + +export interface DriveSyncJobData { + linkId: string; + reason: 'manual' | 'cron'; +} + +export const driveSyncQueue = new Queue(DRIVE_SYNC_QUEUE_NAME, { + connection: redisConnection, + defaultJobOptions: { + attempts: 3, + backoff: { type: 'exponential', delay: 5000 }, + removeOnComplete: 100, + removeOnFail: 200, + }, +}); + +export async function enqueueDriveSync(data: DriveSyncJobData): Promise { + // jobId dedupes: a queued/active sync for the same link absorbs new triggers. + await driveSyncQueue.add('sync', data, { jobId: `drive-sync:${data.linkId}` }); +} diff --git a/apps/api/src/routes/drive.ts b/apps/api/src/routes/drive.ts new file mode 100644 index 000000000..ee3a1d620 --- /dev/null +++ b/apps/api/src/routes/drive.ts @@ -0,0 +1,268 @@ +/** + * Phase 10 — Google Drive folder linking + two-way sync. + * + * GET /api/drive/connect → redirect to Google consent (offline, configurable scope) + * GET /api/drive/callback → exchange code, store encrypted refresh token + * GET /api/drive/status → { connected, configured, scope } + * GET /api/drive/folders?parent= → list the user's Drive folders (picker) + * GET /api/drive/links → list this workspace's folder links + * POST /api/drive/links → link a Mnema folder ⇄ Drive folder + * PATCH /api/drive/links/:id → edit types / direction / conflict policy / pause + * DELETE /api/drive/links/:id → unlink + * POST /api/drive/links/:id/sync → enqueue a sync now + * + * The refresh token is user-scoped (their Google Drive); links + mappings are + * workspace-scoped and enforced app-layer (filter by workspace_id + requireRole). + */ +import { and, eq, isNotNull } from 'drizzle-orm'; +import type { FastifyPluginAsync, FastifyReply } from 'fastify'; +import { config } from '../config/env.js'; +import { db } from '../db/index.js'; +import { driveFileMappings, driveFolderLinks, folders, workspaceMembers } from '../db/schema.js'; +import { + consentUrl, createDriveFolder, DEFAULT_ACCEPTED_TYPES, driveClientFromRefresh, + driveConfigured, driveScopeUrl, exchangeCode, listDriveFolders, type DriveClient, +} from '../lib/google-drive.js'; +import { enqueueDriveSync } from '../queue/drive-sync.js'; +import { requireRole, RoleError } from '../lib/role.js'; +import { decryptSecret, encryptSecret, signState, verifySignedState } from '../lib/secret-box.js'; + +const DIRECTIONS = ['pull', 'push', 'both'] as const; +const CONFLICT_POLICIES = ['manual', 'lww'] as const; + +export const driveRoutes: FastifyPluginAsync = async (app) => { + function guard(err: unknown, reply: FastifyReply): boolean { + if (err instanceof RoleError) { reply.code(err.status).send({ error: err.reason }); return true; } + return false; + } + + /** The requesting user's Drive client (their token), or null if not connected. */ + async function userDrive(userId: string): Promise { + const rows = await db.select({ tok: workspaceMembers.driveRefreshToken }) + .from(workspaceMembers) + .where(and(eq(workspaceMembers.userId, userId), isNotNull(workspaceMembers.driveRefreshToken))) + .limit(1); + const enc = rows[0]?.tok; + return enc ? driveClientFromRefresh(decryptSecret(enc)) : null; + } + + function normalizeTypes(input: unknown): string[] { + if (!Array.isArray(input)) return []; + return [...new Set(input + .filter((x): x is string => typeof x === 'string') + .map((x) => x.trim().replace(/^\./, '').toLowerCase()) + .filter(Boolean))]; + } + + // ── Start the link flow ──────────────────────────────────────────────────── + app.get('/api/drive/connect', async (req, reply) => { + if (!req.auth) return reply.code(401).send({ error: 'unauthorized' }); + if (!driveConfigured()) return reply.code(503).send({ error: 'drive_not_configured' }); + try { await requireRole(req, 'editor'); } catch (e) { if (guard(e, reply)) return; throw e; } + const state = signState({ sub: req.auth.sub, tenant: req.auth.tenant_id }); + const url = consentUrl(state); + // Navigate to Google WITHOUT a referrer (same reasoning as calendar connect): + // a referrer-less navigation resolves the active session and shows the account + // chooser instead of Google's sign-in form (which 400s without a session). + return reply + .header('Referrer-Policy', 'no-referrer') + .type('text/html') + .send( + `` + + `` + + ``, + ); + }); + + // ── Google redirects back here ───────────────────────────────────────────── + app.get('/api/drive/callback', async (req, reply) => { + const q = req.query as { code?: string; state?: string; error?: string }; + const back = `${config.WEB_BASE_URL}/app/settings/integrations`; + if (q.error || !q.code || !q.state) return reply.redirect(`${back}?drive=error`); + const st = verifySignedState(q.state); + if (!st || !driveConfigured()) return reply.redirect(`${back}?drive=error`); + try { + const tokens = await exchangeCode(q.code); + if (!tokens.refresh_token) { + req.log.warn('drive callback: no refresh_token returned'); + return reply.redirect(`${back}?drive=error`); + } + await db.update(workspaceMembers) + .set({ driveRefreshToken: encryptSecret(tokens.refresh_token) }) + .where(and(eq(workspaceMembers.workspaceId, st.tenant), eq(workspaceMembers.userId, st.sub))); + return reply.redirect(`${back}?drive=connected`); + } catch (err) { + req.log.error({ err }, 'drive callback failed'); + return reply.redirect(`${back}?drive=error`); + } + }); + + // ── Connection status ────────────────────────────────────────────────────── + app.get('/api/drive/status', async (req, reply) => { + if (!req.auth) return reply.code(401).send({ error: 'unauthorized' }); + const rows = await db.select({ tok: workspaceMembers.driveRefreshToken }) + .from(workspaceMembers) + .where(and(eq(workspaceMembers.userId, req.auth.sub), isNotNull(workspaceMembers.driveRefreshToken))) + .limit(1); + return reply.send({ + connected: rows.length > 0, + configured: driveConfigured(), + scope: driveConfigured() ? driveScopeUrl() : null, + defaultTypes: DEFAULT_ACCEPTED_TYPES, + }); + }); + + // ── List the user's Drive folders (for the picker) ───────────────────────── + app.get('/api/drive/folders', async (req, reply) => { + if (!req.auth) return reply.code(401).send({ error: 'unauthorized' }); + if (!driveConfigured()) return reply.code(503).send({ error: 'drive_not_configured' }); + const drive = await userDrive(req.auth.sub); + if (!drive) return reply.code(400).send({ error: 'drive_not_connected' }); + const parent = (req.query as { parent?: string }).parent || 'root'; + try { + const folderList = await listDriveFolders(drive, parent); + return reply.send({ folders: folderList }); + } catch (err) { + req.log.error({ err }, 'drive folders list failed'); + return reply.code(502).send({ error: 'drive_list_failed' }); + } + }); + + // ── List links for the active workspace ──────────────────────────────────── + app.get('/api/drive/links', async (req, reply) => { + if (!req.auth) return reply.code(401).send({ error: 'unauthorized' }); + const ws = req.auth.tenant_id; + const rows = await db.select().from(driveFolderLinks) + .where(eq(driveFolderLinks.workspaceId, ws)); + return reply.send({ links: rows }); + }); + + // ── Create a link ────────────────────────────────────────────────────────── + app.post('/api/drive/links', async (req, reply) => { + if (!req.auth) return reply.code(401).send({ error: 'unauthorized' }); + if (!driveConfigured()) return reply.code(503).send({ error: 'drive_not_configured' }); + try { await requireRole(req, 'editor'); } catch (e) { if (guard(e, reply)) return; throw e; } + + const ws = req.auth.tenant_id; + const body = req.body as { + folderId?: string; + driveFolderId?: string; + createInDrive?: boolean; + driveParentId?: string; + driveFolderName?: string; + acceptedTypes?: string[]; + direction?: string; + conflictPolicy?: string; + }; + if (!body.folderId) return reply.code(400).send({ error: 'folderId_required' }); + + // The Mnema folder must belong to the active workspace. + const [folder] = await db.select({ id: folders.id, name: folders.name }) + .from(folders) + .where(and(eq(folders.id, body.folderId), eq(folders.workspaceId, ws))) + .limit(1); + if (!folder) return reply.code(404).send({ error: 'folder_not_found' }); + + const drive = await userDrive(req.auth.sub); + if (!drive) return reply.code(400).send({ error: 'drive_not_connected' }); + + // Resolve the Drive folder: either an existing one, or create it from the Mnema folder. + let driveFolderId = body.driveFolderId; + let driveFolderName = body.driveFolderName ?? null; + try { + if (body.createInDrive) { + driveFolderId = await createDriveFolder(drive, folder.name, body.driveParentId); + driveFolderName = folder.name; + } + } catch (err) { + req.log.error({ err }, 'drive folder create failed'); + return reply.code(502).send({ error: 'drive_create_failed' }); + } + if (!driveFolderId) return reply.code(400).send({ error: 'driveFolderId_or_createInDrive_required' }); + + const direction = DIRECTIONS.includes(body.direction as never) ? body.direction! : 'both'; + const conflictPolicy = CONFLICT_POLICIES.includes(body.conflictPolicy as never) ? body.conflictPolicy! : 'manual'; + + try { + const [link] = await db.insert(driveFolderLinks).values({ + workspaceId: ws, + folderId: folder.id, + driveFolderId, + driveFolderName, + connectedBy: req.auth.sub, + direction, + conflictPolicy, + acceptedTypes: normalizeTypes(body.acceptedTypes), + }).returning(); + // Kick off an initial sync. + await enqueueDriveSync({ linkId: link!.id, reason: 'manual' }); + return reply.send({ link }); + } catch (err) { + // Unique violation → folder already linked. + req.log.warn({ err }, 'drive link create failed'); + return reply.code(409).send({ error: 'already_linked' }); + } + }); + + // ── Edit a link ──────────────────────────────────────────────────────────── + app.patch('/api/drive/links/:id', async (req, reply) => { + if (!req.auth) return reply.code(401).send({ error: 'unauthorized' }); + try { await requireRole(req, 'editor'); } catch (e) { if (guard(e, reply)) return; throw e; } + const ws = req.auth.tenant_id; + const { id } = req.params as { id: string }; + const body = req.body as { acceptedTypes?: string[]; direction?: string; conflictPolicy?: string; status?: string }; + + const patch: Partial = { updatedAt: new Date() }; + if (body.acceptedTypes !== undefined) patch.acceptedTypes = normalizeTypes(body.acceptedTypes); + if (body.direction && DIRECTIONS.includes(body.direction as never)) patch.direction = body.direction; + if (body.conflictPolicy && CONFLICT_POLICIES.includes(body.conflictPolicy as never)) patch.conflictPolicy = body.conflictPolicy; + if (body.status === 'active' || body.status === 'paused') patch.status = body.status; + + const [link] = await db.update(driveFolderLinks).set(patch) + .where(and(eq(driveFolderLinks.id, id), eq(driveFolderLinks.workspaceId, ws))) + .returning(); + if (!link) return reply.code(404).send({ error: 'link_not_found' }); + return reply.send({ link }); + }); + + // ── Unlink ───────────────────────────────────────────────────────────────── + app.delete('/api/drive/links/:id', async (req, reply) => { + if (!req.auth) return reply.code(401).send({ error: 'unauthorized' }); + try { await requireRole(req, 'editor'); } catch (e) { if (guard(e, reply)) return; throw e; } + const ws = req.auth.tenant_id; + const { id } = req.params as { id: string }; + const [gone] = await db.delete(driveFolderLinks) + .where(and(eq(driveFolderLinks.id, id), eq(driveFolderLinks.workspaceId, ws))) + .returning({ id: driveFolderLinks.id }); + if (!gone) return reply.code(404).send({ error: 'link_not_found' }); + return reply.send({ ok: true }); + }); + + // ── Sync now ─────────────────────────────────────────────────────────────── + app.post('/api/drive/links/:id/sync', async (req, reply) => { + if (!req.auth) return reply.code(401).send({ error: 'unauthorized' }); + try { await requireRole(req, 'editor'); } catch (e) { if (guard(e, reply)) return; throw e; } + const ws = req.auth.tenant_id; + const { id } = req.params as { id: string }; + const [link] = await db.select({ id: driveFolderLinks.id }).from(driveFolderLinks) + .where(and(eq(driveFolderLinks.id, id), eq(driveFolderLinks.workspaceId, ws))) + .limit(1); + if (!link) return reply.code(404).send({ error: 'link_not_found' }); + await enqueueDriveSync({ linkId: id, reason: 'manual' }); + return reply.send({ ok: true, queued: true }); + }); + + // ── Conflicts for a link (for the resolver UI) ───────────────────────────── + app.get('/api/drive/links/:id/conflicts', async (req, reply) => { + if (!req.auth) return reply.code(401).send({ error: 'unauthorized' }); + const ws = req.auth.tenant_id; + const { id } = req.params as { id: string }; + const [link] = await db.select({ id: driveFolderLinks.id }).from(driveFolderLinks) + .where(and(eq(driveFolderLinks.id, id), eq(driveFolderLinks.workspaceId, ws))) + .limit(1); + if (!link) return reply.code(404).send({ error: 'link_not_found' }); + const rows = await db.select().from(driveFileMappings) + .where(and(eq(driveFileMappings.linkId, id), eq(driveFileMappings.syncState, 'conflict'))); + return reply.send({ conflicts: rows }); + }); +}; diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index e881cc241..cf0c642ae 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -30,6 +30,7 @@ import { healthRoutes } from './routes/health.js'; import { invitationsRoutes } from './routes/invitations.js'; import { membersRoutes } from './routes/members.js'; import { calendarRoutes } from './routes/calendar.js'; +import { driveRoutes } from './routes/drive.js'; import { notificationsRoutes } from './routes/notifications.js'; import { workspacesRoutes } from './routes/workspaces.js'; import { byokRoutes } from './routes/byok.js'; @@ -147,6 +148,7 @@ await app.register(communityRoutes); await app.register(invitationsRoutes); await app.register(membersRoutes); await app.register(calendarRoutes); +await app.register(driveRoutes); await app.register(notificationsRoutes); await app.register(workspacesRoutes); await app.register(byokRoutes); diff --git a/apps/api/src/workers/drive-sync/worker.ts b/apps/api/src/workers/drive-sync/worker.ts new file mode 100644 index 000000000..57bc4b4ab --- /dev/null +++ b/apps/api/src/workers/drive-sync/worker.ts @@ -0,0 +1,41 @@ +/** + * Google Drive sync worker (Phase 10). + * + * Consumes drive-sync jobs (manual "Sync now", push webhook, reconcile cron) and + * runs syncLink() for the link — pulling Drive→Mnema and/or pushing Mnema→Drive + * per the link's direction. Idempotent; safe to re-run. + */ +import { Worker } from 'bullmq'; +import { eq } from 'drizzle-orm'; +import IORedis from 'ioredis'; +import { config } from '../../config/env.js'; +import { db } from '../../db/index.js'; +import { driveFolderLinks } from '../../db/schema.js'; +import { syncLink } from '../../lib/drive-sync.js'; +import { DRIVE_SYNC_QUEUE_NAME, type DriveSyncJobData } from '../../queue/drive-sync.js'; + +export function startDriveSyncWorker(): Worker { + // Dedicated connection for the worker (separate from queue connection). + const connection = new IORedis(config.REDIS_URL, { maxRetriesPerRequest: null }); + + const worker = new Worker( + DRIVE_SYNC_QUEUE_NAME, + async (job) => { + const rows = await db.select().from(driveFolderLinks) + .where(eq(driveFolderLinks.id, job.data.linkId)).limit(1); + const link = rows[0]; + if (!link) return; + const res = await syncLink(link); + // eslint-disable-next-line no-console + console.log(`[drive-sync] ${job.data.linkId} reason=${job.data.reason}`, res); + }, + { connection, concurrency: 4 }, + ); + + worker.on('failed', (job, err) => { + console.error(`[drive-sync] ${job?.id ?? '?'} failed (attempt ${job?.attemptsMade ?? '?'}):`, err.message); + }); + worker.on('error', (err) => { console.error('[drive-sync] worker error:', err); }); + + return worker; +} diff --git a/apps/api/src/workers/server.ts b/apps/api/src/workers/server.ts index 6569ac127..8926fea9b 100644 --- a/apps/api/src/workers/server.ts +++ b/apps/api/src/workers/server.ts @@ -12,6 +12,7 @@ import { startEmbeddingsWorker } from './embeddings/worker.js'; import { startEmailWorker } from './email/worker.js'; import { startHookEventsWorker } from './hook-events/worker.js'; +import { startDriveSyncWorker } from './drive-sync/worker.js'; import { startRetryWorker } from './retry/worker.js'; import { startCronWorkers } from './cron.js'; import { startPdfGenerationWorker } from './pdf-generation/worker.js'; @@ -29,6 +30,10 @@ const hookEventsWorker = startHookEventsWorker(); // eslint-disable-next-line no-console console.log('[workers] hook-events worker started'); +const driveSyncWorker = startDriveSyncWorker(); +// eslint-disable-next-line no-console +console.log('[workers] drive-sync worker started'); + const retryWorker = startRetryWorker(); // eslint-disable-next-line no-console console.log('[workers] retry worker started'); @@ -58,6 +63,7 @@ const shutdown = async (signal: string): Promise => { await embeddings.close(); await emailWorker.close(); await hookEventsWorker.close(); + await driveSyncWorker.close(); await retryWorker.close(); await cronWorkers.close(); if (pdfWorker) await pdfWorker.close(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 51129081d..6fc27b533 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -141,6 +141,9 @@ importers: fastify-plugin: specifier: ^5.0.0 version: 5.1.0 + googleapis: + specifier: ^173.0.0 + version: 173.0.0 gpt-tokenizer: specifier: ^3.4.0 version: 3.4.0 @@ -929,11 +932,11 @@ packages: '@esbuild-kit/core-utils@3.3.2': resolution: {integrity: sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ==} - deprecated: 'Merged into tsx: https://tsx.is' + deprecated: 'Merged into tsx: https://tsx.hirok.io' '@esbuild-kit/esm-loader@2.6.5': resolution: {integrity: sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA==} - deprecated: 'Merged into tsx: https://tsx.is' + deprecated: 'Merged into tsx: https://tsx.hirok.io' '@esbuild/aix-ppc64@0.25.12': resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} @@ -1827,6 +1830,10 @@ packages: '@ioredis/commands@1.10.0': resolution: {integrity: sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==} + '@isaacs/cliui@8.0.2': + resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} + engines: {node: '>=12'} + '@isaacs/fs-minipass@4.0.1': resolution: {integrity: sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==} engines: {node: '>=18.0.0'} @@ -2240,6 +2247,10 @@ packages: '@pinojs/redact@0.4.0': resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} + '@pkgjs/parseargs@0.11.0': + resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} + engines: {node: '>=14'} + '@prosemirror-adapter/core@0.5.3': resolution: {integrity: sha512-jGcI/eq3USFqf1fCaejgCmifmXYDyFco6/5KjyyNBnuEhLqyLjD+020CtcduEOsnN4xPZr5ikclgHIR6kEFkQg==} @@ -3318,6 +3329,9 @@ packages: bidi-js@1.0.3: resolution: {integrity: sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==} + bignumber.js@9.3.1: + resolution: {integrity: sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==} + bindings@1.5.0: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} @@ -3337,6 +3351,9 @@ packages: brace-expansion@1.1.14: resolution: {integrity: sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==} + brace-expansion@2.1.2: + resolution: {integrity: sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==} + brace-expansion@5.0.7: resolution: {integrity: sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==} engines: {node: 18 || 20 || >=22} @@ -3346,6 +3363,9 @@ packages: engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true + buffer-equal-constant-time@1.0.1: + resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} @@ -3762,6 +3782,10 @@ packages: resolution: {integrity: sha512-QmU3lyEnbENQPo0M1F9BMu4s6cqNNp8iJA+b/HP2sSb7pf3dxwF3+EP1eO69rwBfH9kFJ1apmzrtogAmVt2/Xw==} engines: {node: '>=12'} + data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + data-urls@7.0.0: resolution: {integrity: sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==} engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0} @@ -3974,6 +3998,12 @@ packages: resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} engines: {node: '>= 0.4'} + eastasianwidth@0.2.0: + resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + + ecdsa-sig-formatter@1.0.11: + resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} + ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} @@ -3989,6 +4019,9 @@ packages: emoji-regex@8.0.0: resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + emoji-regex@9.2.2: + resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + encodeurl@2.0.0: resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} engines: {node: '>= 0.8'} @@ -4237,6 +4270,10 @@ packages: picomatch: optional: true + fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + fflate@0.8.3: resolution: {integrity: sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==} @@ -4285,10 +4322,18 @@ packages: resolution: {integrity: sha512-TdJ2KbkoiDQ7NIRx8IPGD0mAXXpLhamS7c+b7W98b0MHG7lphnda1VOQX/98UDTsttIAdH4TcP0l0MauSnLK8w==} engines: {node: '>=12'} + foreground-child@3.3.1: + resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} + engines: {node: '>=14'} + format@0.2.2: resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==} engines: {node: '>=0.4.x'} + formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + forwarded@0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -4310,6 +4355,22 @@ packages: function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + gaxios@7.1.3: + resolution: {integrity: sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==} + engines: {node: '>=18'} + + gaxios@7.2.0: + resolution: {integrity: sha512-CUVb4wcYe+771XevyH6HtGmXFAGGKkIC3kswAP8Z1JCe0j80JMaTPZH930DWFrvo0atjh18Arc0pEyUCWa5bfg==} + engines: {node: '>=18'} + + gcp-metadata@8.1.2: + resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==} + engines: {node: '>=18'} + + gcp-metadata@8.1.3: + resolution: {integrity: sha512-ziTrzUhhpL9Zk5k0HHzgP/KIpWDJT0VMBC/ynt/QIBvTW+UUcSivQRl6VlwTf/EilDxtSWklHoRsKy1c4k+59w==} + engines: {node: '>=18'} + gensync@1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} engines: {node: '>=6.9.0'} @@ -4344,6 +4405,11 @@ packages: resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} engines: {node: '>=10.13.0'} + glob@10.5.0: + resolution: {integrity: sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + hasBin: true + glob@13.0.6: resolution: {integrity: sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==} engines: {node: 18 || 20 || >=22} @@ -4356,6 +4422,26 @@ packages: resolution: {integrity: sha512-Czmyns5dUsq4seFBR/Kdydhmo8y9kC79hiSkPn0YcGtNnYWnrgt0vjrSjx9tspoDGWm2CMarffRuLjM4xUz8xg==} engines: {node: '>=18'} + google-auth-library@10.5.0: + resolution: {integrity: sha512-7ABviyMOlX5hIVD60YOfHw4/CxOfBhyduaYB+wbFWCWoni4N7SLcV46hrVRktuBbZjFC9ONyqamZITN7q3n32w==} + engines: {node: '>=18'} + + google-auth-library@10.9.0: + resolution: {integrity: sha512-xtvUqvINPhTaBm7nXqlYPcrMHJPm1lCNdSovxnKKhTm+4JsvQ+KGVYJViLoH9Yxu8w+T0Qv5HubzYT9BLrppJg==} + engines: {node: '>=18'} + + google-logging-utils@1.1.3: + resolution: {integrity: sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==} + engines: {node: '>=14'} + + googleapis-common@8.0.2: + resolution: {integrity: sha512-5MXeQzIZaqCH7B+HJWqhQm946VARpZep6acbWSr/fcgF2cQANq7allgX+i/G0EqF0WyUxB277gtWMzRYHMl9tg==} + engines: {node: '>=18.0.0'} + + googleapis@173.0.0: + resolution: {integrity: sha512-xEJJYLZ4qeenVyfzispNfRjCe9bsv7CzBv5zYFLvScOze9snJ8S9W6hjQ729CWPQt5mvn/JrcRaCHzQiukt0ng==} + engines: {node: '>=18'} + gopd@1.2.0: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} @@ -4404,6 +4490,10 @@ packages: peerDependencies: graphology-types: '>=0.24.0' + gtoken@8.0.0: + resolution: {integrity: sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==} + engines: {node: '>=18'} + h3@1.15.11: resolution: {integrity: sha512-L3THSe2MPeBwgIZVSH5zLdBBU90TOxarvhK9d04IDY2AmVS8j2Jz2LIWtwsGOU3lu2I5jCN7FNvVfY2+XyF+mg==} @@ -4617,6 +4707,9 @@ packages: isomorphic.js@0.2.5: resolution: {integrity: sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==} + jackspeak@3.4.3: + resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + jerrypick@1.1.2: resolution: {integrity: sha512-YKnxXEekXKzhpf7CLYA0A+oDP8V0OhICNCr5lv96FvSsDEmrb0GKM776JgQvHTMjr7DTTPEVv/1Ciaw0uEWzBA==} engines: {node: '>=12'} @@ -4659,6 +4752,9 @@ packages: engines: {node: '>=6'} hasBin: true + json-bigint@1.0.0: + resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} + json-buffer@3.0.1: resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} @@ -4703,6 +4799,12 @@ packages: engines: {node: '>=18.17'} hasBin: true + jwa@2.0.1: + resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} + + jws@4.0.1: + resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} + kapsule@1.16.3: resolution: {integrity: sha512-4+5mNNf4vZDSwPhKprKwz3330iisPrb08JyMgbsdFrimBCKNHecua/WBwvVg3n7vwx0C1ARjfhwIpbrbd9n5wg==} engines: {node: '>=12'} @@ -4842,6 +4944,9 @@ packages: loupe@3.2.1: resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} + lru-cache@10.4.3: + resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + lru-cache@11.3.6: resolution: {integrity: sha512-Gf/KoL3C/MlI7Bt0PGI9I+TeTC/I6r/csU58N4BSNc4lppLBeKsOdFYkK+dX0ABDUMJNfCHTyPpzwwO21Awd3A==} engines: {node: 20 || >=22} @@ -5078,6 +5183,10 @@ packages: minimatch@3.1.5: resolution: {integrity: sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==} + minimatch@9.0.9: + resolution: {integrity: sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==} + engines: {node: '>=16 || 14 >=14.17'} + minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} @@ -5154,6 +5263,11 @@ packages: node-abort-controller@3.1.1: resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + node-fetch-native@1.6.7: resolution: {integrity: sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==} @@ -5166,6 +5280,10 @@ packages: encoding: optional: true + node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + node-gyp-build-optional-packages@5.2.2: resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==} hasBin: true @@ -5281,6 +5399,9 @@ packages: resolution: {integrity: sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg==} engines: {node: '>=20'} + package-json-from-dist@1.0.1: + resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} + package-manager-detector@1.7.0: resolution: {integrity: sha512-xg1eHpwYL/D/HEdWw2goFZP6vV0FH7W+PZ5rFkGjdIDLtxq7EkzBUeT3m+lndYCt8wKbmofUu1MUdMCXkCk9ZQ==} @@ -5331,6 +5452,10 @@ packages: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} + path-scurry@1.11.1: + resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} + engines: {node: '>=16 || 14 >=14.18'} + path-scurry@2.0.2: resolution: {integrity: sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==} engines: {node: 18 || 20 || >=22} @@ -5769,6 +5894,10 @@ packages: rfdc@1.4.1: resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==} + rimraf@5.0.10: + resolution: {integrity: sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==} + hasBin: true + robust-predicates@3.0.3: resolution: {integrity: sha512-NS3levdsRIUOmiJ8FZWCP7LG3QpJyrs/TE0Zpf1yvZu8cAJJ6QMW92H1c7kWpdIHo8RvmLxN/o2JXTKHp74lUA==} @@ -5897,6 +6026,10 @@ packages: siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + simple-icons@16.25.0: resolution: {integrity: sha512-ReYPfbqjkTBovxjerHw2tf6E1rQdaCqNJKaJbd78Mui8w+78xQlE1aDO/2ekID++n8qa+1t+oaueYLeiSoQsIA==} engines: {node: '>=0.12.18'} @@ -5963,6 +6096,10 @@ packages: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} + string-width@5.1.2: + resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} + engines: {node: '>=12'} + string-width@7.2.0: resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} engines: {node: '>=18'} @@ -6316,6 +6453,9 @@ packages: uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + url-template@2.0.8: + resolution: {integrity: sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==} + use-sync-external-store@1.6.0: resolution: {integrity: sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==} peerDependencies: @@ -6555,6 +6695,10 @@ packages: resolution: {integrity: sha512-Ezr98sqXW/+OCGoUEXuOKVR+oVFlSdn1tIySEEJdiSAw4IjrW8hQkwARSSBJTSB5Us5dnytDgL0ZDliAYBhaNA==} engines: {node: '>=10.0.0'} + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -6604,6 +6748,10 @@ packages: resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} engines: {node: '>=10'} + wrap-ansi@8.1.0: + resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} + engines: {node: '>=12'} + wrap-ansi@9.0.2: resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} engines: {node: '>=18'} @@ -8208,6 +8356,15 @@ snapshots: '@ioredis/commands@1.10.0': {} + '@isaacs/cliui@8.0.2': + dependencies: + string-width: 5.1.2 + string-width-cjs: string-width@4.2.3 + strip-ansi: 7.2.0 + strip-ansi-cjs: strip-ansi@6.0.1 + wrap-ansi: 8.1.0 + wrap-ansi-cjs: wrap-ansi@7.0.0 + '@isaacs/fs-minipass@4.0.1': dependencies: minipass: 7.1.3 @@ -8881,6 +9038,9 @@ snapshots: '@pinojs/redact@0.4.0': {} + '@pkgjs/parseargs@0.11.0': + optional: true + '@prosemirror-adapter/core@0.5.3': dependencies: '@ocavue/utils': 1.6.0 @@ -10127,6 +10287,8 @@ snapshots: dependencies: require-from-string: 2.0.2 + bignumber.js@9.3.1: {} + bindings@1.5.0: dependencies: file-uri-to-path: 1.0.0 @@ -10156,6 +10318,10 @@ snapshots: balanced-match: 1.0.2 concat-map: 0.0.1 + brace-expansion@2.1.2: + dependencies: + balanced-match: 1.0.2 + brace-expansion@5.0.7: dependencies: balanced-match: 4.0.4 @@ -10168,6 +10334,8 @@ snapshots: node-releases: 2.0.50 update-browserslist-db: 1.2.3(browserslist@4.28.5) + buffer-equal-constant-time@1.0.1: {} + buffer-from@1.1.2: {} bullmq@5.76.8: @@ -10605,6 +10773,8 @@ snapshots: dependencies: accessor-fn: 1.5.3 + data-uri-to-buffer@4.0.1: {} + data-urls@7.0.0: dependencies: whatwg-mimetype: 5.0.0 @@ -10720,6 +10890,12 @@ snapshots: es-errors: 1.3.0 gopd: 1.2.0 + eastasianwidth@0.2.0: {} + + ecdsa-sig-formatter@1.0.11: + dependencies: + safe-buffer: 5.1.2 + ee-first@1.1.1: {} electron-to-chromium@1.5.389: {} @@ -10733,6 +10909,8 @@ snapshots: emoji-regex@8.0.0: {} + emoji-regex@9.2.2: {} + encodeurl@2.0.0: {} encoding-sniffer@0.2.1: @@ -11100,6 +11278,11 @@ snapshots: optionalDependencies: picomatch: 4.0.5 + fetch-blob@3.2.0: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + fflate@0.8.3: {} file-entry-cache@8.0.0: @@ -11171,8 +11354,17 @@ snapshots: kapsule: 1.16.3 lodash-es: 4.18.1 + foreground-child@3.3.1: + dependencies: + cross-spawn: 7.0.6 + signal-exit: 4.1.0 + format@0.2.2: {} + formdata-polyfill@4.0.10: + dependencies: + fetch-blob: 3.2.0 + forwarded@0.2.0: {} fresh@2.0.0: {} @@ -11185,6 +11377,39 @@ snapshots: function-bind@1.1.2: {} + gaxios@7.1.3: + dependencies: + extend: 3.0.2 + https-proxy-agent: 7.0.6 + node-fetch: 3.3.2 + rimraf: 5.0.10 + transitivePeerDependencies: + - supports-color + + gaxios@7.2.0: + dependencies: + extend: 3.0.2 + https-proxy-agent: 7.0.6 + node-fetch: 3.3.2 + transitivePeerDependencies: + - supports-color + + gcp-metadata@8.1.2: + dependencies: + gaxios: 7.2.0 + google-logging-utils: 1.1.3 + json-bigint: 1.0.0 + transitivePeerDependencies: + - supports-color + + gcp-metadata@8.1.3: + dependencies: + gaxios: 7.1.3 + google-logging-utils: 1.1.3 + json-bigint: 1.0.0 + transitivePeerDependencies: + - supports-color + gensync@1.0.0-beta.2: {} get-caller-file@2.0.5: {} @@ -11223,6 +11448,15 @@ snapshots: dependencies: is-glob: 4.0.3 + glob@10.5.0: + dependencies: + foreground-child: 3.3.1 + jackspeak: 3.4.3 + minimatch: 9.0.9 + minipass: 7.1.3 + package-json-from-dist: 1.0.1 + path-scurry: 1.11.1 + glob@13.0.6: dependencies: minimatch: 10.2.5 @@ -11233,6 +11467,49 @@ snapshots: globals@17.7.0: {} + google-auth-library@10.5.0: + dependencies: + base64-js: 1.5.1 + ecdsa-sig-formatter: 1.0.11 + gaxios: 7.1.3 + gcp-metadata: 8.1.3 + google-logging-utils: 1.1.3 + gtoken: 8.0.0 + jws: 4.0.1 + transitivePeerDependencies: + - supports-color + + google-auth-library@10.9.0: + dependencies: + base64-js: 1.5.1 + ecdsa-sig-formatter: 1.0.11 + gaxios: 7.2.0 + gcp-metadata: 8.1.2 + google-logging-utils: 1.1.3 + jws: 4.0.1 + transitivePeerDependencies: + - supports-color + + google-logging-utils@1.1.3: {} + + googleapis-common@8.0.2: + dependencies: + extend: 3.0.2 + gaxios: 7.1.3 + google-auth-library: 10.5.0 + google-logging-utils: 1.1.3 + qs: 6.15.2 + url-template: 2.0.8 + transitivePeerDependencies: + - supports-color + + googleapis@173.0.0: + dependencies: + google-auth-library: 10.9.0 + googleapis-common: 8.0.2 + transitivePeerDependencies: + - supports-color + gopd@1.2.0: {} gpt-tokenizer@3.4.0: {} @@ -11287,6 +11564,13 @@ snapshots: events: 3.3.0 graphology-types: 0.24.8 + gtoken@8.0.0: + dependencies: + gaxios: 7.1.3 + jws: 4.0.1 + transitivePeerDependencies: + - supports-color + h3@1.15.11: dependencies: cookie-es: 1.2.3 @@ -11545,6 +11829,12 @@ snapshots: isomorphic.js@0.2.5: {} + jackspeak@3.4.3: + dependencies: + '@isaacs/cliui': 8.0.2 + optionalDependencies: + '@pkgjs/parseargs': 0.11.0 + jerrypick@1.1.2: {} jiti@2.7.0: {} @@ -11591,6 +11881,10 @@ snapshots: jsesc@3.1.0: {} + json-bigint@1.0.0: + dependencies: + bignumber.js: 9.3.1 + json-buffer@3.0.1: {} json-schema-ref-resolver@3.0.0: @@ -11634,6 +11928,17 @@ snapshots: slick: 1.12.2 web-resource-inliner: 8.0.0 + jwa@2.0.1: + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.1.2 + + jws@4.0.1: + dependencies: + jwa: 2.0.1 + safe-buffer: 5.1.2 + kapsule@1.16.3: dependencies: lodash-es: 4.18.1 @@ -11752,6 +12057,8 @@ snapshots: loupe@3.2.1: {} + lru-cache@10.4.3: {} + lru-cache@11.3.6: {} lru-cache@11.5.2: {} @@ -12220,6 +12527,10 @@ snapshots: dependencies: brace-expansion: 1.1.14 + minimatch@9.0.9: + dependencies: + brace-expansion: 2.1.2 + minimist@1.2.8: {} minipass@7.1.3: {} @@ -12288,12 +12599,20 @@ snapshots: node-abort-controller@3.1.1: {} + node-domexception@1.0.0: {} + node-fetch-native@1.6.7: {} node-fetch@2.7.0: dependencies: whatwg-url: 5.0.0 + node-fetch@3.3.2: + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + node-gyp-build-optional-packages@5.2.2: dependencies: detect-libc: 2.1.2 @@ -12388,6 +12707,8 @@ snapshots: p-timeout@7.0.1: {} + package-json-from-dist@1.0.1: {} + package-manager-detector@1.7.0: {} pako@1.0.11: {} @@ -12438,6 +12759,11 @@ snapshots: path-key@3.1.1: {} + path-scurry@1.11.1: + dependencies: + lru-cache: 10.4.3 + minipass: 7.1.3 + path-scurry@2.0.2: dependencies: lru-cache: 11.5.2 @@ -12963,6 +13289,10 @@ snapshots: rfdc@1.4.1: {} + rimraf@5.0.10: + dependencies: + glob: 10.5.0 + robust-predicates@3.0.3: {} rollup@4.62.2: @@ -13163,6 +13493,8 @@ snapshots: siginfo@2.0.0: {} + signal-exit@4.1.0: {} + simple-icons@16.25.0: {} sisteransi@1.0.5: {} @@ -13218,6 +13550,12 @@ snapshots: is-fullwidth-code-point: 3.0.0 strip-ansi: 6.0.1 + string-width@5.1.2: + dependencies: + eastasianwidth: 0.2.0 + emoji-regex: 9.2.2 + strip-ansi: 7.2.0 + string-width@7.2.0: dependencies: emoji-regex: 10.6.0 @@ -13521,6 +13859,8 @@ snapshots: dependencies: punycode: 2.3.1 + url-template@2.0.8: {} + use-sync-external-store@1.6.0(react@19.2.7): dependencies: react: 19.2.7 @@ -13858,6 +14198,8 @@ snapshots: mime: 2.6.0 valid-data-url: 3.0.1 + web-streams-polyfill@3.3.3: {} + webidl-conversions@3.0.1: {} webidl-conversions@8.0.1: {} @@ -13902,6 +14244,12 @@ snapshots: string-width: 4.2.3 strip-ansi: 6.0.1 + wrap-ansi@8.1.0: + dependencies: + ansi-styles: 6.2.3 + string-width: 5.1.2 + strip-ansi: 7.2.0 + wrap-ansi@9.0.2: dependencies: ansi-styles: 6.2.3 diff --git a/scripts/check-licenses.mjs b/scripts/check-licenses.mjs index d93c941a6..effe18aed 100755 --- a/scripts/check-licenses.mjs +++ b/scripts/check-licenses.mjs @@ -79,6 +79,10 @@ const OVERRIDES = { '@sentry/cli-win32-arm64': 'MIT', '@sentry/cli-win32-i686': 'MIT', '@sentry/cli-win32-x64': 'MIT', + // url-template -> BSD-3-Clause (its bundled LICENSE is the 3-clause BSD text; + // package.json reports the legacy non-SPDX "BSD"). Transitive dep pulled in by + // googleapis via googleapis-common. + 'url-template': 'BSD-3-Clause', }; function isAllowedAtom(atom) {