diff --git a/.env.example b/.env.example index 7536fb174..62da0f4bd 100644 --- a/.env.example +++ b/.env.example @@ -46,6 +46,13 @@ UTOPIA_OPEN_REGISTRATION=true # 没配单点登录时这个 false 不生效。单点登录开的账号没有密码、也没法重设,改回 true 也进不去—— # 包括经 SSO 引导出来的首个管理员。务必留一个有密码的管理员账号当后门。 # UTOPIA_PASSWORD_LOGIN=true +# Google Drive 来源(0074):一个人用自己的 Google 账号连接,按他选的文件夹同步。 +# 在 Google Cloud 建一个「Web 应用」类型的 OAuth 客户端,回调地址填下面这个,并启用 Drive API。 +# 三项都填才开放;与上面 SSO 的客户端是两回事(权限范围不同)。drive.readonly 是受限范围: +# Workspace 内部应用不需要 Google 审核,对外发布的需要。 +# UTOPIA_GOOGLE_DRIVE_CLIENT_ID= +# UTOPIA_GOOGLE_DRIVE_CLIENT_SECRET= +# UTOPIA_GOOGLE_DRIVE_REDIRECT_URI=https://your-deployment/api/v1/google-drive/callback # 数据库连接池上限。缺省 32,与 worker 并发的缺省对齐。 # 池子小于并发时症状是请求变慢、而不是任何一处说"池子不够"——调高 worker # 并发时一并考虑这里,也别超过 Postgres 自己的 max_connections(默认 100)。 diff --git a/crates/utopia-cli/src/main.rs b/crates/utopia-cli/src/main.rs index 8f0fe0364..f3683aabd 100644 --- a/crates/utopia-cli/src/main.rs +++ b/crates/utopia-cli/src/main.rs @@ -82,7 +82,7 @@ struct ManifestDataDir { /// not a side effect of a code change. // 是迁移文件的**个数**,不是最大的编号(守卫 `schema_version_policy_compares_against_current` // 按个数比):编号有空缺时两者不同——0071 由一个开放 PR 占着,0072 先落,个数是 71 -const CURRENT_SCHEMA_VERSION: u32 = 105; +const CURRENT_SCHEMA_VERSION: u32 = 106; fn main() -> anyhow::Result<()> { dotenvy::dotenv().ok(); diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index 8eab5f642..890493356 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -259,6 +259,8 @@ pub const SOURCE_SECRET_KEYS: &[&str] = &[ "secret_access_key", "account_key", "service_account_key", + // Google Drive 的 OAuth 刷新令牌(0074):能换出那个人整个 Drive 的只读权限 + "refresh_token", ]; impl Source { @@ -306,6 +308,8 @@ pub enum SourceKind { Gcs, Webdav, Notion, + /// 一个人授权、按他选的文件夹同步(0074) + GoogleDrive, Api, /// 推送的不是文档而是陈述本身(0054):请求体就是开放抽取契约,抽取不问模型 Statements, diff --git a/crates/utopia-server/src/api/google_drive_routes.rs b/crates/utopia-server/src/api/google_drive_routes.rs new file mode 100644 index 000000000..12efe9740 --- /dev/null +++ b/crates/utopia-server/src/api/google_drive_routes.rs @@ -0,0 +1,506 @@ +//! Google Drive 来源的连接与选文件夹(0074)。 +//! +//! **连接由来源的编辑者本人完成。** 他在来源设置里点「连接」,我们记下一次流程 +//! (state、PKCE verifier、是谁、哪个来源),把他送到 Google 的同意页;回调时这个 +//! 浏览器里的会话必须还是他,而且他对那个库还得是 Editor。不核对的话,有人可以 +//! 自己发起一次流程,再诱使别人点完同意——别人的 Drive 就接进了他的来源。 +//! +//! **连上之后,文件夹只由连上的那个人选。** 读的是他的 Drive,库里任何人都看得见同步进来的 +//! 东西:别的编辑者能浏览、能选,就等于能把他没打算交出来的文件夹拉进库里。所以回调记下 +//! `connected_by`,浏览与选择只认这个人;他不再是编辑者时,同步停下来让人重连。 +//! +//! 刷新令牌只进不出:存进来源配置的 `refresh_token`(封印),文件夹浏览器的请求 +//! 在服务器上拿它换访问令牌,浏览器从头到尾拿不到任何一枚 Google 令牌。 + +use axum::extract::{Path, Query, State}; +use axum::response::Html; +use axum::Json; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use serde::Deserialize; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use utopia_core::models::{Role, Source, SourceKind}; +use utopia_core::AppError; +use uuid::Uuid; + +use super::graph_routes::require_kb; +use crate::auth::{self, AuthUser}; +use crate::error::{ApiErr, ApiResult}; +use crate::google_drive::{self, Drive, FolderEntry, OAuthConfig}; +use crate::state::AppState; + +/// 一次连接流程能活多久:同意页上停十分钟还没点完,就当放弃了 +const FLOW_TTL_SECS: u64 = 600; +/// 同时在途的连接流程上限,理由同 SSO:每一次 `connect` 都落一行 +const MAX_PENDING_FLOWS: i64 = 1000; +/// 一个人同时在途的上限:没有它,一个编辑者连点就能占满上面那个总数,十分钟里谁都连不上 +const MAX_PENDING_FLOWS_PER_USER: i64 = 5; +/// 连接这件事落在配置里的几个键。只由连接、断开两条路由写;断开时连同文件夹一起删, +/// 下一个连上的人从头选 +const CONNECTION_KEYS: &[&str] = &[ + "refresh_token", + "account_email", + "connected_at", + "connected_by", + "connected_by_name", + "folders", +]; +/// 一个来源最多选多少个文件夹。选得更多的,选它们共同的上级 +const MAX_FOLDERS: usize = 50; + +fn configured() -> Result { + google_drive::oauth_config().ok_or_else(|| { + AppError::invalid( + "google_drive_not_configured", + "Google Drive is not configured on this server", + ) + }) +} + +/// 取一条 Google Drive 来源,确认它属于路径上的这个库 +async fn drive_source(state: &AppState, kb_id: Uuid, source_id: Uuid) -> Result { + let source = utopia_store::sources::get(&state.pool, source_id).await?; + if source.kb_id != kb_id || SourceKind::parse(&source.kind) != Some(SourceKind::GoogleDrive) { + return Err(AppError::NotFound); + } + Ok(source) +} + +/// 这个来源连着的 Drive 是不是这个人的:浏览与选文件夹只认他(0074 决定 5) +fn require_connector(source: &Source, user_id: Uuid) -> Result<(), AppError> { + if google_drive::connected_by(&source.config) == Some(user_id) { + return Ok(()); + } + Err(AppError::CodedForbidden { + code: "google_drive_not_connector", + message: "Only the person who connected this Google Drive can choose its folders".into(), + }) +} + +/// 别人连着的来源不能直接重连:一点就会换掉他的 Drive、丢掉他选的文件夹。 +/// 要换人,先断开(断开会问一句,说清是谁的 Drive),再由新的人来连 +fn refuse_someone_elses_connection(source: &Source, user_id: Uuid) -> Result<(), AppError> { + match google_drive::connected_by(&source.config) { + Some(connector) if connector != user_id => Err(AppError::CodedForbidden { + code: "google_drive_connected_by_someone_else", + message: "This source reads someone else's Google Drive; disconnect it first".into(), + }), + _ => Ok(()), + } +} + +/// 同一个 Google 账号是不是还连着别的来源。Google 撤销一枚刷新令牌,撤的是这个人对 +/// 我们这个客户端的整份授权,他别的 Drive 来源会一起失效;还有别的在用就不撤 +pub(super) async fn grant_still_used( + pool: &sqlx::PgPool, + source_id: Uuid, + account_email: &str, +) -> Result { + sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM sources + WHERE kind = 'google_drive' AND id <> $1 + AND lower(config->>'account_email') = lower($2) AND config ? 'refresh_token')", + ) + .bind(source_id) + .bind(account_email) + .fetch_one(pool) + .await +} + +/// 重连之后还是不是同一个 Google 账号。邮箱不分大小写;有一边不知道就当换了: +/// 换了账号,上一个账号里选的文件夹不能照着新账号去读 +pub(super) fn same_account(previous: Option<&str>, now: Option<&str>) -> bool { + matches!((previous, now), (Some(a), Some(b)) if a.eq_ignore_ascii_case(b)) +} + +/// 重连之后作废不作废上一份授权:两边的账号都知道、而且不是同一个时才作废。 +/// 有一边不知道,就可能是同一个账号——作废会连刚换来的新令牌一起作废(0074 决定 5) +pub(super) fn revokes_previous(previous: Option<&str>, now: Option<&str>) -> bool { + previous.is_some() && now.is_some() && !same_account(previous, now) +} + +/// 请 Google 作废一枚刷新令牌,除非同一个账号还连着别的来源([`grant_still_used`])。 +/// 不知道是哪个账号就不作废。失败只记一笔:库里已经没有它了 +async fn revoke_unless_still_used( + pool: &sqlx::PgPool, + source_id: Uuid, + token: &str, + account: Option<&str>, +) -> Result<(), sqlx::Error> { + let Some(account) = account else { + return Ok(()); + }; + if grant_still_used(pool, source_id, account).await? { + return Ok(()); + } + let revoked = reqwest::Client::new() + .post("https://oauth2.googleapis.com/revoke") + .form(&[("token", token)]) + .timeout(std::time::Duration::from_secs(10)) + .send() + .await; + match revoked { + Ok(resp) if resp.status().is_success() => {} + Ok(resp) => tracing::warn!(status = %resp.status(), "Google 拒绝作废刷新令牌,库里已删除"), + Err(e) => tracing::warn!(error = %e, "Google 刷新令牌作废失败,库里已删除"), + } + Ok(()) +} + +/// 落一次连接流程。总数与这个人的在途数都在同一条插入里判,满了就不插,回 `None` +pub(super) async fn record_flow( + pool: &sqlx::PgPool, + user_id: Uuid, + source_id: Uuid, +) -> Result, sqlx::Error> { + sqlx::query("DELETE FROM google_drive_flows WHERE expires_at < now()") + .execute(pool) + .await?; + let flow_state = auth::generate_jwt_secret(); + let verifier = auth::generate_jwt_secret(); + let inserted = sqlx::query( + "INSERT INTO google_drive_flows (state, verifier, user_id, source_id, expires_at) + SELECT $1, $2, $3, $4, now() + make_interval(secs => $5) + WHERE (SELECT count(*) FROM google_drive_flows WHERE user_id = $3) < $6 + AND (SELECT count(*) FROM google_drive_flows) < $7", + ) + .bind(&flow_state) + .bind(&verifier) + .bind(user_id) + .bind(source_id) + .bind(FLOW_TTL_SECS as f64) + .bind(MAX_PENDING_FLOWS_PER_USER) + .bind(MAX_PENDING_FLOWS) + .execute(pool) + .await?; + Ok((inserted.rows_affected() == 1).then_some((flow_state, verifier))) +} + +/// 界面据此决定要不要提供这种来源的连接按钮 +pub async fn status(AuthUser(_): AuthUser) -> Json { + Json(json!({ "enabled": google_drive::oauth_config().is_some() })) +} + +/// 开始一次连接:回给界面一个 Google 同意页的地址,界面在弹窗里打开它 +pub async fn connect( + State(state): State, + AuthUser(user): AuthUser, + Path((kb_id, source_id)): Path<(Uuid, Uuid)>, +) -> ApiResult> { + require_kb(&state, &user, kb_id, Role::Editor).await?; + let source = drive_source(&state, kb_id, source_id).await?; + refuse_someone_elses_connection(&source, user.id)?; + let cfg = configured()?; + let (flow_state, verifier) = record_flow(&state.pool, user.id, source_id) + .await? + .ok_or_else(|| { + AppError::invalid( + "google_drive_busy", + "Too many Google Drive connections are in progress; try again in a few minutes", + ) + })?; + let challenge = URL_SAFE_NO_PAD.encode(Sha256::digest(verifier.as_bytes())); + + let mut url = + reqwest::Url::parse(google_drive::AUTHORIZE_URL).map_err(|e| AppError::Other(e.into()))?; + url.query_pairs_mut().extend_pairs([ + ("response_type", "code"), + ("client_id", cfg.client_id.as_str()), + ("redirect_uri", cfg.redirect.as_str()), + ("scope", google_drive::SCOPES), + ("state", flow_state.as_str()), + ("code_challenge", challenge.as_str()), + ("code_challenge_method", "S256"), + // 刷新令牌只在 offline + 每次都请同意时才保证发:重连一个撤过授权的来源也一样 + ("access_type", "offline"), + ("prompt", "consent"), + ("include_granted_scopes", "false"), + ]); + Ok(Json(json!({ "url": url.as_str() }))) +} + +#[derive(Deserialize)] +pub struct Callback { + code: Option, + state: Option, + /// 人在同意页点了取消 + error: Option, +} + +/// 回调落在弹窗里:成功与失败都回一张不带脚本的小页面。界面那头在轮询来源, +/// 看见 `account_email` 出现就关掉弹窗 +pub async fn callback( + State(state): State, + user: Result, + Query(q): Query, +) -> Html { + match finish(&state, user.ok().map(|AuthUser(u)| u), q).await { + Ok(()) => page( + "Google Drive connected", + "You can close this window and choose the folders to sync.", + ), + Err((code, reason)) => { + tracing::warn!(code, reason = %reason, "Google Drive 连接被拒"); + page( + "Google Drive was not connected", + match code { + "cancelled" => "The consent screen was cancelled. Close this window and try again.", + "session" => "This window is not signed in to Utopia as the person who started the connection. Close it and start again from the source settings.", + "expired" => "This connection attempt has expired. Close this window and start again.", + "scope" => "Google did not grant read access to Drive. Close this window, connect again and allow it.", + "taken" => "Someone else connected this source while you were on Google's page. Close this window.", + _ => "Google could not be reached or refused the request. Close this window and try again.", + }, + ) + } + } +} + +pub(super) async fn finish( + state: &AppState, + user: Option, + q: Callback, +) -> Result<(), (&'static str, String)> { + let db = |e: sqlx::Error| ("unavailable", format!("database: {e}")); + let (Some(code), Some(flow_state)) = (q.code, q.state) else { + return Err(match q.error { + Some(e) => ("cancelled", format!("google returned {e}")), + None => ("invalid", "callback without code or state".into()), + }); + }; + // 一次性:先删后用,同一个 state 第二次到就什么都取不出来 + let flow: Option<(String, Uuid, Uuid)> = sqlx::query_as( + "DELETE FROM google_drive_flows WHERE state = $1 AND expires_at > now() + RETURNING verifier, user_id, source_id", + ) + .bind(&flow_state) + .fetch_optional(&state.pool) + .await + .map_err(db)?; + let (verifier, flow_user, source_id) = + flow.ok_or(("expired", "unknown or expired state".to_string()))?; + if let Some(e) = q.error { + return Err(("cancelled", format!("google returned {e}"))); + } + let user = user.filter(|u| u.id == flow_user).ok_or(( + "session", + "callback session is not the user who started it".to_string(), + ))?; + let source = utopia_store::sources::get(&state.pool, source_id) + .await + .map_err(|e| ("invalid", format!("source: {e}")))?; + // 发起之后的这几分钟里权限可能被收回了 + require_kb(state, &user, source.kb_id, Role::Editor) + .await + .map_err(|_| { + ( + "session", + "user is no longer an editor of the knowledge base".to_string(), + ) + })?; + // 同意页上的这几分钟里,别人可能先连上了 + refuse_someone_elses_connection(&source, user.id).map_err(|_| { + ( + "taken", + "someone else connected this source in the meantime".to_string(), + ) + })?; + + let cfg = google_drive::oauth_config().ok_or(("unavailable", "not configured".to_string()))?; + let grant = google_drive::exchange_code(&cfg, &code, &verifier) + .await + .map_err(|e| { + let msg = format!("{e:#}"); + ( + if msg.contains("read access to Drive") { + "scope" + } else { + "unavailable" + }, + msg, + ) + })?; + + let previous_token = source.config["refresh_token"].as_str(); + let previous_account = source.config["account_email"].as_str(); + // 连上的人换了一个 Google 账号:上一个账号里选的文件夹不留,它的授权在没别人用时作废。 + // 同一个账号绝不作废——Google 作废的是整份授权,刚换来的新令牌也会跟着失效 + let changed = !same_account(previous_account, grant.account_email.as_deref()); + let grant_keys = json!({ + "refresh_token": grant.refresh_token, + "account_email": grant.account_email, + // 列表里看不见令牌,界面凭这一项知道「连上了」:没验证过的邮箱不显示,不能拿它当标志 + "connected_at": chrono::Utc::now().to_rfc3339(), + "connected_by": user.id, + // 断开前问一句时说清是谁的 Drive;库的成员列表只给管理员,编辑者那边查不到名字 + "connected_by_name": user.display_name, + }); + let dropped: &[&str] = if changed { &["folders"] } else { &[] }; + utopia_store::sources::patch_config(&state.pool, source_id, &grant_keys, dropped) + .await + .map_err(|e| ("unavailable", format!("saving the grant: {e}")))?; + let revoke = revokes_previous(previous_account, grant.account_email.as_deref()); + if let (true, Some(token)) = (revoke, previous_token) { + revoke_unless_still_used(&state.pool, source_id, token, previous_account) + .await + .map_err(|e| ("unavailable", format!("checking the previous grant: {e}")))?; + } + state.emit_source(source.kb_id); + // 审计不落令牌,只记连上的是哪个 Google 账号 + let _ = utopia_store::audit::record( + &state.pool, + Some(source.kb_id), + user.id, + "source.google_drive_connected", + "source", + Some(source_id), + json!({ "account_email": grant.account_email }), + ) + .await; + Ok(()) +} + +/// 断开:删掉刷新令牌与选的文件夹,再按 [`revoke_unless_still_used`] 请 Google 作废 +pub async fn disconnect( + State(state): State, + AuthUser(user): AuthUser, + Path((kb_id, source_id)): Path<(Uuid, Uuid)>, +) -> ApiResult> { + require_kb(&state, &user, kb_id, Role::Editor).await?; + let source = drive_source(&state, kb_id, source_id).await?; + utopia_store::sources::patch_config(&state.pool, source_id, &json!({}), CONNECTION_KEYS) + .await?; + if let Some(token) = source.config["refresh_token"].as_str() { + revoke_unless_still_used( + &state.pool, + source_id, + token, + source.config["account_email"].as_str(), + ) + .await?; + } + state.emit_source(kb_id); + let _ = utopia_store::audit::record( + &state.pool, + Some(kb_id), + user.id, + "source.google_drive_disconnected", + "source", + Some(source_id), + json!({}), + ) + .await; + Ok(Json(json!({ "ok": true }))) +} + +#[derive(Deserialize)] +pub struct FoldersQuery { + /// 不给 = 第一层(我的云端硬盘、与我共享、各个共享云端硬盘) + parent: Option, +} + +/// 文件夹浏览器:以连上的那个人的身份列子文件夹,只给他本人看 +pub async fn folders( + State(state): State, + AuthUser(user): AuthUser, + Path((kb_id, source_id)): Path<(Uuid, Uuid)>, + Query(q): Query, +) -> ApiResult> { + require_kb(&state, &user, kb_id, Role::Editor).await?; + let source = drive_source(&state, kb_id, source_id).await?; + require_connector(&source, user.id)?; + let cfg = configured()?; + let refresh = source.config["refresh_token"] + .as_str() + .filter(|t| !t.is_empty()) + .ok_or_else(|| { + AppError::invalid( + "google_drive_not_connected", + "Connect a Google account first", + ) + })?; + let unavailable = |e: anyhow::Error| { + AppError::invalid_detail( + "google_drive_unavailable", + "Google Drive could not be read", + format!("{e:#}"), + ) + }; + let drive = Drive::connect(&cfg, refresh).await.map_err(unavailable)?; + let folders: Vec = match q.parent.as_deref().filter(|p| !p.is_empty()) { + None => drive.roots().await, + Some(parent) => drive.child_folders(parent).await, + } + .map_err(unavailable)?; + Ok(Json(json!({ "folders": folders }))) +} + +#[derive(Deserialize)] +pub struct SelectBody { + folders: Vec, +} + +/// 存下选中的文件夹,并立即同步一次。只有连上的那个人能选。换了文件夹,上一批里 +/// 不在新范围内的文档会在这一轮被标成「不在来源中」,删不删由人决定,与别的来源一样 +pub async fn select( + State(state): State, + AuthUser(user): AuthUser, + Path((kb_id, source_id)): Path<(Uuid, Uuid)>, + Json(body): Json, +) -> ApiResult> { + require_kb(&state, &user, kb_id, Role::Editor).await?; + let source = drive_source(&state, kb_id, source_id).await?; + require_connector(&source, user.id)?; + if body.folders.len() > MAX_FOLDERS { + return Err(AppError::invalid( + "google_drive_too_many_folders", + format!("Choose at most {MAX_FOLDERS} folders"), + ) + .into()); + } + if body.folders.iter().any(|f| !google_drive::valid_id(&f.id)) { + return Err(AppError::invalid("google_drive_invalid_folder", "Invalid folder id").into()); + } + let updated = utopia_store::sources::patch_config( + &state.pool, + source_id, + &json!({ "folders": body.folders }), + &[], + ) + .await?; + if !body.folders.is_empty() { + let _ = utopia_store::sources::mark_queued(&state.pool, source_id).await; + utopia_store::jobs::enqueue( + &state.pool, + "sync_source", + json!({ "source_id": source_id }), + ) + .await?; + } + state.emit_source(kb_id); + let _ = utopia_store::audit::record( + &state.pool, + Some(kb_id), + user.id, + "source.updated", + "source", + Some(source_id), + json!({ "google_drive_folders": body.folders.len() }), + ) + .await; + Ok(Json(json!({ "source": updated.without_secrets() }))) +} + +/// 一张小页面,不带脚本。字符串都是这里写死的,不拼任何请求里来的东西 +fn page(title: &str, body: &str) -> Html { + Html(format!( + "{title}\ + \ + \ +

{title}

{body}

" + )) +} + +#[cfg(test)] +#[path = "google_drive_routes_tests.rs"] +mod tests; diff --git a/crates/utopia-server/src/api/google_drive_routes_tests.rs b/crates/utopia-server/src/api/google_drive_routes_tests.rs new file mode 100644 index 000000000..06174863a --- /dev/null +++ b/crates/utopia-server/src/api/google_drive_routes_tests.rs @@ -0,0 +1,412 @@ +//! 连接流程的几道关卡与「谁连上的谁来选」(0074 决定 3、5),连真库跑。 +//! 测试环境没配 OAuth 客户端:过了所有关卡的回调停在「没配置」,不会去敲 Google +use super::*; +use std::sync::Arc; +use utopia_core::models::User; + +struct Fixture { + pool: sqlx::PgPool, + state: AppState, + kb: Uuid, + source: Uuid, + connector: User, + colleague: User, + _dir: tempfile::TempDir, +} + +impl Fixture { + async fn new() -> anyhow::Result> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(None); + }; + let pool = sqlx::PgPool::connect(&url).await?; + utopia_store::db::migrate(&pool).await?; + let id = Uuid::now_v7; + let (org, ws, kb, connector, colleague) = (id(), id(), id(), id(), id()); + // 插进去的只有本地生成的 UUID + sqlx::raw_sql(&format!( + "INSERT INTO organizations(id,name) VALUES ('{org}','google-drive-test'); + INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','google-drive-test'); + INSERT INTO users(id,org_id,email,display_name,password_hash) VALUES + ('{connector}','{org}','{connector}@drive.test','Connector','unused'), + ('{colleague}','{org}','{colleague}@drive.test','Colleague','unused'); + INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ('{kb}','{ws}','drive'); + INSERT INTO kb_members(kb_id,user_id,role) VALUES + ('{kb}','{connector}','editor'), ('{kb}','{colleague}','editor');" + )) + .execute(&pool) + .await?; + let source = utopia_store::sources::create( + &pool, + kb, + "google_drive", + "Drive", + &json!({}), + None, + None, + None, + ) + .await? + .id; + utopia_store::sources::patch_config( + &pool, + source, + &json!({ + "refresh_token": "1//grant", + "connected_by": connector, + "folders": [{"id": "chosen", "name": "Chosen"}], + }), + &[], + ) + .await?; + let dir = tempfile::tempdir()?; + let cfg = utopia_core::config::AppConfig { + data_dir: dir.path().to_string_lossy().into_owned(), + ..Default::default() + }; + let search = Arc::new(utopia_search::SearchIndex::open( + &dir.path().join("search"), + )?); + let state = AppState::new(pool.clone(), &cfg, search, "test-only".into()); + let user = |id| utopia_store::accounts::find_user_by_id(&pool, id); + let (connector, colleague) = (user(connector).await?, user(colleague).await?); + Ok(Some(Self { + state, + kb, + source, + connector: connector.expect("connector exists"), + colleague: colleague.expect("colleague exists"), + pool, + _dir: dir, + })) + } + + async fn flow(&self) -> anyhow::Result { + let (flow_state, _) = record_flow(&self.pool, self.connector.id, self.source) + .await? + .expect("room for a flow"); + Ok(flow_state) + } + + async fn callback(&self, session: Option<&User>, flow_state: &str) -> &'static str { + let q = Callback { + code: Some("code".into()), + state: Some(flow_state.into()), + error: None, + }; + match finish(&self.state, session.cloned(), q).await { + Ok(()) => "connected", + Err((code, _)) => code, + } + } + + async fn demote_connector(&self) -> anyhow::Result<()> { + sqlx::query("UPDATE kb_members SET role = 'viewer' WHERE kb_id = $1 AND user_id = $2") + .bind(self.kb) + .bind(self.connector.id) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn config(&self) -> anyhow::Result { + Ok(utopia_store::sources::get(&self.pool, self.source) + .await? + .config) + } +} + +fn denied_code(result: ApiResult) -> Option<&'static str> { + match result { + Err(ApiErr(AppError::CodedForbidden { code, .. } | AppError::Invalid { code, .. })) => { + Some(code) + } + _ => None, + } +} + +#[tokio::test] +async fn a_state_used_a_second_time_is_expired() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let flow_state = fx.flow().await?; + assert_eq!( + fx.callback(Some(&fx.connector), &flow_state).await, + "unavailable" + ); + assert_eq!( + fx.callback(Some(&fx.connector), &flow_state).await, + "expired" + ); + Ok(()) +} + +#[tokio::test] +async fn a_callback_in_another_session_is_refused() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let flow_state = fx.flow().await?; + assert_eq!( + fx.callback(Some(&fx.colleague), &flow_state).await, + "session" + ); + let flow_state = fx.flow().await?; + assert_eq!(fx.callback(None, &flow_state).await, "session"); + Ok(()) +} + +#[tokio::test] +async fn a_callback_after_the_person_stopped_being_an_editor_is_refused() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let flow_state = fx.flow().await?; + fx.demote_connector().await?; + assert_eq!( + fx.callback(Some(&fx.connector), &flow_state).await, + "session" + ); + Ok(()) +} + +#[tokio::test] +async fn a_callback_after_ten_minutes_is_expired() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let flow_state = fx.flow().await?; + sqlx::query( + "UPDATE google_drive_flows SET expires_at = now() - interval '1 second' WHERE state = $1", + ) + .bind(&flow_state) + .execute(&fx.pool) + .await?; + assert_eq!( + fx.callback(Some(&fx.connector), &flow_state).await, + "expired" + ); + Ok(()) +} + +#[tokio::test] +async fn a_person_has_at_most_five_connections_in_progress() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + for _ in 0..5 { + fx.flow().await?; + } + let sixth = record_flow(&fx.pool, fx.connector.id, fx.source).await?; + let colleagues_first = record_flow(&fx.pool, fx.colleague.id, fx.source).await?; + assert_eq!((sixth.is_some(), colleagues_first.is_some()), (false, true)); + Ok(()) +} + +#[tokio::test] +async fn a_colleague_cannot_browse_or_choose_the_connected_persons_folders() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let path = || Path((fx.kb, fx.source)); + let browsed = folders( + State(fx.state.clone()), + AuthUser(fx.colleague.clone()), + path(), + Query(FoldersQuery { parent: None }), + ) + .await; + let chosen = select( + State(fx.state.clone()), + AuthUser(fx.colleague.clone()), + path(), + Json(SelectBody { + folders: vec![FolderEntry { + id: "theirs".into(), + name: "Theirs".into(), + }], + }), + ) + .await; + assert_eq!( + (denied_code(browsed), denied_code(chosen)), + ( + Some("google_drive_not_connector"), + Some("google_drive_not_connector") + ) + ); + assert_eq!( + fx.config().await?["folders"], + json!([{"id": "chosen", "name": "Chosen"}]) + ); + Ok(()) +} + +#[tokio::test] +async fn the_source_settings_cannot_rewrite_a_drive_connection() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let before = fx.config().await?; + let _ = super::super::sources_routes::update( + State(fx.state.clone()), + AuthUser(fx.colleague.clone()), + Path((fx.kb, fx.source)), + Json(super::super::sources_routes::UpdateBody { + name: None, + config: Some(json!({ + "connected_by": fx.colleague.id, + "folders": [{"id": "theirs", "name": "Theirs"}], + })), + icon: None, + schedule: None, + }), + ) + .await + .map_err(|ApiErr(e)| e)?; + assert_eq!(fx.config().await?, before); + Ok(()) +} + +#[tokio::test] +async fn a_sync_stops_once_the_connected_person_is_no_longer_an_editor() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let reason = |e: anyhow::Error| format!("{e:#}"); + let while_editor = crate::ingest_sources::sync_source(&fx.state, fx.source) + .await + .map_err(reason); + fx.demote_connector().await?; + let after = crate::ingest_sources::sync_source(&fx.state, fx.source) + .await + .map_err(reason); + let says_reconnect = |r: &Result<(), String>| { + r.as_ref() + .is_err_and(|e| e.contains("no longer an editor") && e.contains("reconnect")) + }; + assert_eq!( + (says_reconnect(&while_editor), says_reconnect(&after)), + (false, true) + ); + Ok(()) +} + +#[tokio::test] +async fn a_colleague_cannot_reconnect_the_connected_persons_drive() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let started = connect( + State(fx.state.clone()), + AuthUser(fx.colleague.clone()), + Path((fx.kb, fx.source)), + ) + .await; + assert_eq!( + denied_code(started), + Some("google_drive_connected_by_someone_else") + ); + Ok(()) +} + +#[tokio::test] +async fn a_callback_after_someone_else_connected_is_refused() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let (flow_state, _) = record_flow(&fx.pool, fx.colleague.id, fx.source) + .await? + .expect("room for a flow"); + assert_eq!(fx.callback(Some(&fx.colleague), &flow_state).await, "taken"); + Ok(()) +} + +#[tokio::test] +async fn a_grant_is_kept_while_another_source_uses_the_same_account() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let account = format!("{}@drive.test", Uuid::now_v7()); + let other = utopia_store::sources::create( + &fx.pool, + fx.kb, + "google_drive", + "Other Drive", + &json!({}), + None, + None, + None, + ) + .await? + .id; + // 同一个邮箱,大小写不同:Google 账号不分大小写 + for (id, email) in [ + (fx.source, account.clone()), + (other, account.to_uppercase()), + ] { + let grant = json!({ "refresh_token": "1//grant", "account_email": email }); + utopia_store::sources::patch_config(&fx.pool, id, &grant, &[]).await?; + } + let while_other_connected = grant_still_used(&fx.pool, fx.source, &account).await?; + utopia_store::sources::patch_config(&fx.pool, other, &json!({}), CONNECTION_KEYS).await?; + let after_other_disconnected = grant_still_used(&fx.pool, fx.source, &account).await?; + assert_eq!( + (while_other_connected, after_other_disconnected), + (true, false) + ); + Ok(()) +} + +#[tokio::test] +async fn a_drive_source_cannot_be_created_without_an_oauth_client() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let created = super::super::sources_routes::create( + State(fx.state.clone()), + crate::auth::SessionOrToken { + user: fx.connector.clone(), + pat: None, + }, + Path(fx.kb), + Json(super::super::sources_routes::CreateBody { + kind: "google_drive".into(), + name: "Drive".into(), + config: json!({}), + icon: None, + sync_interval_minutes: None, + sync_cron: None, + }), + ) + .await; + assert_eq!(denied_code(created), Some("google_drive_not_configured")); + Ok(()) +} + +#[test] +fn a_reconnect_keeps_the_grant_only_for_the_same_account() { + assert_eq!( + [ + same_account(Some("Bruno@re-cinq.com"), Some("bruno@re-cinq.com")), + same_account(Some("bruno@re-cinq.com"), Some("ana@re-cinq.com")), + same_account(None, Some("bruno@re-cinq.com")), + same_account(None, None), + ], + [true, false, false, false] + ); +} + +#[test] +fn a_reconnect_revokes_the_previous_grant_only_when_both_accounts_are_known_and_differ() { + assert_eq!( + [ + revokes_previous(Some("bruno@re-cinq.com"), Some("ana@re-cinq.com")), + revokes_previous(Some("bruno@re-cinq.com"), Some("Bruno@re-cinq.com")), + revokes_previous(Some("bruno@re-cinq.com"), None), + revokes_previous(None, Some("ana@re-cinq.com")), + ], + [true, false, false, false] + ); +} diff --git a/crates/utopia-server/src/api/mod.rs b/crates/utopia-server/src/api/mod.rs index 7d488e3f1..2b01f5426 100644 --- a/crates/utopia-server/src/api/mod.rs +++ b/crates/utopia-server/src/api/mod.rs @@ -8,6 +8,7 @@ mod datasource_routes; pub(crate) mod documents_routes; mod events_routes; mod export_routes; +mod google_drive_routes; mod graph_routes; mod jobs_routes; mod kbs; @@ -122,6 +123,9 @@ pub fn router(state: AppState, cfg: &AppConfig) -> Router { .route("/auth/oidc/callback", get(oidc_routes::callback)) // 受信的另一个应用拿它用户刚到手的 ID token 换一个会话(0066) .route("/auth/oidc/exchange", post(oidc_routes::exchange)) + // Google Drive 来源(0074):部署配了 OAuth 客户端才有;回调落在连接弹窗里 + .route("/google-drive/status", get(google_drive_routes::status)) + .route("/google-drive/callback", get(google_drive_routes::callback)) // 我自己的绑定:看、解绑。绑定走 `/auth/oidc/start?link=1`,由本人完成 .route( "/auth/oidc/me", @@ -505,6 +509,18 @@ pub fn router(state: AppState, cfg: &AppConfig) -> Router { 4 * sources_routes::STATEMENTS_MAX_BYTES, )), ) + .route( + "/kbs/{id}/sources/{source_id}/google-drive/connect", + post(google_drive_routes::connect), + ) + .route( + "/kbs/{id}/sources/{source_id}/google-drive/connection", + axum::routing::delete(google_drive_routes::disconnect), + ) + .route( + "/kbs/{id}/sources/{source_id}/google-drive/folders", + get(google_drive_routes::folders).put(google_drive_routes::select), + ) .route( "/kbs/{id}/sources/{source_id}/token", get(sources_routes::get_token), diff --git a/crates/utopia-server/src/api/sources_routes.rs b/crates/utopia-server/src/api/sources_routes.rs index 7a4e62eb8..ddd859dd8 100644 --- a/crates/utopia-server/src/api/sources_routes.rs +++ b/crates/utopia-server/src/api/sources_routes.rs @@ -6,7 +6,7 @@ use axum::Json; use chrono::{DateTime, Utc}; use serde::Deserialize; use serde_json::json; -use utopia_core::models::{Role, SOURCE_SECRET_KEYS}; +use utopia_core::models::{Role, SourceKind, SOURCE_SECRET_KEYS}; use uuid::Uuid; use super::graph_routes::require_kb; @@ -109,12 +109,22 @@ pub async fn create( let user = caller.user; require_kb(&state, &user, kb_id, Role::Editor).await?; validate_rss_config(&body.kind, &body.config)?; + let drive = config_is_drive_owned(&body.kind); + // 没配 OAuth 客户端的部署连不上 Google,建出来也只会一直失败(0074 决定 2) + if drive && crate::google_drive::oauth_config().is_none() { + return Err(utopia_core::AppError::invalid( + "google_drive_not_configured", + "Google Drive is not configured on this server", + ) + .into()); + } + let config = if drive { json!({}) } else { body.config }; let source = utopia_store::sources::create( &state.pool, kb_id, &body.kind, &body.name, - &body.config, + &config, body.icon.as_deref(), body.sync_interval_minutes, body.sync_cron.as_deref(), @@ -214,6 +224,12 @@ fn validate_rss_config(kind: &str, config: &serde_json::Value) -> utopia_core::A Ok(()) } +/// Google Drive 来源的配置只由它自己的连接、选文件夹、断开三条路由写(0074)。 +/// 通用的建、改要是也能写,任何编辑者都能绕过「只有连上的那个人能选文件夹」 +fn config_is_drive_owned(kind: &str) -> bool { + SourceKind::parse(kind) == Some(SourceKind::GoogleDrive) +} + /// 更新时凭据的合并规则,每个 `SOURCE_SECRET_KEYS` 里的键一样:新配置里**没有**这个键 /// 或值是空串 → 保留库里的原值(表单留空就是「别动」);显式 `null` → 删掉; /// 其余照新值。响应从不回显,所以客户端没有办法把旧值原样送回来,规则只能长在这里 @@ -266,7 +282,9 @@ pub async fn update( require_kb(&state, &user, kb_id, Role::Editor).await?; let existing = source_in_kb(&state, kb_id, source_id).await?; // 凭据只进不出:响应从不回显,表单留空 / 没传 = 保留库里原值 - let mut config = body.config; + let mut config = body + .config + .filter(|_| !config_is_drive_owned(&existing.kind)); if let Some(cfg) = config.as_mut() { keep_secrets(cfg, &existing.config); } diff --git a/crates/utopia-server/src/google_drive.rs b/crates/utopia-server/src/google_drive.rs new file mode 100644 index 000000000..c48994e8a --- /dev/null +++ b/crates/utopia-server/src/google_drive.rs @@ -0,0 +1,846 @@ +//! Google Drive 来源:一个人用自己的 Google 账号授权,按他选的文件夹同步(0074)。 +//! +//! 按 [0013](../../docs/decisions/0013-a-source-should-hand-over-its-history.md) 的四条判据: +//! +//! | 判据 | Google Drive | +//! |---|---| +//! | 真实时间戳 | `modifiedTime`,文件内容最后一次改动的时刻 | +//! | 会不会自我推翻 | 方案、纪要、合同的新版本覆盖旧版本 | +//! | 稳定身份 | 文件 id,改名、挪目录都不变 | +//! | 企业知识住不住在那儿 | 大多数团队的文档就住在这儿 | +//! +//! 与 Notion 一样**只交出现状**:修订历史在 API 里有,但 Google 文档的旧修订只能 +//! 导出成当时的全文,抽样哪几个是另一个决定,这一版不做。 +//! +//! ## 授权是一个人的,不是部署的 +//! +//! Notion、对象存储是管理员贴一把密钥;这里是**这个来源的编辑者**点「连接 Google +//! Drive」,在 Google 那边同意只读授权,我们存下刷新令牌(封印,`SOURCE_SECRET_KEYS`)。 +//! OAuth 客户端是部署级的(`UTOPIA_GOOGLE_DRIVE_CLIENT_ID` 等),没配就没有这个按钮。 +//! 授权顺带要 `openid email`,只为在界面上说清「这是以谁的身份在读」。 +//! +//! ## 几个容易踩的 +//! +//! **没有递归查询。** `q` 只认 `'{id}' in parents`,一层一层走;两个选中的文件夹 +//! 互相嵌套时同一个文件会出现两次,按 id 去重。 +//! +//! **共享云端硬盘里的东西默认看不见。** 每个请求都得带 `supportsAllDrives=true`, +//! 列表还得带 `includeItemsFromAllDrives=true`,少了不报错,只是结果是空的。 +//! +//! **Google 自己的格式要导出。** Docs/Sheets/Slides 没有字节,`alt=media` 回 403; +//! 走 `export` 换成 DOCX/XLSX/PPTX,交给已有的解析器(标题层级、表格、多张工作表都在)。 +//! 导出上限 10 MB,超了是 403 `exportSizeLimitExceeded`,按读不出来的那一份处理。 +//! +//! **导出的字节每次都不同**(压缩包里带时间戳),所以「变没变」不能靠内容哈希, +//! 靠 Drive 给的标记:二进制用 `md5Checksum`,Google 格式用 `modifiedTime`。 +//! +//! 错误文案是英文:它会原样落到来源的同步状态里给用户看。 + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::time::Duration; + +use anyhow::Context as _; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use chrono::{DateTime, Utc}; +use reqwest::StatusCode; +use serde_json::Value; +use uuid::Uuid; + +pub const AUTHORIZE_URL: &str = "https://accounts.google.com/o/oauth2/v2/auth"; +const TOKEN_URL: &str = "https://oauth2.googleapis.com/token"; +const API_ROOT: &str = "https://www.googleapis.com/drive/v3"; + +/// 只读整个 Drive。`drive.file` 不够:它只给应用自己建的、或经 Picker 逐个选中的 +/// 文件,选一个文件夹并不连带它里面的东西 +pub const DRIVE_SCOPE: &str = "https://www.googleapis.com/auth/drive.readonly"; +pub const SCOPES: &str = "openid email https://www.googleapis.com/auth/drive.readonly"; + +const FOLDER_MIME: &str = "application/vnd.google-apps.folder"; +const SHORTCUT_MIME: &str = "application/vnd.google-apps.shortcut"; +/// 一个文件要用到的元数据,列表与快捷方式的目标都取这些 +const FILE_FIELDS: &str = "id,name,mimeType,modifiedTime,md5Checksum,size"; + +/// 一次遍历最多列多少个文件。列表只是元数据,标记没变的文件不下载,所以这里放得宽; +/// 它只防一棵大得离谱的树把内存撑爆。到了这个数,这一轮说出来,不报成功 +const MAX_FILES_LISTED: usize = 50_000; +/// 一次同步最多下载(或导出)多少个文件,与 WebDAV、对象存储同一个量级:摄入不可逆。 +/// 上限卡在下载上而不是列表上:列表每轮都是全的,没变的不花钱,到了上限的这一轮 +/// 说出来,下一轮从还没下的接着下,不会每次都停在同一个地方(#1212 那类缺口) +pub const MAX_DOWNLOADS_PER_SYNC: usize = 2_000; +/// 单个文件的上限,同对象存储 +const MAX_FILE_BYTES: u64 = 32 * 1024 * 1024; +/// 选中文件夹往下走几层。再深的留着不走,比一轮同步卡在一棵树上好 +const MAX_DEPTH: usize = 16; +/// 文件夹浏览器一次最多回多少个子文件夹 +const MAX_FOLDERS_LISTED: usize = 1_000; +/// 429 / 5xx 最多再试几回 +const MAX_RETRIES: u32 = 3; +const JSON_LIMIT: usize = 4 * 1024 * 1024; + +/// 部署级的 OAuth 客户端。三项都得有,缺一项就当没配 +#[derive(Clone)] +pub struct OAuthConfig { + pub client_id: String, + pub client_secret: String, + pub redirect: String, +} + +pub fn oauth_config() -> Option { + let get = |key: &str| { + std::env::var(key) + .ok() + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) + }; + Some(OAuthConfig { + client_id: get("UTOPIA_GOOGLE_DRIVE_CLIENT_ID")?, + client_secret: get("UTOPIA_GOOGLE_DRIVE_CLIENT_SECRET")?, + redirect: get("UTOPIA_GOOGLE_DRIVE_REDIRECT_URI")?, + }) +} + +/// 文件、文件夹、共享盘的 id 只有这些字符。它们要拼进 `q` 的单引号里, +/// 不在这张表上的一律不收——比转义可靠 +pub fn valid_id(id: &str) -> bool { + !id.is_empty() + && id.len() <= 128 + && id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') +} + +/// 来源配置里选中的文件夹:`folders: [{id, name}]`。名字只给界面看,同步只认 id +pub fn selected_folders(config: &Value) -> Vec { + config["folders"] + .as_array() + .map(|a| { + a.iter() + .filter_map(|f| f["id"].as_str().or_else(|| f.as_str())) + .map(str::trim) + .filter(|id| valid_id(id)) + .map(String::from) + .collect() + }) + .unwrap_or_default() +} + +/// 谁连上的这个来源:回调时记下的 Utopia 用户。没有 = 还没连,或是这条规则之前连的 +pub fn connected_by(config: &Value) -> Option { + config["connected_by"].as_str()?.parse().ok() +} + +/// 授权码换来的东西 +#[derive(Debug)] +pub struct Grant { + pub refresh_token: String, + /// 授权的那个 Google 账号,ID token 里核验过的邮箱;只用来在界面上显示 + pub account_email: Option, +} + +fn http() -> anyhow::Result { + Ok(reqwest::Client::builder() + .timeout(Duration::from_secs(60)) + .build()?) +} + +/// 授权码 + PKCE verifier 换令牌。要的是刷新令牌:访问令牌一小时就过期, +/// 同步是在没人看着的时候跑的 +pub async fn exchange_code(cfg: &OAuthConfig, code: &str, verifier: &str) -> anyhow::Result { + exchange_code_at(&http()?, TOKEN_URL, cfg, code, verifier).await +} + +async fn exchange_code_at( + http: &reqwest::Client, + token_url: &str, + cfg: &OAuthConfig, + code: &str, + verifier: &str, +) -> anyhow::Result { + let resp = http + .post(token_url) + .form(&[ + ("grant_type", "authorization_code"), + ("code", code), + ("code_verifier", verifier), + ("client_id", &cfg.client_id), + ("client_secret", &cfg.client_secret), + ("redirect_uri", &cfg.redirect), + ]) + .send() + .await + .context("google token")?; + let v = token_response(resp).await?; + // 同意页上 Google 允许人把某一项权限的勾去掉;去掉了 Drive 的那一项, + // 令牌照发,只是什么都读不到。在这里说清楚,比让第一次同步报 403 好 + let scopes = v["scope"].as_str().unwrap_or(""); + anyhow::ensure!( + scopes.split_whitespace().any(|s| s == DRIVE_SCOPE), + "Google did not grant read access to Drive; connect again and allow it" + ); + // 只有第一次同意(或 `prompt=consent`)才发刷新令牌;`begin` 每次都带 consent + let refresh_token = v["refresh_token"] + .as_str() + .filter(|t| !t.is_empty()) + .context("Google returned no refresh token; connect again")? + .to_string(); + let account_email = match v["id_token"].as_str() { + Some(t) => id_token_email(t, &cfg.client_id)?, + None => None, + }; + Ok(Grant { + refresh_token, + account_email, + }) +} + +/// 刷新令牌换一个访问令牌 +async fn access_token_at( + http: &reqwest::Client, + token_url: &str, + cfg: &OAuthConfig, + refresh_token: &str, +) -> anyhow::Result { + let resp = http + .post(token_url) + .form(&[ + ("grant_type", "refresh_token"), + ("refresh_token", refresh_token), + ("client_id", &cfg.client_id), + ("client_secret", &cfg.client_secret), + ]) + .send() + .await + .context("google token")?; + let v = token_response(resp).await?; + Ok(v["access_token"] + .as_str() + .filter(|t| !t.is_empty()) + .context("google token: response has no access_token")? + .to_string()) +} + +/// 令牌端点的响应。`invalid_grant` 单独说:那是人撤了授权、改了密码,或者 +/// 刷新令牌半年没用过期了——重连能好,重试不能 +async fn token_response(resp: reqwest::Response) -> anyhow::Result { + let status = resp.status(); + let v: Value = resp + .json() + .await + .context("google token: reading the response")?; + if status.is_success() { + return Ok(v); + } + if v["error"] == "invalid_grant" { + anyhow::bail!( + "Google Drive access was revoked or has expired; reconnect the source in its settings" + ); + } + anyhow::bail!( + "google token returned {status}: {}", + v["error_description"] + .as_str() + .or(v["error"].as_str()) + .unwrap_or("no message") + ) +} + +/// ID token 里的邮箱。 +/// +/// **不验签名**:这枚令牌是我们自己拿授权码、经 TLS 直接从 Google 的令牌端点换回来的, +/// OIDC Core 3.1.3.7 允许这时以 TLS 代替签名校验。issuer 与 audience 照查——查的是 +/// 「这是 Google 签给本部署的」。邮箱只拿来显示,不认身份、不开账号; +/// 没验证过的邮箱不显示,免得界面上写着一个那人并不拥有的地址 +fn id_token_email(id_token: &str, client_id: &str) -> anyhow::Result> { + let payload = id_token + .split('.') + .nth(1) + .context("google id_token is not a JWT")?; + let claims: Value = serde_json::from_slice( + &URL_SAFE_NO_PAD + .decode(payload.trim_end_matches('=')) + .context("google id_token payload is not base64url")?, + ) + .context("google id_token payload is not JSON")?; + anyhow::ensure!( + matches!( + claims["iss"].as_str(), + Some("https://accounts.google.com" | "accounts.google.com") + ), + "google id_token has an unexpected issuer" + ); + anyhow::ensure!( + claims["aud"].as_str() == Some(client_id), + "google id_token was issued to a different client" + ); + anyhow::ensure!( + claims["exp"] + .as_i64() + .is_some_and(|exp| exp > Utc::now().timestamp()), + "google id_token has expired" + ); + Ok(match claims["email_verified"].as_bool() { + Some(true) => claims["email"].as_str().map(String::from), + _ => None, + }) +} + +/// 文件夹浏览器里的一项 +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct FolderEntry { + pub id: String, + pub name: String, +} + +/// 选中的文件夹里的一个文件 +#[derive(Debug, Clone)] +pub struct DriveFile { + pub id: String, + pub name: String, + pub mime: String, + pub modified: Option>, + md5: Option, + modified_raw: Option, +} + +impl DriveFile { + /// `gdrive://{file_id}`——文件 id 是它最稳的身份 + pub fn external_key(&self) -> String { + format!("gdrive://{}", self.id) + } + + /// 「变没变」的标记,见模块注释 + pub fn marker(&self) -> String { + self.md5 + .clone() + .or_else(|| self.modified_raw.clone()) + .unwrap_or_default() + } +} + +/// 一轮遍历的结果 +#[derive(Debug, Default)] +pub struct Listing { + pub files: Vec, + /// 到了文件数上限,或有文件夹深过 `MAX_DEPTH` + pub truncated: bool, + /// 选中却打不开的文件夹(删了、或不再共享给这个人)。有它时不对账: + /// 打不开不等于里面的东西都没了 + pub unreachable: Vec, + /// 跳过的文件,每个一句原因:表单、超大的 + pub skipped: usize, + /// 指向这个人打不开的文件的快捷方式(没共享给他、删了),按快捷方式的名字 + pub broken_shortcuts: Vec, +} + +/// 读一个文件拿到的字节 +pub struct Content { + pub bytes: Vec, + pub filename: String, + pub mime: &'static str, +} + +/// Google 自己的格式导出成什么。不在这张表上的 Google 格式(表单、绘图、 +/// 快捷方式、站点……)没有能读的正文,跳过 +fn export_as(mime: &str) -> Option<(&'static str, &'static str)> { + Some(match mime { + "application/vnd.google-apps.document" => ( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "docx", + ), + "application/vnd.google-apps.spreadsheet" => ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "xlsx", + ), + "application/vnd.google-apps.presentation" => ( + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "pptx", + ), + _ => return None, + }) +} + +/// 拿着一个访问令牌的 Drive 客户端 +pub struct Drive { + http: reqwest::Client, + token: std::sync::Mutex, + /// 访问令牌一小时就过期,一轮同步可以更长:遇到 401 凭刷新令牌换一枚新的,再试一次 + renewal: Option, + // 私有地址入口让 HTTP 测试走真实的遍历逻辑,不增加用户配置 + api_root: String, +} + +struct Renewal { + cfg: OAuthConfig, + refresh_token: String, + token_url: String, +} + +impl Drive { + /// 用刷新令牌换访问令牌,得到一个能用的客户端 + pub async fn connect(cfg: &OAuthConfig, refresh_token: &str) -> anyhow::Result { + let http = http()?; + let token = access_token_at(&http, TOKEN_URL, cfg, refresh_token).await?; + Ok(Self { + http, + token: std::sync::Mutex::new(token), + renewal: Some(Renewal { + cfg: cfg.clone(), + refresh_token: refresh_token.into(), + token_url: TOKEN_URL.into(), + }), + api_root: API_ROOT.into(), + }) + } + + /// 发一次 GET。429 与 5xx 退避重试;401 换一枚访问令牌再试一次(只一次:换来的也被拒, + /// 就是授权本身出了问题);其余非 2xx 带上 Google 自己的 message 报错, + /// 状态码与 `reason` 留在错误里给调用方分辨(404 = 打不开) + async fn get( + &self, + what: &str, + url: &str, + query: &[(&str, &str)], + ) -> anyhow::Result { + let url = reqwest::Url::parse_with_params(url, query) + .with_context(|| format!("google drive {what}: building the URL"))?; + let mut attempt = 0u32; + let mut renewed = false; + loop { + let token = self.token.lock().expect("token lock").clone(); + let resp = self + .http + .get(url.clone()) + .bearer_auth(&token) + .send() + .await + .with_context(|| format!("google drive {what}"))?; + let status = resp.status(); + if let (StatusCode::UNAUTHORIZED, false, Some(r)) = (status, renewed, &self.renewal) { + renewed = true; + let fresh = + access_token_at(&self.http, &r.token_url, &r.cfg, &r.refresh_token).await?; + *self.token.lock().expect("token lock") = fresh; + continue; + } + if (status == StatusCode::TOO_MANY_REQUESTS || status.is_server_error()) + && attempt < MAX_RETRIES + { + attempt += 1; + let wait = Duration::from_millis(500 * 2u64.pow(attempt)); + tracing::info!(what, attempt, %status, "google drive busy, waiting before the next attempt"); + tokio::time::sleep(wait).await; + continue; + } + if status.is_success() { + return Ok(resp); + } + let v: Value = resp.json().await.unwrap_or(Value::Null); + let reason = v["error"]["errors"][0]["reason"].as_str().unwrap_or(""); + return Err(DriveError { + status, + message: format!( + "google drive {what} returned {status}{}: {}", + if reason.is_empty() { + String::new() + } else { + format!(" ({reason})") + }, + v["error"]["message"].as_str().unwrap_or("no message") + ), + } + .into()); + } + } + + async fn get_json( + &self, + what: &str, + url: &str, + query: &[(&str, &str)], + ) -> anyhow::Result { + let resp = self.get(what, url, query).await?; + let bytes = read_capped(resp, JSON_LIMIT as u64) + .await + .with_context(|| format!("google drive {what}"))?; + serde_json::from_slice(&bytes).with_context(|| format!("google drive {what}: not JSON")) + } + + /// 按 `q` 列文件,翻完所有页或到 `limit` 为止。第二项:是不是因为到了上限才停的 + async fn list( + &self, + q: &str, + fields: &str, + limit: usize, + ) -> anyhow::Result<(Vec, bool)> { + let url = format!("{}/files", self.api_root); + let fields = format!("nextPageToken,files({fields})"); + let mut out = Vec::new(); + let mut page_token: Option = None; + loop { + let mut query = vec![ + ("q", q), + ("fields", fields.as_str()), + ("pageSize", "1000"), + ("orderBy", "name"), + ("supportsAllDrives", "true"), + ("includeItemsFromAllDrives", "true"), + ]; + if let Some(t) = &page_token { + query.push(("pageToken", t)); + } + let v = self.get_json("files.list", &url, &query).await?; + for f in v["files"].as_array().into_iter().flatten() { + if out.len() >= limit { + return Ok((out, true)); + } + out.push(f.clone()); + } + page_token = v["nextPageToken"].as_str().map(String::from); + if page_token.is_none() { + return Ok((out, false)); + } + } + } + + /// 文件夹浏览器的第一层:我的云端硬盘、与我共享的、每个看得见的共享云端硬盘。 + /// 前两项的名字由界面按 id 换成它的语言 + pub async fn roots(&self) -> anyhow::Result> { + let mut out = vec![ + FolderEntry { + id: "root".into(), + name: "My Drive".into(), + }, + FolderEntry { + id: SHARED_WITH_ME.into(), + name: "Shared with me".into(), + }, + ]; + let url = format!("{}/drives", self.api_root); + let mut page_token: Option = None; + loop { + let mut query = vec![ + ("pageSize", "100"), + ("fields", "nextPageToken,drives(id,name)"), + ]; + if let Some(t) = &page_token { + query.push(("pageToken", t)); + } + let v = self.get_json("drives.list", &url, &query).await?; + for d in v["drives"].as_array().into_iter().flatten() { + if let (Some(id), Some(name)) = (d["id"].as_str(), d["name"].as_str()) { + out.push(FolderEntry { + id: id.into(), + name: name.into(), + }); + } + } + page_token = v["nextPageToken"].as_str().map(String::from); + if page_token.is_none() || out.len() >= MAX_FOLDERS_LISTED { + return Ok(out); + } + } + } + + /// 一个文件夹(或共享盘、或「与我共享」)下面的子文件夹 + pub async fn child_folders(&self, parent: &str) -> anyhow::Result> { + let q = if parent == SHARED_WITH_ME { + format!("sharedWithMe = true and mimeType = '{FOLDER_MIME}' and trashed = false") + } else { + anyhow::ensure!(valid_id(parent), "invalid folder id"); + format!("'{parent}' in parents and mimeType = '{FOLDER_MIME}' and trashed = false") + }; + let (items, _) = self.list(&q, "id,name", MAX_FOLDERS_LISTED).await?; + Ok(items + .iter() + .filter_map(|f| { + Some(FolderEntry { + id: f["id"].as_str()?.into(), + name: f["name"].as_str()?.into(), + }) + }) + .collect()) + } + + /// 走遍选中的文件夹,收集文件。 + /// + /// 广度优先,访问过的文件夹不再进(两个选中的文件夹互相嵌套时),文件按 id 去重 + pub async fn walk(&self, folders: &[String]) -> anyhow::Result { + let mut listing = Listing::default(); + let mut visited: HashSet = HashSet::new(); + let mut seen_files: HashSet = HashSet::new(); + let mut queue: VecDeque<(String, usize)> = VecDeque::new(); + + for id in folders { + match self.folder_is_open(id).await { + Ok(true) => queue.push_back((id.clone(), 0)), + // 进了回收站:里面的东西对这个人已经不算数了,不进去,让对账把它们标出来 + Ok(false) => {} + Err(e) if is_not_found(&e) => listing.unreachable.push(id.clone()), + Err(e) => return Err(e), + } + } + + while let Some((folder, depth)) = queue.pop_front() { + if !visited.insert(folder.clone()) { + continue; + } + let q = format!("'{folder}' in parents and trashed = false"); + let room = MAX_FILES_LISTED.saturating_sub(listing.files.len()) + MAX_FOLDERS_LISTED; + let fields = format!("{FILE_FIELDS},shortcutDetails(targetId,targetMimeType)"); + let (items, cut) = self.list(&q, &fields, room).await?; + if cut { + listing.truncated = true; + } + for item in items { + // 别人组织的会议,纪要与转写住在他的 Drive 里,这边的文件夹里只有快捷方式: + // 换成它指向的那一个,之后与普通文件一样 + let f = if item["mimeType"] == SHORTCUT_MIME { + match self.shortcut_target(&item).await { + Ok(Some(target)) => target, + Ok(None) => { + listing.skipped += 1; + continue; + } + Err(e) if is_inaccessible(&e) => { + let name = item["name"].as_str().unwrap_or("a shortcut"); + listing.broken_shortcuts.push(name.to_string()); + continue; + } + Err(e) => return Err(e), + } + } else { + item + }; + let (Some(id), Some(name), Some(mime)) = + (f["id"].as_str(), f["name"].as_str(), f["mimeType"].as_str()) + else { + continue; + }; + // Drive 回来的 id 之后要拼进查询和地址:与选中的文件夹同一个字符集,不合的不收 + if !valid_id(id) { + listing.skipped += 1; + continue; + } + if mime == FOLDER_MIME { + if depth + 1 > MAX_DEPTH { + listing.truncated = true; + } else { + queue.push_back((id.to_string(), depth + 1)); + } + continue; + } + let google_format = mime.starts_with("application/vnd.google-apps."); + if google_format && export_as(mime).is_none() { + listing.skipped += 1; + continue; + } + // Drive 的 size 是字符串里的整数;Google 格式没有 size + let size = f["size"].as_str().and_then(|s| s.parse::().ok()); + if !google_format && size.is_none_or(|n| n == 0 || n > MAX_FILE_BYTES) { + listing.skipped += 1; + continue; + } + if !seen_files.insert(id.to_string()) { + continue; + } + if listing.files.len() >= MAX_FILES_LISTED { + listing.truncated = true; + return Ok(listing); + } + let modified_raw = f["modifiedTime"].as_str().map(String::from); + listing.files.push(DriveFile { + id: id.into(), + name: name.into(), + mime: mime.into(), + modified: modified_raw + .as_deref() + .and_then(|s| DateTime::parse_from_rfc3339(s).ok()) + .map(|d| d.with_timezone(&Utc)), + md5: f["md5Checksum"].as_str().map(String::from), + modified_raw, + }); + } + } + Ok(listing) + } + + /// 快捷方式指向的那一个。指向文件夹的照文件夹走;指向文件的取目标自己的元数据, + /// 变更标记、大小、导出格式都按目标算,文件也按目标的 id 认(直接选到与经快捷方式 + /// 到达的是同一份)。目标在回收站里 → `None`;打不开 → 403/404 的错误 + async fn shortcut_target(&self, shortcut: &Value) -> anyhow::Result> { + let details = &shortcut["shortcutDetails"]; + let Some(target) = details["targetId"].as_str().filter(|id| valid_id(id)) else { + return Ok(None); + }; + if details["targetMimeType"] == FOLDER_MIME { + return Ok(Some(serde_json::json!({ + "id": target, + "name": shortcut["name"], + "mimeType": FOLDER_MIME, + }))); + } + let url = format!("{}/files/{target}", self.api_root); + let fields = format!("{FILE_FIELDS},trashed"); + let v = self + .get_json( + "files.get", + &url, + &[("fields", fields.as_str()), ("supportsAllDrives", "true")], + ) + .await?; + Ok((v["trashed"].as_bool() != Some(true)).then_some(v)) + } + + /// 选中的这个还在不在:在且没进回收站 → true;进了回收站 → false; + /// 打不开 → 404 的错误。`root` 是「我的云端硬盘」,共享盘的 id 也能当文件夹用 + async fn folder_is_open(&self, id: &str) -> anyhow::Result { + let url = format!("{}/files/{id}", self.api_root); + let v = self + .get_json( + "files.get", + &url, + &[ + ("fields", "id,mimeType,trashed"), + ("supportsAllDrives", "true"), + ], + ) + .await?; + anyhow::ensure!( + v["mimeType"] == FOLDER_MIME, + "google drive: selected item {id} is not a folder" + ); + Ok(v["trashed"].as_bool() != Some(true)) + } + + /// 一个文件的字节:Google 格式导出,其余原样下载 + pub async fn content(&self, file: &DriveFile) -> anyhow::Result { + anyhow::ensure!( + valid_id(&file.id), + "google drive: file id {:?} is not a Drive id", + file.id + ); + if let Some((mime, ext)) = export_as(&file.mime) { + let url = format!("{}/files/{}/export", self.api_root, file.id); + let resp = self.get("export", &url, &[("mimeType", mime)]).await?; + let bytes = read_capped(resp, MAX_FILE_BYTES).await?; + let filename = if file.name.to_lowercase().ends_with(&format!(".{ext}")) { + file.name.clone() + } else { + format!("{}.{ext}", file.name) + }; + return Ok(Content { + bytes, + filename, + mime, + }); + } + let url = format!("{}/files/{}", self.api_root, file.id); + let resp = self + .get( + "download", + &url, + &[("alt", "media"), ("supportsAllDrives", "true")], + ) + .await?; + // 不猜 mime,理由同对象存储:解析先看魔数、再看扩展名 + Ok(Content { + bytes: read_capped(resp, MAX_FILE_BYTES).await?, + filename: file.name.clone(), + mime: "application/octet-stream", + }) + } +} + +/// 文件夹浏览器里「与我共享」的占位 id。真的 id 里没有冒号,撞不上 +pub const SHARED_WITH_ME: &str = "shared:with-me"; + +/// Drive 的一个非 2xx 响应 +#[derive(Debug)] +struct DriveError { + status: StatusCode, + message: String, +} + +impl std::fmt::Display for DriveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for DriveError {} + +fn is_not_found(e: &anyhow::Error) -> bool { + e.downcast_ref::() + .is_some_and(|d| d.status == StatusCode::NOT_FOUND) +} + +/// 打不开:Drive 对没共享给这个人的文件有时回 404,有时回 403 +fn is_inaccessible(e: &anyhow::Error) -> bool { + e.downcast_ref::() + .is_some_and(|d| d.status == StatusCode::NOT_FOUND || d.status == StatusCode::FORBIDDEN) +} + +/// 读一个限了体积的响应体。先看 Content-Length,再边读边数 +async fn read_capped(mut resp: reqwest::Response, limit: u64) -> anyhow::Result> { + if resp.content_length().is_some_and(|n| n > limit) { + anyhow::bail!("response is larger than {limit} bytes"); + } + let mut body = Vec::new(); + while let Some(chunk) = resp.chunk().await? { + body.extend_from_slice(&chunk); + if body.len() as u64 > limit { + anyhow::bail!("response is larger than {limit} bytes"); + } + } + Ok(body) +} + +/// 上一轮摄入时每个文件的标记 +pub async fn markers( + pool: &sqlx::PgPool, + source_id: Uuid, +) -> anyhow::Result> { + let rows: Vec<(String, String)> = + sqlx::query_as("SELECT file_id, marker FROM google_drive_files WHERE source_id = $1") + .bind(source_id) + .fetch_all(pool) + .await?; + Ok(rows.into_iter().collect()) +} + +pub async fn remember( + pool: &sqlx::PgPool, + source_id: Uuid, + file_id: &str, + marker: &str, +) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO google_drive_files (source_id, file_id, marker) VALUES ($1, $2, $3) + ON CONFLICT (source_id, file_id) DO UPDATE SET marker = EXCLUDED.marker", + ) + .bind(source_id) + .bind(file_id) + .bind(marker) + .execute(pool) + .await?; + Ok(()) +} + +/// 这一轮没出现的文件忘掉标记:它再出现时得重新读一遍,不能凭旧标记跳过。 +/// 只在遍历完整的轮次调用,与对账同一个条件 +pub async fn forget_others( + pool: &sqlx::PgPool, + source_id: Uuid, + seen: &[String], +) -> anyhow::Result<()> { + sqlx::query("DELETE FROM google_drive_files WHERE source_id = $1 AND NOT (file_id = ANY($2))") + .bind(source_id) + .bind(seen) + .execute(pool) + .await?; + Ok(()) +} + +#[cfg(test)] +#[path = "google_drive_tests.rs"] +mod tests; diff --git a/crates/utopia-server/src/google_drive_tests.rs b/crates/utopia-server/src/google_drive_tests.rs new file mode 100644 index 000000000..c248057d8 --- /dev/null +++ b/crates/utopia-server/src/google_drive_tests.rs @@ -0,0 +1,630 @@ +//! Google Drive 的遍历、导出与授权。响应的形状照 Drive v3 与 Google 令牌端点的文档写, +//! **没有对一个真实的 Drive 跑过**(0013 决定 4 要求说出来):真连的那条在最下面, +//! 给了令牌才跑。 +use super::*; +use serde_json::json; +use wiremock::matchers::{body_string_contains, header, method, path, query_param}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +fn drive(server: &MockServer) -> Drive { + Drive { + // 本地 mock 不走开发机的系统代理 + http: reqwest::Client::builder().no_proxy().build().unwrap(), + token: std::sync::Mutex::new("ya29.test".into()), + renewal: None, + api_root: server.uri(), + } +} + +fn cfg() -> OAuthConfig { + OAuthConfig { + client_id: "client-1.apps.googleusercontent.com".into(), + client_secret: "secret".into(), + redirect: "https://utopia.example/api/v1/google-drive/callback".into(), + } +} + +fn folder_meta(id: &str, trashed: bool) -> serde_json::Value { + json!({"id": id, "mimeType": FOLDER_MIME, "trashed": trashed}) +} + +async fn mount_folder(server: &MockServer, id: &str, trashed: bool) { + Mock::given(method("GET")) + .and(path(format!("/files/{id}"))) + .respond_with(ResponseTemplate::new(200).set_body_json(folder_meta(id, trashed))) + .mount(server) + .await; +} + +async fn mount_children(server: &MockServer, parent: &str, files: serde_json::Value) { + Mock::given(method("GET")) + .and(path("/files")) + .and(query_param( + "q", + format!("'{parent}' in parents and trashed = false"), + )) + .and(query_param("supportsAllDrives", "true")) + .and(query_param("includeItemsFromAllDrives", "true")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "files": files }))) + .mount(server) + .await; +} + +/// Google 签的 ID token 一小时后过期;没写 `exp` 的按刚签发算 +fn id_token(mut claims: serde_json::Value) -> String { + if claims.get("exp").is_none() { + claims["exp"] = json!(Utc::now().timestamp() + 3600); + } + format!( + "{}.{}.sig", + URL_SAFE_NO_PAD.encode(br#"{"alg":"RS256"}"#), + URL_SAFE_NO_PAD.encode(claims.to_string()) + ) +} + +/// 子文件夹往下走;Google 表单这类没有正文的格式与超大文件跳过; +/// 两个选中的文件夹互相嵌套时同一个文件只收一次 +#[tokio::test] +async fn a_walk_descends_skips_what_cannot_be_read_and_counts_each_file_once() -> anyhow::Result<()> +{ + let server = MockServer::start().await; + mount_folder(&server, "top", false).await; + mount_folder(&server, "sub", false).await; + mount_children( + &server, + "top", + json!([ + {"id": "sub", "name": "Sub", "mimeType": FOLDER_MIME}, + {"id": "doc1", "name": "Plan", "mimeType": "application/vnd.google-apps.document", + "modifiedTime": "2026-10-01T09:30:00.000Z"}, + {"id": "form1", "name": "Survey", "mimeType": "application/vnd.google-apps.form"}, + {"id": "big", "name": "video.mp4", "mimeType": "video/mp4", "size": "999999999"} + ]), + ) + .await; + mount_children( + &server, + "sub", + json!([ + {"id": "pdf1", "name": "contract.pdf", "mimeType": "application/pdf", + "size": "1024", "md5Checksum": "abc", "modifiedTime": "2026-09-01T00:00:00Z"} + ]), + ) + .await; + + // `sub` 也被单独选中:它的文件不该出现两次 + let listing = drive(&server) + .walk(&["top".to_string(), "sub".to_string()]) + .await?; + let ids: Vec<&str> = listing.files.iter().map(|f| f.id.as_str()).collect(); + assert_eq!(ids, ["doc1", "pdf1"]); + assert_eq!(listing.skipped, 2, "表单与超大文件"); + assert!(!listing.truncated); + assert!(listing.unreachable.is_empty()); + + let doc = &listing.files[0]; + assert_eq!(doc.external_key(), "gdrive://doc1"); + assert_eq!( + doc.marker(), + "2026-10-01T09:30:00.000Z", + "Google 格式没有 md5,用修改时刻" + ); + assert_eq!( + doc.modified.map(|d| d.to_rfc3339()), + Some("2026-10-01T09:30:00+00:00".into()) + ); + assert_eq!(listing.files[1].marker(), "abc", "二进制文件用 md5"); + Ok(()) +} + +/// Drive 回来的 id 要拼进查询与地址:不合字符表的文件夹不进、文件不收 +#[tokio::test] +async fn ids_from_drive_outside_the_id_alphabet_are_skipped() -> anyhow::Result<()> { + let server = MockServer::start().await; + mount_folder(&server, "top", false).await; + mount_children( + &server, + "top", + json!([ + {"id": "x' or name contains '", "name": "Sneaky", "mimeType": FOLDER_MIME}, + {"id": "../../about", "name": "notes.pdf", "mimeType": "application/pdf", + "size": "10", "md5Checksum": "a"}, + {"id": "ok1", "name": "fine.pdf", "mimeType": "application/pdf", + "size": "10", "md5Checksum": "b"} + ]), + ) + .await; + + let listing = drive(&server).walk(&["top".to_string()]).await?; + let ids: Vec<&str> = listing.files.iter().map(|f| f.id.as_str()).collect(); + assert_eq!((ids, listing.skipped), (vec!["ok1"], 2)); + Ok(()) +} + +/// 别人组织的会议:文件夹里只有快捷方式,纪要住在组织者的 Drive 里。 +/// 指向文件的按目标收(目标自己的标记与时间),指向文件夹的往里走; +/// 同一个目标既被直接选到、又经快捷方式到达,只收一次 +#[tokio::test] +async fn a_shortcut_brings_in_what_it_points_to() -> anyhow::Result<()> { + let server = MockServer::start().await; + mount_folder(&server, "meeting", false).await; + mount_folder(&server, "theirs", false).await; + mount_children( + &server, + "meeting", + json!([ + {"id": "s1", "name": "Notes by Gemini", "mimeType": SHORTCUT_MIME, + "shortcutDetails": {"targetId": "notes1", + "targetMimeType": "application/vnd.google-apps.document"}}, + {"id": "s2", "name": "Recordings", "mimeType": SHORTCUT_MIME, + "shortcutDetails": {"targetId": "theirs", "targetMimeType": FOLDER_MIME}} + ]), + ) + .await; + mount_children( + &server, + "theirs", + json!([ + {"id": "notes1", "name": "Notes by Gemini", "mimeType": "application/vnd.google-apps.document", + "modifiedTime": "2026-10-20T10:00:00.000Z"}, + {"id": "t1", "name": "Transcript", "mimeType": "application/vnd.google-apps.document", + "modifiedTime": "2026-10-20T10:05:00.000Z"} + ]), + ) + .await; + Mock::given(method("GET")) + .and(path("/files/notes1")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "notes1", "name": "Notes by Gemini", + "mimeType": "application/vnd.google-apps.document", + "modifiedTime": "2026-10-20T10:00:00.000Z", "trashed": false + }))) + .mount(&server) + .await; + + let listing = drive(&server).walk(&["meeting".to_string()]).await?; + let files: Vec<(&str, String)> = listing + .files + .iter() + .map(|f| (f.id.as_str(), f.marker())) + .collect(); + assert_eq!( + files, + [ + ("notes1", "2026-10-20T10:00:00.000Z".to_string()), + ("t1", "2026-10-20T10:05:00.000Z".to_string()) + ] + ); + Ok(()) +} + +/// 快捷方式指向的文件没共享给这个人:点名,不当作「没东西」 +#[tokio::test] +async fn a_shortcut_to_a_file_the_person_cannot_open_is_named() -> anyhow::Result<()> { + let server = MockServer::start().await; + mount_folder(&server, "meeting", false).await; + mount_children( + &server, + "meeting", + json!([ + {"id": "s1", "name": "Transcript", "mimeType": SHORTCUT_MIME, + "shortcutDetails": {"targetId": "private1", + "targetMimeType": "application/vnd.google-apps.document"}} + ]), + ) + .await; + Mock::given(method("GET")) + .and(path("/files/private1")) + .respond_with(ResponseTemplate::new(404).set_body_json(json!({ + "error": {"message": "File not found: private1."} + }))) + .mount(&server) + .await; + + let listing = drive(&server).walk(&["meeting".to_string()]).await?; + assert_eq!( + (listing.files.len(), listing.broken_shortcuts), + (0, vec!["Transcript".to_string()]) + ); + Ok(()) +} + +/// 选中的文件夹打不开(删了、或不再共享)不是错误,但要点名——同步据此不对账 +#[tokio::test] +async fn an_unreachable_folder_is_named_not_fatal() -> anyhow::Result<()> { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/files/gone")) + .respond_with(ResponseTemplate::new(404).set_body_json(json!({ + "error": {"code": 404, "message": "File not found: gone.", + "errors": [{"reason": "notFound"}]} + }))) + .mount(&server) + .await; + mount_folder(&server, "here", false).await; + mount_children(&server, "here", json!([])).await; + + let listing = drive(&server) + .walk(&["gone".to_string(), "here".to_string()]) + .await?; + assert_eq!(listing.unreachable, ["gone"]); + Ok(()) +} + +/// 进了回收站的文件夹不进去,也不算打不开:里面的东西对这个人已经不算数了 +#[tokio::test] +async fn a_trashed_folder_is_not_walked() -> anyhow::Result<()> { + let server = MockServer::start().await; + mount_folder(&server, "binned", true).await; + let listing = drive(&server).walk(&["binned".to_string()]).await?; + assert!(listing.files.is_empty()); + assert!(listing.unreachable.is_empty()); + // 没有任何 files.list 请求 + let lists = server + .received_requests() + .await + .unwrap() + .iter() + .filter(|r| r.url.path() == "/files") + .count(); + assert_eq!(lists, 0); + Ok(()) +} + +/// 列表翻页:`nextPageToken` 一直跟到没有为止 +#[tokio::test] +async fn a_listing_follows_page_tokens() -> anyhow::Result<()> { + let server = MockServer::start().await; + mount_folder(&server, "top", false).await; + let file = |id: &str| { + json!({"id": id, "name": format!("{id}.txt"), "mimeType": "text/plain", + "size": "3", "md5Checksum": id}) + }; + Mock::given(method("GET")) + .and(path("/files")) + .and(query_param("pageToken", "p2")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"files": [file("b")]}))) + .with_priority(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/files")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({"files": [file("a")], "nextPageToken": "p2"})), + ) + .with_priority(2) + .mount(&server) + .await; + + let listing = drive(&server).walk(&["top".to_string()]).await?; + let ids: Vec<&str> = listing.files.iter().map(|f| f.id.as_str()).collect(); + assert_eq!(ids, ["a", "b"]); + Ok(()) +} + +/// Google 文档走 export 换成 DOCX,文件名补扩展名;别的原样下载,不猜 mime +#[tokio::test] +async fn google_formats_are_exported_and_files_are_downloaded() -> anyhow::Result<()> { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/files/doc1/export")) + .and(query_param( + "mimeType", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + )) + .respond_with(ResponseTemplate::new(200).set_body_bytes(b"PK-docx".to_vec())) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/files/pdf1")) + .and(query_param("alt", "media")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(b"%PDF-1.7".to_vec())) + .mount(&server) + .await; + + let d = drive(&server); + let doc = DriveFile { + id: "doc1".into(), + name: "Plan".into(), + mime: "application/vnd.google-apps.document".into(), + modified: None, + md5: None, + modified_raw: None, + }; + let c = d.content(&doc).await?; + assert_eq!(c.bytes, b"PK-docx"); + assert_eq!(c.filename, "Plan.docx"); + + let pdf = DriveFile { + id: "pdf1".into(), + name: "contract.pdf".into(), + mime: "application/pdf".into(), + modified: None, + md5: Some("x".into()), + modified_raw: None, + }; + let c = d.content(&pdf).await?; + assert_eq!(c.bytes, b"%PDF-1.7"); + assert_eq!(c.filename, "contract.pdf"); + assert_eq!(c.mime, "application/octet-stream"); + Ok(()) +} + +/// 导出超过 10 MB 时 Google 回 403 带 `exportSizeLimitExceeded`;错误里要留着这个原因, +/// 同步把它报给人看 +#[tokio::test] +async fn an_export_refusal_keeps_googles_reason() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/files/huge/export")) + .respond_with(ResponseTemplate::new(403).set_body_json(json!({ + "error": {"code": 403, "message": "This file is too large to be exported.", + "errors": [{"reason": "exportSizeLimitExceeded"}]} + }))) + .mount(&server) + .await; + let f = DriveFile { + id: "huge".into(), + name: "Budget".into(), + mime: "application/vnd.google-apps.spreadsheet".into(), + modified: None, + md5: None, + modified_raw: None, + }; + let Err(e) = drive(&server).content(&f).await else { + panic!("a 403 must not read as content"); + }; + let msg = format!("{e:#}"); + assert!(msg.contains("exportSizeLimitExceeded"), "{msg}"); + assert!(msg.contains("too large"), "{msg}"); +} + +/// 文件夹浏览器:「与我共享」用 sharedWithMe 查询,别的按父文件夹查,只要文件夹 +#[tokio::test] +async fn the_folder_browser_lists_only_folders() -> anyhow::Result<()> { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/files")) + .and(query_param( + "q", + format!("sharedWithMe = true and mimeType = '{FOLDER_MIME}' and trashed = false"), + )) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "files": [{"id": "s1", "name": "Client X"}] + }))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/files")) + .and(query_param( + "q", + format!("'root' in parents and mimeType = '{FOLDER_MIME}' and trashed = false"), + )) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "files": [{"id": "f1", "name": "Projects"}] + }))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/drives")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "drives": [{"id": "0AAdrive", "name": "Team"}] + }))) + .mount(&server) + .await; + + let d = drive(&server); + assert_eq!( + d.child_folders(SHARED_WITH_ME).await?, + [FolderEntry { + id: "s1".into(), + name: "Client X".into() + }] + ); + assert_eq!( + d.child_folders("root").await?, + [FolderEntry { + id: "f1".into(), + name: "Projects".into() + }] + ); + let roots: Vec = d.roots().await?.into_iter().map(|r| r.id).collect(); + assert_eq!(roots, ["root", SHARED_WITH_ME, "0AAdrive"]); + // 拼进 q 的 id 不收引号 + assert!(d.child_folders("x' or '1'='1").await.is_err()); + Ok(()) +} + +/// 换令牌:要刷新令牌、要 Drive 那一项权限;邮箱从 ID token 里拿 +#[tokio::test] +async fn a_code_exchange_needs_a_refresh_token_and_the_drive_scope() -> anyhow::Result<()> { + let token_ok = json!({ + "access_token": "ya29.a", "refresh_token": "1//r", "expires_in": 3599, + "scope": format!("openid {DRIVE_SCOPE} https://www.googleapis.com/auth/userinfo.email"), + "id_token": id_token(json!({ + "iss": "https://accounts.google.com", "aud": cfg().client_id, + "sub": "1", "email": "bruno@example.com", "email_verified": true + })) + }); + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/token")) + .and(body_string_contains("code=good")) + .and(body_string_contains("code_verifier=v")) + .respond_with(ResponseTemplate::new(200).set_body_json(token_ok.clone())) + .mount(&server) + .await; + let mut no_drive = token_ok.clone(); + no_drive["scope"] = json!("openid email"); + Mock::given(method("POST")) + .and(path("/token")) + .and(body_string_contains("code=nodrive")) + .respond_with(ResponseTemplate::new(200).set_body_json(no_drive)) + .mount(&server) + .await; + let mut no_refresh = token_ok.clone(); + no_refresh.as_object_mut().unwrap().remove("refresh_token"); + Mock::given(method("POST")) + .and(path("/token")) + .and(body_string_contains("code=norefresh")) + .respond_with(ResponseTemplate::new(200).set_body_json(no_refresh)) + .mount(&server) + .await; + + let http = reqwest::Client::builder().no_proxy().build()?; + let url = format!("{}/token", server.uri()); + let grant = exchange_code_at(&http, &url, &cfg(), "good", "v").await?; + assert_eq!(grant.refresh_token, "1//r"); + assert_eq!(grant.account_email.as_deref(), Some("bruno@example.com")); + + let e = exchange_code_at(&http, &url, &cfg(), "nodrive", "v") + .await + .unwrap_err(); + assert!(e.to_string().contains("read access to Drive"), "{e}"); + let e = exchange_code_at(&http, &url, &cfg(), "norefresh", "v") + .await + .unwrap_err(); + assert!(e.to_string().contains("no refresh token"), "{e}"); + Ok(()) +} + +/// 撤了授权或刷新令牌过期:说「去重连」,而不是一句 400 +#[tokio::test] +async fn a_revoked_grant_says_reconnect() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/token")) + .respond_with(ResponseTemplate::new(400).set_body_json(json!({ + "error": "invalid_grant", "error_description": "Token has been expired or revoked." + }))) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let e = access_token_at(&http, &format!("{}/token", server.uri()), &cfg(), "1//old") + .await + .unwrap_err(); + assert!(e.to_string().contains("reconnect"), "{e}"); +} + +/// 一轮同步超过一小时,访问令牌就过期了:401 换一枚新的再试,不把文件报成「读不出来」 +#[tokio::test] +async fn an_expired_access_token_is_renewed_once_mid_sync() -> anyhow::Result<()> { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/files/f1")) + .and(header("authorization", "Bearer ya29.expired")) + .respond_with(ResponseTemplate::new(401)) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/token")) + .and(body_string_contains("refresh_token=1%2F%2Fgrant")) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({"access_token": "ya29.fresh"})), + ) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/files/f1")) + .and(header("authorization", "Bearer ya29.fresh")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(b"%PDF-1.4 body".to_vec())) + .mount(&server) + .await; + let drive = Drive { + http: reqwest::Client::builder().no_proxy().build()?, + token: std::sync::Mutex::new("ya29.expired".into()), + renewal: Some(Renewal { + cfg: cfg(), + refresh_token: "1//grant".into(), + token_url: format!("{}/token", server.uri()), + }), + api_root: server.uri(), + }; + let first = drive + .get("download", &format!("{}/files/f1", server.uri()), &[]) + .await?; + let second = drive + .get("download", &format!("{}/files/f1", server.uri()), &[]) + .await?; + assert_eq!( + (first.bytes().await?.as_ref(), second.status()), + (b"%PDF-1.4 body".as_ref(), StatusCode::OK) + ); + Ok(()) +} + +/// ID token:签给别的 client 的、别的 issuer 的不收;没验证过的邮箱不显示 +#[test] +fn an_id_token_must_be_googles_and_ours() { + let ours = cfg().client_id; + let ok = id_token(json!({"iss": "accounts.google.com", "aud": ours, + "email": "a@b.c", "email_verified": true})); + assert_eq!( + id_token_email(&ok, &ours).unwrap().as_deref(), + Some("a@b.c") + ); + + let unverified = id_token(json!({"iss": "https://accounts.google.com", "aud": ours, + "email": "a@b.c", "email_verified": false})); + assert_eq!(id_token_email(&unverified, &ours).unwrap(), None); + + let other_client = id_token( + json!({"iss": "https://accounts.google.com", "aud": "someone-else", + "email": "a@b.c", "email_verified": true}), + ); + assert!(id_token_email(&other_client, &ours).is_err()); + + let other_issuer = id_token(json!({"iss": "https://evil.example", "aud": ours, + "email": "a@b.c", "email_verified": true})); + assert!(id_token_email(&other_issuer, &ours).is_err()); + + let expired = id_token(json!({"iss": "accounts.google.com", "aud": ours, + "email": "a@b.c", "email_verified": true, + "exp": Utc::now().timestamp() - 60})); + assert!(id_token_email(&expired, &ours).is_err()); +} + +/// 配置里的文件夹:对象或字符串都认,id 不合字符表的丢掉 +#[test] +fn selected_folders_keep_only_valid_ids() { + let cfg = json!({"folders": [ + {"id": "1AbC-_x", "name": "Clients"}, "root", {"id": "bad'id"}, {"name": "no id"} + ]}); + assert_eq!(selected_folders(&cfg), ["1AbC-_x", "root"]); + assert!(selected_folders(&json!({})).is_empty()); +} + +/// 真连一个 Google Drive。给了令牌才跑,CI 上永远跳过。 +/// +/// ```text +/// # 在 Google Cloud 建一个 OAuth 客户端(Web 应用),在 Utopia 里连接一个 google_drive 来源, +/// # 然后用那个客户端和一枚刷新令牌: +/// UTOPIA_GOOGLE_DRIVE_CLIENT_ID=… UTOPIA_GOOGLE_DRIVE_CLIENT_SECRET=… \ +/// UTOPIA_GOOGLE_DRIVE_REDIRECT_URI=… UTOPIA_GOOGLE_DRIVE_TEST_REFRESH_TOKEN=1//… \ +/// UTOPIA_GOOGLE_DRIVE_TEST_FOLDER= cargo test -p utopia-server google_drive +/// ``` +#[tokio::test] +async fn it_reads_from_a_real_drive() -> anyhow::Result<()> { + let (Some(cfg), Ok(refresh), Ok(folder)) = ( + oauth_config(), + std::env::var("UTOPIA_GOOGLE_DRIVE_TEST_REFRESH_TOKEN"), + std::env::var("UTOPIA_GOOGLE_DRIVE_TEST_FOLDER"), + ) else { + eprintln!("跳过:未设 UTOPIA_GOOGLE_DRIVE_TEST_REFRESH_TOKEN / _FOLDER"); + return Ok(()); + }; + let d = Drive::connect(&cfg, &refresh).await?; + assert!(!d.roots().await?.is_empty()); + let listing = d.walk(&[folder]).await?; + assert!( + !listing.files.is_empty(), + "那个文件夹里一个能读的文件都没有" + ); + let first = &listing.files[0]; + assert!(first.modified.is_some(), "modifiedTime 是 doc_time 的来源"); + let c = d.content(first).await?; + assert!(!c.bytes.is_empty()); + Ok(()) +} diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rs index 114c6d635..a45359802 100644 --- a/crates/utopia-server/src/ingest_sources.rs +++ b/crates/utopia-server/src/ingest_sources.rs @@ -7,8 +7,9 @@ use crate::state::AppState; use chrono::{DateTime, Utc}; use sha2::{Digest, Sha256}; -use utopia_core::models::Source; use utopia_core::models::SourceKind; +use utopia_core::models::{Role, Source}; +use utopia_core::AppError; use uuid::Uuid; #[cfg(test)] @@ -91,6 +92,7 @@ pub async fn sync_source(state: &AppState, source_id: Uuid) -> anyhow::Result<() } Some(SourceKind::Webdav) => sync_webdav(state, &source).await, Some(SourceKind::Notion) => sync_notion(state, &source).await, + Some(SourceKind::GoogleDrive) => sync_google_drive(state, &source).await, // 被动容器:folder / api / memory / upload 没有拉取语义 Some( SourceKind::Folder @@ -1523,6 +1525,172 @@ async fn sync_notion(state: &AppState, source: &Source) -> anyhow::Result anyhow::Result { + use crate::google_drive; + + let refresh = source.config["refresh_token"] + .as_str() + .map(str::trim) + .filter(|s| !s.is_empty()) + .ok_or_else(|| { + anyhow::anyhow!( + "google_drive source is not connected: connect a Google account in its settings" + ) + })?; + // 读的是连上的那个人的 Drive:他离开了这个库、被降成只读、或账号停用了, + // 这份授权就不再替库读东西,等库里的人重连 + let connector = match google_drive::connected_by(&source.config) { + Some(id) => utopia_store::accounts::find_user_by_id(&state.pool, id).await?, + None => None, + }; + let access = match connector { + Some(user) => { + utopia_store::access::require_kb(&state.pool, &user, source.kb_id, Role::Editor).await + } + None => Err(AppError::NotFound), + }; + let still_editor = match access { + Ok(_) => true, + Err(AppError::NotFound | AppError::Forbidden) => false, + Err(e) => return Err(e.into()), + }; + anyhow::ensure!( + still_editor, + "the person who connected this Google Drive is no longer an editor of this knowledge base: reconnect it in the source settings" + ); + let cfg = google_drive::oauth_config().ok_or_else(|| { + anyhow::anyhow!( + "Google Drive is not configured on this server (UTOPIA_GOOGLE_DRIVE_CLIENT_ID)" + ) + })?; + let folders = google_drive::selected_folders(&source.config); + if folders.is_empty() { + anyhow::bail!("google_drive source has no folders selected"); + } + + let drive = google_drive::Drive::connect(&cfg, refresh).await?; + let listing = drive.walk(&folders).await?; + + let known = google_drive::markers(&state.pool, source.id).await?; + let mut stats = SyncStats::default(); + let mut seen_keys = Vec::with_capacity(listing.files.len()); + let mut seen_ids = Vec::with_capacity(listing.files.len()); + let mut unread: Vec = Vec::new(); + let mut downloads = 0usize; + let mut deferred = 0usize; + for f in &listing.files { + let key = f.external_key(); + seen_keys.push(key.clone()); + seen_ids.push(f.id.clone()); + let marker = f.marker(); + if known.get(&f.id) == Some(&marker) { + let live = utopia_store::documents::find_by_external_key(&state.pool, source.id, &key) + .await? + .is_some_and(|d| d.deleted_at.is_none()); + if live { + continue; + } + } + if downloads == google_drive::MAX_DOWNLOADS_PER_SYNC { + deferred += 1; + continue; + } + downloads += 1; + // 读不出来的文件跳过,库里已有的原样留着,其余照常同步——同 Notion + let content = match drive.content(f).await { + Ok(c) => c, + Err(error) => { + tracing::warn!(id = %f.id, error = %format!("{error:#}"), "google drive file could not be read, leaving the stored document as it is"); + unread.push(format!("{}: {error:#}", f.name)); + continue; + } + }; + let action = ingest_item( + state, + source.kb_id, + source.id, + &key, + &content.filename, + content.mime, + &content.bytes, + f.modified, + ) + .await?; + google_drive::remember(&state.pool, source.id, &f.id, &marker).await?; + stats.absorb(action); + } + + if listing.truncated || !listing.unreachable.is_empty() { + utopia_store::documents::clear_missing_keys(&state.pool, source.id, &seen_keys).await?; + } else { + utopia_store::documents::reconcile_missing(&state.pool, source.id, &seen_keys).await?; + google_drive::forget_others(&state.pool, source.id, &seen_ids).await?; + } + + let mut gaps: Vec = Vec::new(); + if !listing.unreachable.is_empty() { + gaps.push(format!( + "{} selected folder(s) could not be opened (deleted, or no longer shared with the connected account): {}", + listing.unreachable.len(), + listing.unreachable.join(", ") + )); + } + if !unread.is_empty() { + gaps.push(format!( + "{} could not be read and keep their stored content: {}", + unread.len(), + unread + .iter() + .take(3) + .map(String::as_str) + .collect::>() + .join("; ") + )); + } + if !listing.broken_shortcuts.is_empty() { + gaps.push(format!( + "{} shortcut(s) point to files the connected account cannot open: {}", + listing.broken_shortcuts.len(), + listing + .broken_shortcuts + .iter() + .take(3) + .map(String::as_str) + .collect::>() + .join("; ") + )); + } + if listing.truncated { + gaps.push( + "the folders go deeper or hold more files than one walk lists, so the rest were not looked at" + .into(), + ); + } + if deferred > 0 { + gaps.push(format!( + "{deferred} more file(s) wait for the next sync (at most {} downloads per run)", + google_drive::MAX_DOWNLOADS_PER_SYNC + )); + } + anyhow::ensure!( + gaps.is_empty(), + "{} file(s) synced; {}", + stats.total(), + gaps.join("; ") + ); + Ok(stats) +} + /// 标题无扩展名时按 mime 补一个,接入解析矩阵的分派。 fn ensure_extension(title: &str, mime: &str) -> String { let has_ext = title diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rs index 83d113057..df3141134 100644 --- a/crates/utopia-server/src/main.rs +++ b/crates/utopia-server/src/main.rs @@ -12,6 +12,7 @@ mod extraction_open; mod github_issues; mod github_repo; mod gitlab_repo; +mod google_drive; mod governance; mod http_fetch; mod implication; diff --git a/crates/utopia-store/src/sources.rs b/crates/utopia-store/src/sources.rs index a669d0151..6eb3d01c0 100644 --- a/crates/utopia-store/src/sources.rs +++ b/crates/utopia-store/src/sources.rs @@ -310,6 +310,30 @@ pub async fn update( opened(source) } +/// 只动配置里的几个键:`set` 里的写进去(密钥照样封印),`remove` 里的删掉,别的键原样。 +/// 一条语句完成,不先读后写——两个请求同时改不同的键时,谁也不会把对方刚写的盖掉 +pub async fn patch_config( + pool: &PgPool, + id: Uuid, + set: &serde_json::Value, + remove: &[&str], +) -> AppResult { + let mut set = set.clone(); + secrets::seal_json_keys(&mut set, SOURCE_SECRET_KEYS); + let source = sqlx::query_as( + "UPDATE sources SET config = + (CASE WHEN jsonb_typeof(config) = 'object' THEN config ELSE '{}'::jsonb END - $3::text[]) || $2 + WHERE id = $1 RETURNING *", + ) + .bind(id) + .bind(set) + .bind(remove) + .fetch_optional(pool) + .await? + .ok_or(AppError::NotFound)?; + opened(source) +} + /// 删除来源;其文档保留(source_id 置 NULL,落回 Uploads 组)。 pub async fn delete(pool: &PgPool, id: Uuid) -> AppResult<()> { let res = sqlx::query("DELETE FROM sources WHERE id = $1") diff --git a/crates/utopia-store/tests/store/a_secret_is_sealed_at_rest.rs b/crates/utopia-store/tests/store/a_secret_is_sealed_at_rest.rs index 279e8437d..12ff065f5 100644 --- a/crates/utopia-store/tests/store/a_secret_is_sealed_at_rest.rs +++ b/crates/utopia-store/tests/store/a_secret_is_sealed_at_rest.rs @@ -173,6 +173,32 @@ async fn a_secret_is_sealed_at_rest() -> anyhow::Result<()> { sources::get(&pool, src.id).await?.config["auth_header"], "Bearer t0k" ); + // 只改几个键的那条路(Google Drive 的连接)照样封印,别的键原样留着 + let patched = sources::patch_config( + &pool, + src.id, + &serde_json::json!({ "refresh_token": "1//grant" }), + &[], + ) + .await?; + let stored = raw( + &pool, + "SELECT config->>'refresh_token' FROM sources WHERE id = $1", + src.id, + ) + .await? + .unwrap(); + assert!(secrets::is_sealed(&stored)); + assert_eq!( + ( + &patched.config["refresh_token"], + &patched.config["auth_header"] + ), + ( + &serde_json::json!("1//grant"), + &serde_json::json!("Bearer t0k") + ) + ); sources::set_ingest_token(&pool, src.id, "utp_push_1").await?; let stored = raw( &pool, diff --git a/docs/decisions/0074-a-google-drive-source-is-connected-by-a-person.md b/docs/decisions/0074-a-google-drive-source-is-connected-by-a-person.md new file mode 100644 index 000000000..0ae245121 --- /dev/null +++ b/docs/decisions/0074-a-google-drive-source-is-connected-by-a-person.md @@ -0,0 +1,48 @@ +# 0074 · A Google Drive source is connected by a person + +- **Status**: Implemented · 2026-10-23 · migration 0120 · tried by hand against a real Drive; the test fixtures are written from the documentation, not recorded (0013 decision 4) · open: revision history, change-feed sync +- **Written**: 2026-10-14, revised 2026-10-23 after two reviews: the connecting person owns the selection and the connection, the cap is on downloads, a revoke spares a grant still in use (conventions in the [README](README.md)) +- **Related**: [0013](0013-a-source-should-hand-over-its-history.md) (what a source must pass, fixtures from real responses); [0014](0014-identity-from-the-person-scope-from-the-token.md) (identity comes from the person); migration `0056` (`oidc_flows`, the shape the connection flow copies); [#246](https://github.com/deeplethe/utopia/issues/246) (a credential never reaches a viewer) + +## Problem + +Most teams keep their documents in Google Drive. Today the way in is to export them by hand, or to copy them into a bucket and sync that, which loses each file's identity and its modification time. Drive passes the tests of 0013: `modifiedTime` is the content's own clock, documents are rewritten in place, a file id survives renames and moves, and the knowledge actually lives there. + +Every existing connector is configured with a key an administrator pastes in. Drive has no such key for a person's files. A service account sees only what is shared with it, so every folder would first have to be shared with a robot address, and domain-wide delegation lets one key read everyone's Drive, far more than anyone chose to include. The natural grant is the one a person gives: "read these folders of mine, as me". + +## Decisions + +1. **A person connects the source with OAuth, and it reads Drive as them.** An editor of the knowledge base opens the source's settings and clicks Connect. Google's consent screen asks for `drive.readonly` plus `openid email`, and the refresh token is stored in the source's config under `refresh_token`, which is in `SOURCE_SECRET_KEYS`: sealed at rest, never in a listing, never in a response (#246). The browser never holds a Google token. Even the folder browser asks the server, which reads Drive with the stored grant. `drive.file` was not enough: it covers only files the app created or the person picked one by one, and picking a folder does not include what is inside it. + +2. **The OAuth client belongs to the deployment.** `UTOPIA_GOOGLE_DRIVE_CLIENT_ID`, `_CLIENT_SECRET` and `_REDIRECT_URI` name a Google "web application" client the operator registers. When any of the three is missing, `GET /google-drive/status` says so, creating the kind is refused (`google_drive_not_configured`), and a sync fails with that sentence rather than a 401. This client is separate from the SSO client of 0056. The two ask for different scopes, and one deployment may sign people in elsewhere and still read Drive. + +3. **The flow is bound to the person who started it.** `POST …/google-drive/connect` records a single-use `state`, a PKCE verifier, the user and the source in `google_drive_flows` (ten minutes, at most 5 pending per person and 1000 in all, both checked in the insert itself), and returns Google's URL. The callback deletes the row before using it, requires the browser's session to be that same user, and checks again that the user is still an editor. Without the session check, someone could start a flow for their own source and get another person to click through consent, and the other person's Drive would be connected to their source. `prompt=consent` and `access_type=offline` are always sent, so a reconnect after revocation gets a new refresh token. The access token a sync works with lasts an hour and a run can last longer, so a 401 mid-run gets a new access token from the refresh token and the request is tried once more. + +4. **The ID token names the account, and that is all it does.** `openid email` exists so the settings can say "connected as bruno@…". The ID token comes straight from Google's token endpoint over TLS, so issuer, audience and expiry are checked and the signature is not (OIDC Core 3.1.3.7). An unverified email is not shown. The email never identifies a Utopia account and never links one; that stays as 0056 decided. `connected_at` in the config is what the interface reads as "connected", because the email may legitimately be absent. + +5. **The person who connected chooses the folders, and subfolders come with them.** The callback records `connected_by`. Only that person may browse (`GET …/google-drive/folders`) and save a selection (`PUT …/google-drive/folders`); another editor gets 403. Everyone in the base can read what is synced, so a colleague who could browse would be choosing what leaves someone else's Drive. For the same reason a Drive source's config is written only by its connect, select and disconnect routes: the general source create and update leave it alone. Only the person who connected may reconnect: anyone else gets 403 from `connect`, and a callback that finds someone else connected in the meantime is refused. Otherwise one click by a colleague would replace that person's Drive with their own. To change who reads, an editor disconnects first, which removes the selection with the grant (decision 11), and then connects. When the person who connected reconnects with a different Google account, the selection made in the old account is dropped and the old grant is revoked as in decision 11. A reconnect with the same account never revokes, because revoking ends the whole grant and would take the new token with it. Accounts compare by email ignoring case; when either email is unknown the account counts as changed, so the folders are dropped, but nothing is revoked. `config.folders` is `[{id, name}]`. The name is only for the interface. The browser lists My Drive, Shared with me and every shared drive. Saving the selection queues a sync at once. Ids are checked against `[A-Za-z0-9_-]` before they go into a Drive query or URL, which is simpler to trust than escaping. That holds for the ids Drive itself returns too: a child folder, file or shortcut target outside that alphabet is skipped. Each of these routes changes only its own keys in one statement (`patch_config`), so a selection saved during a callback cannot drop the token the callback just wrote. + +6. **The grant reads only while its person is an editor.** A sync first checks that `connected_by` is still an active editor of the base. When they have left, been made a viewer or been deactivated, the run fails with "reconnect it in the source settings" and reads nothing. Their Google grant would otherwise keep pulling their files into a base they no longer belong to. + +7. **The cap is on downloads, not on the listing.** A run downloads or exports at most 2000 files, the same order as WebDAV and object storage. The listing is metadata only and unchanged files cost nothing (decision 9), so it is bounded only by a guard of 50,000 files, 16 levels and 32 MB per file. A run that hits the download cap fails with how many files wait, and the next run starts where it stopped, because the files it already took now match their markers. A listing cut at the same place every time would never reach the tail (the gap in #1212). Every limit that is hit is reported, and none of them is reported as success. + +8. **Google formats are exported to Office formats.** Docs become DOCX, Sheets XLSX and Slides PPTX, and the existing parsers read them with headings, tables and every sheet. Plain text or CSV would have lost the headings that 0039's chunks and 0064's section dates depend on, and CSV keeps only the first sheet. Forms, drawings and sites have no readable body and are skipped. A shortcut is followed to what it points at: a folder is walked, and a file is taken with its own metadata, marker and id, so a file reached directly and through a shortcut is one document. This matters for meetings someone else organised, whose notes and transcripts live in the organiser's Drive and reach the person only as shortcuts. A shortcut to a file the connected account cannot open is named in the run's result, which is reported as failed, not as an empty success. Everything else is downloaded as bytes with `application/octet-stream`, because the parser trusts magic bytes over names, as object storage already does. + +9. **Change is detected from Drive's marker, not from the bytes.** An export of an unchanged Doc is a new zip with new timestamps, so a content hash changes on every run, and every run would re-extract every Doc. `google_drive_files` keeps one marker per file: `md5Checksum` for binary files, and `modifiedTime` for Google formats, which have no checksum. When the marker is unchanged and the document is live, the file is not downloaded at all. + +10. **Retraction follows a complete walk only.** Files not seen are marked "not in source" (`reconcile_missing`) only when every selected folder opened and the listing was complete. The download cap does not stop it, because the listing still saw every file. A folder that returns 404, because it was deleted or is no longer shared with the connected person, fails the run by name and marks nothing. Not being able to look is not the same as finding it empty. A file that cannot be read (Google refuses exports over 10 MB) keeps its stored document, the rest of the run continues, and the run is reported as failed with the names, as Notion does. + +11. **Disconnecting forgets the grant, and revokes it at Google when nothing else uses it.** `DELETE …/google-drive/connection` removes the token and the selection from the config first. Revoking a refresh token at Google ends the person's whole grant to our OAuth client, every token issued under it, so it would also stop their other Drive sources. The revoke is therefore sent only when no other Google Drive source holds a token for the same account (`account_email`). When the account is unknown because the email was unverified, nothing is revoked: the token is gone from Utopia, and the person can still remove the grant in their Google account. A failed revoke, by network or by a non-2xx status, is logged and does not undo the disconnect. Any editor may disconnect, since stopping a read never exposes anything. Deleting a source does not revoke: its token goes with the row, and the grant stays at Google until the person removes it there or disconnects another source of the same account. Connecting and disconnecting are audited with the account email and never the token. + +## Dead ends + +- **A service account with folders shared to it.** It is robust, and nobody's departure breaks it. But selecting a folder becomes "share it with a robot address and paste an id", and a shared drive must add the robot as a member. It may still be worth having for organisations that will not tie a sync to one person; it would be a second way to fill the same `refresh_token`-shaped slot, not a different source. +- **The Google Picker.** It needs a browser API key and an app id beside the OAuth client, puts a Google token in the browser, and with `drive.file` would still not grant a folder's contents. A server-side folder list costs one endpoint and keeps decision 1 intact. +- **Comparing content hashes**, as every other connector does. That costs a re-extraction of every Google Doc on every run (decision 9). +- **The changes feed** (`changes.list` with a saved page token). It is cheaper on large drives, but it reports changes across the whole Drive and not under a folder, so each change needs an ancestry walk, and moves out of a selected folder are the hard case. The full walk is simple and bounded. Revisit when a deployment hits the 50,000-file listing guard. + +## Open questions + +- **Revision history.** Drive keeps revisions, and 0013 prefers a source that hands its history over. Exporting old revisions of Google formats is possible but expensive; which revisions to sample is its own decision, as it is for Feishu and Confluence. +- **When the person leaves.** The grant is theirs, and a sync stops once they are no longer an editor (decision 6). The failed run raises the usual sync alert, but a dedicated "this source needs a new reader" alert would say more plainly what to do. +- **Verification by Google.** `drive.readonly` is a restricted scope. An "internal" OAuth client in a Workspace organisation needs no verification; an "external" one used beyond a hundred test users needs Google's security assessment. The operator decides that, but the setup guide should say so. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 8fc45e2d8..64b16381b 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -116,6 +116,7 @@ By domain; the domains are the files of [../design/](../design/README.md). **Sta | 0054 | [A source may push statements in the open contract](0054-a-source-may-push-statements-in-the-open-contract.md) | Implemented | | | 0065 | [The OCR reader chooses its provider](0065-readers-choose-their-provider.md) | Implemented | | | 0072 | [A source checkpoint ends at a timestamp boundary](0072-a-source-checkpoint-ends-at-a-timestamp-boundary.md) | Proposed | | +| 0074 | [A Google Drive source is connected by a person](0074-a-google-drive-source-is-connected-by-a-person.md) | Implemented | | ### [lakehouse-and-actions](../design/lakehouse-and-actions.md) diff --git a/docs/design/sources.md b/docs/design/sources.md index 0e6869d2f..8af6d16c9 100644 --- a/docs/design/sources.md +++ b/docs/design/sources.md @@ -1,6 +1,6 @@ # A source hands over its history -Records: [0013] (connectors), [0023] and [0033] (RSS), [0039] (chunks), [0040] (origin and media), +Records: [0013] (connectors), [0074] (Google Drive), [0023] and [0033] (RSS), [0039] (chunks), [0040] (origin and media), [0035] (vector index), [0036] d7 (schema documents), [0005] (sync alerts), [0022] and [0045] d3 (document dates), [0016] E. @@ -14,7 +14,7 @@ source, upload_time, none) is the document's own date; upload time is recorded t document date, so an upload-dated document counts as undated for attestation and ordering [0022, 0045, #714]. -**Connectors.** GitHub issues, Jira, Notion, RSS, WebDAV, S3 / Azure / GCS, and custom push with an +**Connectors.** GitHub issues, Jira, Notion, Google Drive, RSS, WebDAV, S3 / Azure / GCS, and custom push with an `ingest_token`; the kinds come from one `SourceKind` enum that the allowlist, the dispatcher and the web list derive from; credentials never leave the server (`SOURCE_SECRET_KEYS`) and a blank field keeps the stored value; a truncated round says so [0013]. A ticket is one document whose body @@ -26,6 +26,8 @@ changed is not fetched again, and identical files at several paths go in once [# runs past the page cap is a truncated round, and its entries carry no size, so the size cap is read from a `HEAD` on the file before anything is downloaded [#1327]. +**Google Drive.** A person connects it with OAuth and it reads Drive as them; the refresh token is a sealed source secret and the browser never holds a Google token; the flow is bound to the session that started it; only the person who connected picks folders, subfolders come along, and a sync stops once that person is no longer an editor; a run downloads at most 2000 files and says so when it stops there; Docs, Sheets and Slides are exported to DOCX, XLSX and PPTX; change is Drive's marker (`md5Checksum`, or `modifiedTime` for Google formats) because an export's bytes differ every time; "not in source" only after a complete walk [0074]. + **Statements.** A `statements` source takes the open extraction contract itself (`e`, `s`, `n`) on `POST /sources/{id}/statements`, with the `api` push's identity, versions, tombstones and run history; the payload is the document and its single chunk, and extraction parses the chunk instead of prompting a model, then runs the unchanged path, so a pushed statement is an open statement with the item as its evidence (null offsets) and reaches the typed graph only through alignment. The contract has no slot for a property or a class and unknown keys are refused; a second push under the same identity marks the earlier statements stale rather than closing them; tables belong on a mount, not here [0054]. **RSS.** Observations are not documents: one table with baseline, candidate and no_source rows; diff --git a/migrations/0120_a_google_drive_source_is_connected_by_a_person.sql b/migrations/0120_a_google_drive_source_is_connected_by_a_person.sql new file mode 100644 index 000000000..986a01318 --- /dev/null +++ b/migrations/0120_a_google_drive_source_is_connected_by_a_person.sql @@ -0,0 +1,25 @@ +-- Google Drive 来源(0074):由一个人授权连上,按他选的文件夹同步。 +-- +-- 一次连接尝试的临时状态:授权码换令牌之前,state 与 PKCE verifier 得先落着, +-- 还要记下是谁、替哪个来源发起的——回调时这个浏览器里的会话必须还是他。 +-- 短命、一次性:回调一到就删,半途弃单的过期扫走。与 oidc_flows 同一个形状。 +CREATE TABLE IF NOT EXISTS google_drive_flows ( + state TEXT PRIMARY KEY, + verifier TEXT NOT NULL, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + source_id UUID NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX IF NOT EXISTS google_drive_flows_expiry_idx ON google_drive_flows (expires_at); + +-- 每个文件上一次摄入时 Drive 给的变更标记:二进制文件是 md5Checksum,Google 文档 +-- 没有 md5,用 modifiedTime。**不能靠内容哈希判「没变」**:Docs/Sheets/Slides 每次 +-- 导出成 Office 格式,压缩包里的时间戳都不同,哈希每轮都变,每轮都会重新抽取。 +-- 标记没变、文档还在,就连下载都省了。 +CREATE TABLE IF NOT EXISTS google_drive_files ( + source_id UUID NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + file_id TEXT NOT NULL, + marker TEXT NOT NULL, + PRIMARY KEY (source_id, file_id) +); diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 709965c24..ff17a18dc 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -651,6 +651,7 @@ export const en = { gcs: "Google Cloud Storage", webdav: "WebDAV", notion: "Notion", + google_drive: "Google Drive", }, sourceKindHints: { folder: "Holds the files you upload: add, drag in and remove them here.", @@ -689,6 +690,10 @@ export const en = { "Syncs the pages a Notion integration can see — share a page with the integration " + "and it appears here. Dated by when the page was last edited, which is the page's " + "own clock rather than ours.", + google_drive: + "Syncs the folders you choose from your Google Drive, shared drives included. " + + "**You connect with your own Google account**, read-only; Docs, Sheets and Slides " + + "arrive as Word, Excel and PowerPoint files. Dated by each file's last modification.", api: "External systems push JSON documents here, authenticated with this source's own token.", statements: "Your system pushes statements already in the extraction contract; no model reads them. Send events, not the state of a table.", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index 6f6d0a3bb..98a486d27 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -590,6 +590,7 @@ export const zh: Strings = { gcs: "Google Cloud Storage", webdav: "WebDAV", notion: "Notion", + google_drive: "Google Drive", }, sourceKindHints: { folder: "可在文件夹中管理上传的文件:上传、拖入、删除都在这里。", @@ -624,6 +625,10 @@ export const zh: Strings = { notion: "同步一个 Notion 集成能看见的页面——把页面分享给集成,它就会出现在这里。" + "日期取页面最后一次编辑的时刻,那是页面自己的时钟,不是我们抓它的时刻。", + google_drive: + "同步你在 Google Drive 里选的文件夹,共享云端硬盘也算。" + + "**用你自己的 Google 账号连接**,只读;Docs、Sheets、Slides 以 Word、Excel、PowerPoint " + + "文件进来。日期取每个文件最后一次修改的时刻。", api: "外部系统把 JSON 文档推送到这里,用这个来源自己的令牌认证。", statements: "外部系统把已经是抽取契约形状的陈述推送到这里,不经模型;发事件,别发整张表的状态。", diff --git a/web/src/pages/SourcesRail.tsx b/web/src/pages/SourcesRail.tsx index 5e9e5fe98..1be67f7d8 100644 --- a/web/src/pages/SourcesRail.tsx +++ b/web/src/pages/SourcesRail.tsx @@ -80,6 +80,7 @@ export const KIND_ICON: Record = { gcs: Cloud, webdav: FolderOpen, notion: Notebook, + google_drive: HardDrive, memory: Brain, upload: Upload, }; @@ -98,6 +99,7 @@ export const SYNCING_KINDS = new Set([ "gcs", "webdav", "notion", + "google_drive", ]); export const SYNC_DOT: Record = { diff --git a/web/src/sourceKinds.ts b/web/src/sourceKinds.ts index 3eb713d67..3174e0792 100644 --- a/web/src/sourceKinds.ts +++ b/web/src/sourceKinds.ts @@ -21,6 +21,7 @@ export const CREATABLE_SOURCE_KINDS = [ "gcs", "webdav", "notion", + "google_drive", "api", "statements", "custom",