diff --git a/.github/workflows/gridfs-fix-validation.yml b/.github/workflows/gridfs-fix-validation.yml new file mode 100644 index 0000000..48b72b8 --- /dev/null +++ b/.github/workflows/gridfs-fix-validation.yml @@ -0,0 +1,48 @@ +name: GridFS Ownership Safety Validation + +on: + pull_request: + branches: [main, 'fix/gridfs-id-ownership-v01'] + +concurrency: + group: gridfs-safety-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + gridfs: + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + mongo-version: ['7.0.37', '8.0.26'] + steps: + - name: Checkout + uses: actions/checkout@v5 + - name: Setup Node 22 + uses: actions/setup-node@v6 + with: + node-version: '22.x' + cache: npm + - name: Install dependencies + run: npm ci + - name: Cache MongoDB binaries + uses: actions/cache@v5 + with: + path: .cache/mongodb-memory-server/binaries + key: mongodb-memory-server-${{ runner.os }}-${{ runner.arch }}-gridfs + - name: Build + run: npm run build && npm run build:tests + - name: Execute GridFS safety and compatibility cases + env: + MONSQLIZE_MEMORY_MONGO_BINARY_VERSION: ${{ matrix.mongo-version }} + run: | + node --test --test-concurrency=1 \ + .generated/test-dist/test/unit/gridfs/gridfs.test.js \ + .generated/test-dist/test/integration/gridfs/gridfs.test.js \ + .generated/test-dist/test/integration/gridfs/gridfs-scan-regressions.test.js \ + .generated/test-dist/test/integration/gridfs/gridfs-ownership-regressions.test.js \ + .generated/test-dist/test/integration/gridfs/gridfs-stale-recovery.test.js diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 83a6fe0..b421ae3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,11 +4,18 @@ on: push: branches: [main] pull_request: - branches: [main] + branches: [main, 'fix/gridfs-id-ownership-v01'] + +# Superseded PR revisions must not accumulate long-running test jobs. +# Cancelled prior SHAs are never accepted as evidence for the current HEAD. +concurrency: + group: ci-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test: runs-on: ubuntu-latest + timeout-minutes: 60 strategy: fail-fast: false matrix: @@ -50,6 +57,7 @@ jobs: release-gate: runs-on: ubuntu-latest + timeout-minutes: 60 steps: - name: Checkout uses: actions/checkout@v5 diff --git a/changelogs/unreleased.md b/changelogs/unreleased.md index b1713eb..c0a6a01 100644 --- a/changelogs/unreleased.md +++ b/changelogs/unreleased.md @@ -2,7 +2,7 @@ ## Changes after v3.5.3 -No changes have been staged after 3.5.3. The GridFS and Search index fixes from the 2026-11-02 weekly scan are assigned to the [3.5.3 versioned notes](./v3.5.3.md); the vector search guidance and Search docs regrouping are in [3.5.2](./v3.5.2.md). Publication is established by the npm registry and immutable tag, not this file. +Candidate work (not yet released): GridFS per-file ID ownership, durable append/finalize/abort state, deletion fencing, and expiresAt idle-TTL migration. These changes require a coordinated writer drain before upgrading and are pending full compatibility verification. No release version or publication is claimed. The GridFS and Search index fixes from the 2026-11-02 weekly scan are assigned to the [3.5.3 versioned notes](./v3.5.3.md); the vector search guidance and Search docs regrouping are in [3.5.2](./v3.5.2.md). Publication is established by the npm registry and immutable tag, not this file. ## Archived preparation notes for the 3.3.2 candidate diff --git a/config/package-budget.json b/config/package-budget.json index 84c957e..19f781d 100644 --- a/config/package-budget.json +++ b/config/package-budget.json @@ -1,8 +1,8 @@ { - "comment": "GridFS 3.5.0 stays in the main bundle: sync gridfs() cannot be split out of the single-file build. CJS 881096 and ESM 878096 cover driver-error mapping, write-path iid matching, and the 2026-11-02 scan fixes (GridFS cleanup/cursor/session repairs and segmented Search index wait timers, idempotent finalize retries), plus 4 KiB of headroom each approved on 2026-11-02.", + "comment": "The 3.5.3 bundle ceilings were 881096 CJS and 878096 ESM. The GridFS cross-operation ownership, durable session CAS/abort fencing, delete protection and idle-TTL migration add ~4.4/4.7 KiB over that ceiling (CI build 2026-11-12: 885536 CJS / 882819 ESM). Raise to 890000 / 887000, keeping ~4 KiB reviewed headroom; no dependency change. Enforce these ceilings in release preflight.", "artifacts": { - "dist/cjs/index.cjs": 881096, - "dist/esm/index.mjs": 878096, + "dist/cjs/index.cjs": 890000, + "dist/esm/index.mjs": 887000, "dist/cjs/cli/data-task.cjs": 50000 } } diff --git a/docs/en/gridfs-recovery.md b/docs/en/gridfs-recovery.md new file mode 100644 index 0000000..07915c8 --- /dev/null +++ b/docs/en/gridfs-recovery.md @@ -0,0 +1,71 @@ +# GridFS crash recovery and no-data-loss upgrade runbook (V0.1 candidate) + +> **Operator-only.** This is a draft recovery design, not a completed or verified release. Never run recovery against a running, paused or network-partitioned GridFS writer. **Age alone is not proof that a writer is dead.** No automated TTL-based owner unlock is permitted. + +## Scope and invariant + +The ownership registry is the per-bucket `.uploads` collection. A reservation excludes ordinary upload, uploadStream, resumable append/finalize/abort, delete, rename and orphan cleanup for the same ObjectId. A missing file document **does not** prove its chunks are orphaned: an ordinary upload may still be writing, or GridFS driver delete may have removed the files document before chunks. Recovering while an old writer can resume can destroy new data, even with a token-CAS. The maintenance process cannot prove cluster/process shutdown on its own. + +The script `scripts/maintenance/gridfs-recover.cjs` uses a default read-only preview, exact state/token/timestamp checks, a persisted `recovering` ownership claim, verified preservation of committed visible chunks, and an append-only-by-intent audit collection `.ownership_recovery_audit`. It never performs a time-based lock steal. + +## Deployment and rollback gates (required) + +1. Stop all HTTP/worker/job/cron processes that can mutate this GridFS bucket, across **all** deployments, driver versions, replica instances, queue consumers and old binaries. Disable automatic restarts and new GridFS requests. Resolve in-flight operations, check for network-partitioned/paused workers, and ensure the old writer fleet is fully drained. A mixed old/new-writer deployment is unsupported because the old version ignores the new registry. +2. Back up `.files`, `.chunks`, `.uploads`, current indexes and relevant deployment version/worker inventory. Record a timestamp, operator, target database/bucket/id, intended delete/finalize outcome, and why no writer can resume. Retain backups and audit logs outside the cluster. +3. Deploy the new version while writes remain disabled. Run `ensureIndexes()` after verifying the old `createdAt_1` TTL and any legacy pending sessions. The migration derives `expiresAt` from the last update for eligible idle sessions, removes the legacy TTL index, and creates `expiresAt_1` with `expireAfterSeconds: 0`. TTL can already have removed sessions before migration; examine orphan chunks separately. Never treat an expiring session as safe while a legacy writer may still run. +4. Inspect every retained owner that must be recovered. Reconcile visible file and chunks, determine its original operation and the desired post-recovery state, and record the evidence. An owner without expiry is not automatically stale. Use the script only for an owner that has been conclusively quiesced. +5. Run dry-run and compare the proposed action with the recorded intent. To apply, supply the exact observed `status`, `updatedAt` ISO timestamp, and `ownerToken`, `activeAppend.token`, or `recovery.token` when present. Any mismatch is a stop signal. An audit document is written before a change; if a step fails the state remains `recovering` and must be re-inspected. +6. Recheck file visibility, stored chunk count, lengths, and any relevant SHA. Complete the latest-HEAD Node 18/20/22 CI, MongoDB 7.0.37/8.0.26 targeted suite, applicable Driver 6/7 compatibility checks and release preflight before unfreezing. Verify no `recovering` or unintended active reservations remain. +7. Only then restart new writers. Rollback to an old binary is **not safe while a new-version writer or reservation is active**; first stop/drain all new writers, reconcile operations, and assess the old binary's inability to honor `.uploads` ownership. Preserve the migrated index configuration; do not recreate `createdAt` TTL blindly. + +## Manual command + +Use a checkout with installed dependencies and a MongoDB driver. Never pass the connection string in CLI arguments or paste credentials into an issue. + +```sh +export MONSQLIZE_RECOVERY_MONGODB_URI='' +node scripts/maintenance/gridfs-recover.cjs \ + --db=my_database --bucket=fs \ + --id=0123456789abcdef01234567 \ + --expected-status=aborting \ + --expected-updated-at=2026-11-12T02:00:00.000Z +``` + +This previews but does not write. Retrieve the exact observed fields from the backed-up `.uploads` collection first. If the document contains a token, append `--expected-token=...`. **Do not copy** the example ObjectId or timestamp. Apply only after external process quiescence is verified and logged: + +```sh +node scripts/maintenance/gridfs-recover.cjs \ + --db=my_database --bucket=fs \ + --id=0123456789abcdef01234567 \ + --expected-status=aborting \ + --expected-updated-at=2026-11-12T02:00:00.000Z \ + --operator=oncall-id --reason=verified-writer-shutdown \ + --quiesced --apply +``` + +For an already `recovering` owner only, verify that the previous recovery executor has stopped and add `--prior-recovery-stopped` to the apply command; this is separate from stopping ordinary GridFS writers. + +Every apply attempt includes an audit entry. A crash can leave a `status: recovering` record with `recovery.token`; after proving **both all writers and the previous recovery executor have stopped**, re-inspect that record and use its new status, updatedAt and recovery token to retry. Supply the additional `--prior-recovery-stopped` flag. The restart acquires a fresh recovery token via atomic CAS. Two operators with the same observed snapshot cannot perform destructive steps simultaneously, but in-flight writes by an unquiesced prior executor still cannot be fenced by MongoDB alone. Never delete an ownership record by ID alone. + +## State-by-state recovery contract + +| Observed owner | After proven quiescence | Critical preservation rule | +|---|---|---| +| `kind:direct,status:uploading` | When a file is visible, verify every chunk, length and any claimed SHA before releasing ownership; otherwise delete orphan chunks and release | A mismatched caller-supplied metadata SHA blocks recovery; visible-file chunks are never deleted | +| `kind:delete,status:deleting` | Complete the interrupted delete of file and chunks before releasing ownership | No new upload may claim the same ID until both steps finish | +| `kind:cleanup,status:cleaning` | Check visible file; delete chunks only if no file exists; release | Never delete a visible file's chunks | +| `kind:rename,status:renaming` | Verify the resulting visible file has complete chunks, then release; missing or corrupt files remain locked | The tool does not choose or replay a rename target; the operator must inspect the intended filename | +| `status:pending,activeAppend` | Recompute the contiguous chunk prefix, clear only the observed append claim, restore seven-day idle expiry | Existing chunks are preserved; caller replays/checks the data | +| `status:finalizing` without file | Restore pending with expiry; retry normal finalize, which revalidates bytes | Never discard chunks to unlock finalize | +| `status:finalizing` with file | Re-hash the **actual** ordered chunks and check expected file length, chunk size and digest before releasing session | Forged metadata SHA is not sufficient | +| `status:aborting` without file | Delete invisible chunks then remove session | No newly created file can take ownership during recovery | +| `status:aborting` with file | **Blocked** for human forensic review | Never silently delete or detach a visible file | +| `status:recovering` | Resume by exact recovery token and reverified quiescence | Keep claim if any destructive step is ambiguous | + +### Known limitations and release criteria + +- MongoDB cannot revoke an already-running Node.js process's in-memory chunk write. `--quiesced` is a mandatory **external operational assertion**, not a technical fencing guarantee. For non-quiesceable workloads, a future generation-aware writer fencing design and driver integration would be needed; do **not** automate unlocking based on age. +- A direct uploadStream `metadata.sha256` is not authenticated at write time. The tool verifies visible chunk structure and length, but does not treat this metadata as a trusted digest for ordinary uploads. +- The recovery code is a source-checkout maintenance utility. It is not an online public `GridFSBucket` API and must not be exposed to unprivileged application callers. +- `drop()` is bucket-wide and ignores per-ID reservations; maintenance must globally stop writers before dropping the bucket. +- Deterministic tests must show all state paths, wrong-token and wrong-timestamp rejection, fail-closed interrupted recovery, no deletion of visible chunks, and idempotent restart after an interrupted abort/delete. **No merge until the code and this runbook pass the latest-HEAD required CI matrices and independent review.** diff --git a/docs/en/gridfs.md b/docs/en/gridfs.md index 230db4b..d79980a 100644 --- a/docs/en/gridfs.md +++ b/docs/en/gridfs.md @@ -1,6 +1,6 @@ # GridFS -Instance and Model helpers store and read MongoDB GridFS files. The published npm package is still 3.4.0; this API is staged in the source after that release and is not a published 3.5.0 package. +Instance and Model helpers store and read MongoDB GridFS files. GridFS was introduced in monSQLize v3.5.0. This document describes the source checkout; verify the installed npm version and its changelog before relying on newer lifecycle behavior. GridFS is a client convention on ordinary collections (`.files` and `.chunks`). It does not require a Search or Vector Search deployment. Integration coverage uses `mongodb-memory-server` 7.0.37. The formal server matrix remains MongoDB 7.x and 8.x. Older servers, including 4.x, are unverified. @@ -86,6 +86,7 @@ Chunk bytes must follow the session size: - `fromChunk` past the contiguous prefix throws `GRIDFS_RESUME_MISMATCH` with `expectedNextChunk`. - Replaying already stored chunks with the same bytes is idempotent and does not move `nextChunk` backward. +- Concurrent appends to one uploadId are serialized by a durable session claim. A competing append, finalize, or abort fails with `GRIDFS_UPLOAD_BUSY` while another operation owns the session. - Replaying an earlier chunk with different bytes throws `GRIDFS_RESUME_MISMATCH`. - A payload that contains any new chunk must start at `nextChunk`. Starting earlier throws `GRIDFS_RESUME_MISMATCH` and writes nothing. - A duplicate `{ files_id, n }` insert is accepted only when the stored bytes match. @@ -94,9 +95,11 @@ Chunk bytes must follow the session size: Retrying `finalizeResumableUpload` is idempotent while the session still exists: if a file with the same id is already visible and its `metadata.sha256` and length match the session, the call returns that file and deletes the session. A file with a different sha256 or length still throws `GRIDFS_FILE_EXISTS`. Once the session is gone (a finalize that fully succeeded, an abort, or the TTL), there is no digest to compare, so a retry throws `GRIDFS_FILE_NOT_FOUND`; use `getFile(uploadId)` to check the result. -`abortResumableUpload` deletes the session and, when no files document is visible, that upload's chunks. A missing or already aborted session resolves, and any chunks left under that id without a visible file are still deleted. Aborting an id that already has a visible file does not delete that file's chunks. +`abortResumableUpload` atomically claims a pending session before deleting chunks. It fails with `GRIDFS_UPLOAD_BUSY` while append or finalize owns the session. It deletes the session and, when no files document is visible, that upload's chunks. A missing or already aborted session resolves, and any chunks left under that id without a visible file are still deleted. Aborting an id that already has a visible file does not delete that file's chunks. -`.uploads` has a TTL index on `createdAt` with `expireAfterSeconds: 604800` (7 days). The TTL deletes upload session documents only. It does not delete chunks. Abort or finalize sessions that are no longer needed. After a session expired, `abortResumableUpload(uploadId)` removes its leftover chunks, and `beginResumableUpload` with the same explicit `id` clears them before the new session starts. +`.uploads` uses an `expiresAt` TTL index (`expireAfterSeconds: 0`). Pending resumable sessions expire after **7 days without a successful append**; a successful append refreshes their expiry. Active append/finalize/abort claims and direct upload/delete reservations have no TTL and must be recovered explicitly after a crash, otherwise a stale writer could resume after its lock expires. MongoDB TTL deletion is asynchronous and removes only the session document, not chunks. Known orphan chunks can be cleaned with `abortResumableUpload(uploadId)` after an exclusive cleanup claim, or by starting a new resumable upload with the same ID. Existing `createdAt` TTL indexes are migrated to `expiresAt`; legacy pending expiry is derived from their last update. The migration requires stopping/draining old GridFS writers; mixing old and new writer versions is unsafe. + +**Crash and migration recovery:** see [the operator runbook](./gridfs-recovery.md). Recovery is dry-run by default and requires verified process quiescence, an exact state/token check and an audit record; do not delete stale-looking reservations manually. ## Indexes @@ -107,9 +110,9 @@ Retrying `finalizeResumableUpload` is idempotent while the session still exists: | `.files` | `{ filename: 1, uploadDate: 1 }` | | | `.files` | `{ 'metadata.sha256': 1 }` | not unique | | `.chunks` | `{ files_id: 1, n: 1 }` | unique | -| `.uploads` | `{ createdAt: 1 }` | `expireAfterSeconds: 604800` | +| `.uploads` | `{ expiresAt: 1 }` | `expireAfterSeconds: 0`; pending resumable sessions set expiry for seven idle days | -Reads do not create indexes. A write rejected by `writePathPolicy` does not create them either. +Reads do not create indexes. A write rejected by `writePathPolicy` does not create them either. Every file ID is reserved atomically in `.uploads` during upload, uploadStream, rename, and delete. A different operation using the same ID receives `GRIDFS_UPLOAD_BUSY`. `drop()` remains a bucket-wide destructive maintenance operation: stop/drain writers before calling it; the per-ID reservations do not coordinate bucket-wide drop. ## writePathPolicy @@ -135,6 +138,7 @@ A blocked call throws `INVALID_OPERATION` before any GridFS write. `details[0]` | `GRIDFS_INTEGRITY_MISMATCH` | Bytes do not match `sha256`, or a finalized chunk has the wrong size | | `GRIDFS_UPLOAD_INCOMPLETE` | Finalize found gaps. `details[0].missingChunks` lists them | | `GRIDFS_RESUME_MISMATCH` | `fromChunk` is ahead of the contiguous prefix, or a replayed chunk differs | +| `GRIDFS_UPLOAD_BUSY` | Another append/finalize/abort/upload/delete operation owns the ID; do not retry until it finishes or is explicitly recovered | | `GRIDFS_RANGE_INVALID` | `start` / `end` is not a legal half-open range | | `INVALID_ARGUMENT` | Filename, bucket name, chunk size, digest, metadata key, or chunk payload is invalid | | `INVALID_OPERATION` | `writePathPolicy` blocked the call | diff --git a/docs/zh/gridfs-recovery.md b/docs/zh/gridfs-recovery.md new file mode 100644 index 0000000..c5c6139 --- /dev/null +++ b/docs/zh/gridfs-recovery.md @@ -0,0 +1,62 @@ +# GridFS 故障恢复与安全升级操作规程(候选 V0.1) + +> **仅供运维管理员使用,当前未完成 CI 验收。** 不得对仍在运行、暂停或网络分区状态的写入进程执行恢复。所有权记录时间较久,不等于原写入进程已经终止。禁止按超时自动抢占所有权。 + +## 不可破坏的安全约束 + +`.uploads` 持久化同一文件 ID 的上传、追加、完成、取消、删除、重命名和孤儿清理的互斥所有权。**文件记录不存在不代表其 chunks 可删**:上传过程中,files 文档尚未可见;删除过程中,MongoDB Driver 可能先删 files,再删 chunks。 + +任何强制恢复必须先从集群所有节点停止并排空旧版和新版 GridFS 写入进程、队列消费者、定时任务与自动重启,确认没有仍可恢复的旧写入。恢复脚本无法在 MongoDB 层直接阻止旧进程已发出的 chunks 写入;只有外部进程停机隔离后才安全。不得混用忽略 ownership 的旧版与新版本。 + +## 上线和回滚步骤 + +1. 冻结所有 GridFS 写入,禁止新请求和自动重启,核查所有旧版 Writer 已排空;保留相关运行实例、进程、队列、版本和时间证据。 +2. 备份 `.files`、`.chunks`、`.uploads`、索引及配置;备份应与目标集群隔离。 +3. 在冻结状态部署新版本并运行 `ensureIndexes()`。迁移需将 legacy `createdAt_1` TTL 改为 `expiresAt_1`,以最后活动时间计算普通待上传会话的七天有效期,活动所有权和无 TTL 状态不得被自动过期。迁移前就过期的旧会话可能已被 TTL 清除,需要额外核对孤儿 chunks。 +4. 对每一个需要恢复的 ID,核对上传会话状态、ownerToken/activeAppend.token/recovery.token、updatedAt、files 和 chunks 的一致性,确认目标动作并保留审计记录。 +5. **先 dry-run,后 apply**。apply 必须明确声明写入进程已停机、实际操作者、恢复原因,并提供精确原始状态、时间戳和令牌。脚本先写入审计记录,再使用条件更新取得 `recovering` 所有权,失败时保留该所有权,禁止直接删除锁。 +6. 核对实际 bytes、chunks 顺序/大小、可见性、恢复后状态,并完成当前 HEAD 的 Node 18/20/22、MongoDB 7.0.37/8.0.26、对应 Driver 6/7 和 Release Preflight 全矩阵后再恢复写入。 +7. **回滚到旧版并非直接安全**:旧版不会识别新 ownership,必须先停止并排空全部新版写入,审查所有未完成会话和残留锁,再评估可否回滚。不能简单恢复旧的 `createdAt` TTL。 + +## 操作方式 + +从已安装依赖的仓库源码目录执行,MongoDB URI 通过环境变量传递,不要在 shell 命令参数或 Issue 中公开凭证。 + +```sh +export MONSQLIZE_RECOVERY_MONGODB_URI='<从密钥管理系统读取的 MongoDB URI>' + +# 默认仅读取,不做变更;使用数据库里实际查询到的 ID、状态和时间 +node scripts/maintenance/gridfs-recover.cjs \ + --db=database --bucket=fs --id=0123456789abcdef01234567 \ + --expected-status=finalizing \ + --expected-updated-at=2026-11-12T02:00:00.000Z + +# 完成所有进程隔离与审计后,才允许执行 +node scripts/maintenance/gridfs-recover.cjs \ + --db=database --bucket=fs --id=0123456789abcdef01234567 \ + --expected-status=finalizing \ + --expected-updated-at=2026-11-12T02:00:00.000Z \ + --operator=oncall-id --reason=writers-drained \ + --quiesced --apply +``` + +以上 ID、时间戳仅为格式示例。如果记录存在 `ownerToken`、`activeAppend.token` 或 `recovery.token`,必须加 `--expected-token=实际令牌`。任何状态/令牌/时间戳不符都应中止,重新检查。**当 `status: recovering` 时,还必须先确认上一轮恢复执行器确实已停止,执行 apply 时加 `--prior-recovery-stopped`。**恢复续接会通过 CAS 原子轮换令牌,两个持同一旧快照的恢复进程不能同时执行删除;但仍无法隔离未停止的旧进程已经发出的 MongoDB 写入。 + +## 按状态恢复 + +| 状态 | 恢复动作 | 数据保护 | +|---|---|---| +| direct/uploading | 可见文件先验证 chunks 完整性及声明的 SHA 再解锁;没有 files 才清理孤儿 chunks | SHA 元数据不可信,但一旦与实际 bytes 不符就阻断;绝不删除可见文件 chunks | +| delete/deleting | 继续删除 files 和 chunks,之后才释放同 ID 的所有权 | 删除未完成时禁止替换上传 | +| cleanup/cleaning | 可见文件保留 chunks,否则清理孤儿 chunks | 先查 files | +| rename/renaming | 先核验可见文件及完整 chunks,缺失或损坏时保留锁;运维核查文件名 | 工具不猜测目标或重新执行 rename | +| pending + activeAppend | 根据 chunks 的连续前缀恢复 nextChunk,清除中断令牌,重新设置七天闲置过期 | 保留已写 chunks,后续需验证重传 | +| finalizing,文件不可见 | 退回 pending,正常 finalize 重新校验 | 不删已有 chunks | +| finalizing,文件可见 | 核对文件长度、chunk size、实际 SHA 和元数据,再清理遗留会话 | 不信任单独的元数据 SHA | +| aborting,文件不可见 | 清理孤儿 chunks,移除会话 | 无新写入抢占 | +| aborting,文件可见 | **阻断;人工分析** | 严禁删除可见文件 | +| recovering | 必须确认上一轮恢复执行器也已停止,加 `--prior-recovery-stopped` 并通过 CAS 取得新恢复令牌 | 同一旧快照不能双重恢复,失败仍保留锁 | + +## 失败和验收边界 + +恢复脚本位于源码 `scripts/maintenance/gridfs-recover.cjs`,不是对外公开 API。没有绝对可靠的在线“僵尸 Writer 自动判断”,因此不能靠超时解锁。必须以确定性测试覆盖异常中断后状态、错误令牌拒绝、错误时间戳拒绝、可见文件 chunks 绝不误删、finalize 重验 SHA、abort/delete 幂等续接、审计结果及两版 MongoDB。验证结果未全绿前不能把 PR #12 标记 Ready 或合入 main。 \ No newline at end of file diff --git a/docs/zh/gridfs.md b/docs/zh/gridfs.md index 0675e40..f82e31d 100644 --- a/docs/zh/gridfs.md +++ b/docs/zh/gridfs.md @@ -1,6 +1,6 @@ # GridFS -实例和 Model 助手用于存取 MongoDB GridFS 文件。已发布的 npm 包仍是 3.4.0;本 API 位于该版本之后的源码中,还不是已发布的 3.5.0 包。 +实例和 Model 助手用于存取 MongoDB GridFS 文件。GridFS API 首次发布于 monSQLize v3.5.0。本文档描述当前源码检出版本;使用较新的生命周期行为前应核对实际安装的 npm 版本及其变更记录。 GridFS 是普通集合上的客户端约定(`.files` 与 `.chunks`),不依赖 Search 或 Vector Search。集成测试使用 `mongodb-memory-server` 7.0.37。正式服务器矩阵仍是 MongoDB 7.x 与 8.x。更早的服务器(包括 4.x)未经验证。 @@ -86,6 +86,7 @@ await bucket.abortResumableUpload(session.uploadId); - `fromChunk` 超过从 0 开始的连续前缀时,抛出 `GRIDFS_RESUME_MISMATCH`,并带上 `expectedNextChunk`。 - 用相同字节重放已写入的分片是幂等的,不会把 `nextChunk` 往回退。 +- 同一 uploadId 的并发 append 使用持久化所有权串行化;append、finalize 或 abort 遇到其他活跃操作时返回 `GRIDFS_UPLOAD_BUSY`。 - 用不同字节重放已写入的分片会抛出 `GRIDFS_RESUME_MISMATCH`。 - 只要 payload 含有新分片,`fromChunk` 就必须等于 `nextChunk`。从更早的序号开始会抛出 `GRIDFS_RESUME_MISMATCH`,并且不会写入。 - `{ files_id, n }` 唯一索引冲突只有在已存字节相同的时候才算成功。 @@ -94,9 +95,11 @@ await bucket.abortResumableUpload(session.uploadId); 会话仍在时,重试 `finalizeResumableUpload` 是幂等的:若同 id 的文件已可见,且其 `metadata.sha256` 与长度都和会话一致,调用直接返回该文件并删除会话。sha256 或长度不同的文件仍抛 `GRIDFS_FILE_EXISTS`。会话已不存在时(上一次 finalize 已完整成功、已 abort 或 TTL 过期),没有可比对的摘要,重试抛 `GRIDFS_FILE_NOT_FOUND`,可用 `getFile(uploadId)` 确认结果。 -`abortResumableUpload` 删除会话;如果还没有可见的 files 文档,同时删除该上传的 chunks。会话不存在或已经中止时直接成功,并且仍会删除该 id 下没有可见文件的残留 chunks。对已经可见的文件调用 abort 不会删除该文件的 chunks。 +`abortResumableUpload` 先原子取得 pending 会话的所有权;append 或 finalize 仍在执行时返回 `GRIDFS_UPLOAD_BUSY`,绝不删除其 chunks。取得所有权后删除会话;如果还没有可见的 files 文档,同时删除该上传的 chunks。会话不存在或已经中止时直接成功,并且仍会删除该 id 下没有可见文件的残留 chunks。对已经可见的文件调用 abort 不会删除该文件的 chunks。 -`.uploads` 在 `createdAt` 上有 TTL 索引,`expireAfterSeconds` 为 604800(7 天)。TTL 只删除上传会话文档,不删除 chunks。不再需要的会话应 abort 或 finalize。会话过期后,`abortResumableUpload(uploadId)` 会清掉残留 chunks;用同一显式 `id` 调 `beginResumableUpload` 也会在新会话开始前清掉它们。 +`.uploads` 使用 `{ expiresAt: 1 }` TTL 索引,`expireAfterSeconds: 0`。pending 续传会话在**连续 7 天没有成功 append**时过期,每次成功 append 会刷新过期时间;正在执行的 append/finalize/abort,以及普通上传或删除的所有权记录不设置 TTL,进程异常退出后需确认旧进程停止再显式恢复,不能自动释放导致旧写入复活。MongoDB TTL 是异步清理,且只删除 session,不级联 chunks。已知 uploadId 的孤儿块由 `abortResumableUpload(uploadId)` 在取得独占清理权后回收,或复用同一 id 重新开始续传时清理。旧 `createdAt` TTL 索引会迁移为 `expiresAt`;迁移前必须停止/排空旧版 GridFS 写进程,新旧写入协议混跑不安全。 + +**故障恢复与迁移:**参阅[运维恢复规程](./gridfs-recovery.md)。恢复默认只预览,必须在全部写入进程确实排空后,核对精确状态/令牌并留下审计记录;禁止直接删除看似过期的所有权。 ## 索引 @@ -107,9 +110,9 @@ await bucket.abortResumableUpload(session.uploadId); | `.files` | `{ filename: 1, uploadDate: 1 }` | | | `.files` | `{ 'metadata.sha256': 1 }` | 非唯一 | | `.chunks` | `{ files_id: 1, n: 1 }` | 唯一 | -| `.uploads` | `{ createdAt: 1 }` | `expireAfterSeconds: 604800` | +| `.uploads` | `{ expiresAt: 1 }` | `expireAfterSeconds: 0`;pending 会话空闲 7 天过期 | -读取不会创建索引。被 `writePathPolicy` 拒绝的写入也不会创建。 +读取不会创建索引。被 `writePathPolicy` 拒绝的写入也不会创建。upload、uploadStream、rename、delete 会在 `.uploads` 为同一文件 ID 原子取得所有权;其他操作占用 ID 时返回 `GRIDFS_UPLOAD_BUSY`。`drop()` 是整个 bucket 的破坏性维护操作:必须先停止或排空写入,单文件 ID 所有权不能保证与整个 bucket 的 drop 并发安全。 ## writePathPolicy @@ -135,6 +138,7 @@ GridFS 写入的类别是 `gridfs`。命名空间是 `.files`,匹配 | `GRIDFS_INTEGRITY_MISMATCH` | 字节与 `sha256` 不符,或 finalize 时分片长度不对 | | `GRIDFS_UPLOAD_INCOMPLETE` | finalize 发现缺口。`details[0].missingChunks` 列出缺少的序号 | | `GRIDFS_RESUME_MISMATCH` | `fromChunk` 超前于连续前缀,或重放的分片内容不同 | +| `GRIDFS_UPLOAD_BUSY` | 其他 append/finalize/abort/upload/delete 操作正在占用该 ID,必须等待完成或在确认旧进程停止后显式恢复 | | `GRIDFS_RANGE_INVALID` | `start` / `end` 不是合法的半开区间 | | `INVALID_ARGUMENT` | 文件名、bucket 名、分片大小、摘要、metadata 键或分片 payload 不合法 | | `INVALID_OPERATION` | `writePathPolicy` 阻断了调用 | diff --git a/scripts/maintenance/gridfs-recover.cjs b/scripts/maintenance/gridfs-recover.cjs new file mode 100644 index 0000000..d55af79 --- /dev/null +++ b/scripts/maintenance/gridfs-recover.cjs @@ -0,0 +1,304 @@ +'use strict'; + +/** + * Operator-only recovery for GridFS ownership records after an externally + * verified writer shutdown. Never run against a live or potentially paused + * writer; MongoDB cannot fence a previously issued driver chunk write. + * + * Dry-run is default. The caller must supply the exact observed state, + * updatedAt timestamp and (where applicable) ownership token to apply. + * A durable "recovering" claim makes interruption/retry fail closed. + */ +const { randomUUID, createHash } = require('node:crypto'); +const { MongoClient, ObjectId } = require('mongodb'); + +const TTL_MS = 604800000; + +function fail(message) { + const error = new Error(message); + error.code = 'GRIDFS_RECOVERY_PRECONDITION_FAILED'; + throw error; +} + +function toBuffer(data) { + if (Buffer.isBuffer(data)) return data; + if (data && typeof data.value === 'function') return Buffer.from(data.value(true)); + if (data instanceof Uint8Array) return Buffer.from(data); + fail('Encountered a chunk with unsupported BSON binary data'); +} + +async function verifyVisibleFile(db, bucket, id, session) { + const files = db.collection(bucket + '.files'); + const doc = await files.findOne({ _id: id }); + if (!doc) return false; + if (!Number.isSafeInteger(doc.length) || doc.length < 0 || + !Number.isSafeInteger(doc.chunkSize) || doc.chunkSize < 1) { + fail('Visible file has invalid length/chunkSize; preserve ownership'); + } + const totalChunks = Math.ceil(doc.length / doc.chunkSize); + const digest = createHash('sha256'); + let n = 0; + let length = 0; + for await (const chunk of db.collection(bucket + '.chunks') + .find({ files_id: id }).sort({ n: 1 })) { + if (chunk.n !== n || n >= totalChunks) fail('Visible file has a missing, extra or out-of-order chunk'); + const bytes = toBuffer(chunk.data); + const want = n === totalChunks - 1 ? doc.length - n * doc.chunkSize : doc.chunkSize; + if (bytes.length !== want) fail('Visible file has a corrupt chunk length'); + digest.update(bytes); + length += bytes.length; + n++; + } + if (length !== doc.length || n !== totalChunks) { + fail('Visible file is incomplete; preserve ownership and chunks'); + } + const actualSha = digest.digest('hex'); + if (session) { + if (session.length !== doc.length || session.chunkSizeBytes !== doc.chunkSize || + doc.metadata?.sha256 !== session.sha256 || actualSha !== session.sha256) { + fail('Finalizing session and visible bytes mismatch; preserve ownership and chunks'); + } + } else if (typeof doc.metadata?.sha256 === 'string') { + // UploadStream SHA is caller-supplied metadata, not proof of integrity. + // A mismatched claimed hash is nevertheless evidence of inconsistency: + // do not unlock an anomalous direct upload as if it had been verified. + if (!/^[0-9a-f]{64}$/i.test(doc.metadata.sha256) || + actualSha !== doc.metadata.sha256.toLowerCase()) { + fail('Visible direct upload metadata SHA mismatch; preserve ownership and chunks'); + } + } + return true; +} + +function snapshotToken(doc) { + return doc.recovery?.token || doc.ownerToken || doc.activeAppend?.token || null; +} + +function classify(doc) { + let state = doc.status === 'recovering' ? doc.recovery?.fromStatus : null; + if (doc.status === 'recovering' && !state) fail('Recovering record lacks its original state'); + if (!state) { + if (doc.kind === 'direct' && doc.status === 'uploading') state = 'uploading'; + else if (doc.kind === 'delete' && doc.status === 'deleting') state = 'deleting'; + else if (doc.kind === 'rename' && doc.status === 'renaming') state = 'renaming'; + else if (doc.kind === 'cleanup' && doc.status === 'cleaning') state = 'cleaning'; + else if ((!doc.kind || doc.kind === 'resumable') && + doc.status === 'pending' && doc.activeAppend) state = 'activeAppend'; + else if ((!doc.kind || doc.kind === 'resumable') && + (doc.status === 'finalizing' || doc.status === 'aborting')) state = doc.status; + } + // Also validate the persisted origin when a crash left status=recovering: + // do not allow malformed/corrupt origin metadata to unlock arbitrary IDs. + const requiredKind = { + uploading: 'direct', deleting: 'delete', renaming: 'rename', cleaning: 'cleanup', + }; + if (!state || (requiredKind[state] && doc.kind !== requiredKind[state]) || + (['activeAppend', 'finalizing', 'aborting'].includes(state) && + doc.kind && doc.kind !== 'resumable') || + (state === 'activeAppend' && !doc.activeAppend?.token)) { + fail('Inconsistent or unsupported owner origin; preserve ownership'); + } + return state; +} + +async function contiguousChunkPrefix(db, bucket, id, session) { + if (!Number.isSafeInteger(session.length) || session.length < 1 || + !Number.isSafeInteger(session.chunkSizeBytes) || session.chunkSizeBytes < 1) { + fail('Pending session has invalid length/chunkSizeBytes; preserve ownership'); + } + const count = Math.ceil(session.length / session.chunkSizeBytes); + let next = 0; + for await (const doc of db.collection(bucket + '.chunks') + .find({ files_id: id }).sort({ n: 1 })) { + if (doc.n !== next || next >= count) { + fail('Interrupted append contains a gap or extra chunk; preserve ownership'); + } + const want = next === count - 1 + ? session.length - next * session.chunkSizeBytes : session.chunkSizeBytes; + if (toBuffer(doc.data).length !== want) { + fail('Interrupted append contains a malformed chunk; preserve ownership'); + } + next++; + } + return next; +} + +async function recoverGridFSOwnership(input) { + const { db, bucket, id, expectedStatus, expectedUpdatedAt, expectedToken, + quiesced, operator, reason, apply = false, resumeQuiesced = false } = input; + if (!db || !bucket || typeof bucket !== 'string' || !(id instanceof ObjectId)) fail('Invalid database, bucket or ObjectId'); + if (!expectedStatus || !expectedUpdatedAt) fail('Exact expected status and updatedAt are required'); + const uploads = db.collection(bucket + '.uploads'); + const doc = await uploads.findOne({ _id: id }); + if (!doc) return { outcome: 'no_owner', modified: false }; + if (!(doc.updatedAt instanceof Date) || + doc.status !== expectedStatus || + doc.updatedAt.toISOString() !== expectedUpdatedAt) { + fail('Owner status or updatedAt changed since inspection'); + } + const token = snapshotToken(doc); + if ((token || null) !== (expectedToken || null)) fail('Owner token changed since inspection'); + const sourceStatus = classify(doc); + const visible = await db.collection(bucket + '.files').findOne({ _id: id }, { projection: { _id: 1 } }); + const plan = { + bucket, uploadId: id.toHexString(), fromStatus: sourceStatus, + fileVisible: !!visible, + action: ({ + uploading: 'verify-visible-or-remove-orphan-chunks', + deleting: 'complete-file-and-chunk-deletion', + renaming: 'release-after-inspecting-rename-outcome', + cleaning: 'remove-invisible-orphan-chunks', + activeAppend: 'restore-pending-and-reconcile-prefix', + finalizing: visible ? 'verify-visible-bytes-and-release-session' : 'restore-pending-for-retry', + aborting: visible ? 'BLOCK-visible-file-conflict' : 'finish-invisible-chunk-removal', + })[sourceStatus], + }; + if (!apply) return { outcome: 'dry_run', modified: false, plan }; + if (quiesced !== true || typeof operator !== 'string' || !operator.trim() || + typeof reason !== 'string' || !reason.trim()) { + fail('Apply requires external writer quiescence, named operator and reason'); + } + if (sourceStatus === 'aborting' && visible) fail('Visible file conflicts with aborting state; manual inspection required'); + // A recovering owner may belong to a running recovery executor. Caller + // must separately prove that executor has stopped before a manual resume. + if (doc.status === 'recovering' && resumeQuiesced !== true) { + fail('Resuming recovery requires verified shutdown of the prior recovery executor'); + } + + const audit = db.collection(bucket + '.ownership_recovery_audit'); + const attemptId = randomUUID(); + await audit.insertOne({ + _id: attemptId, fileId: id, bucket, fromStatus: sourceStatus, + operator, reason, startedAt: new Date(), outcome: 'started', plan, + }); + let recoveryToken; + if (doc.status === 'recovering') { + // Concurrent recovery attempts with one observed snapshot must not share + // the same destructive authority. Atomically replace the old token. + // A prior executor must ALSO have been externally confirmed stopped. + recoveryToken = randomUUID(); + const claim = await uploads.updateOne( + { _id: id, status: 'recovering', updatedAt: doc.updatedAt, + 'recovery.token': doc.recovery.token }, + { $set: { updatedAt: new Date(), 'recovery.token': recoveryToken, + 'recovery.operator': operator, 'recovery.attemptId': attemptId, + 'recovery.startedAt': new Date() } }, + ); + if (claim.matchedCount !== 1) { + await audit.updateOne({ _id: attemptId }, { + $set: { outcome: 'claim_conflict', finishedAt: new Date() }, + }); + fail('Concurrent recovery executor claimed the same owner first'); + } + await audit.updateOne({ _id: attemptId }, { $set: { outcome: 'resumed', recoveryToken } }); + } else { + recoveryToken = randomUUID(); + const filter = { _id: id, status: doc.status, updatedAt: doc.updatedAt }; + if (doc.kind === undefined) filter.kind = { $exists: false }; + else filter.kind = doc.kind; + if (doc.ownerToken !== undefined) filter.ownerToken = doc.ownerToken; + if (doc.activeAppend) filter['activeAppend.token'] = doc.activeAppend.token; + const claim = await uploads.updateOne(filter, { + $set: { + status: 'recovering', updatedAt: new Date(), + recovery: { token: recoveryToken, fromStatus: sourceStatus, operator, attemptId, startedAt: new Date() }, + }, + $unset: { expiresAt: '' }, + }); + if (claim.matchedCount !== 1) { + await audit.updateOne({ _id: attemptId }, { $set: { outcome: 'claim_conflict', finishedAt: new Date() } }); + fail('Concurrent state change prevented recovery claim'); + } + } + const held = { _id: id, status: 'recovering', 'recovery.token': recoveryToken }; + try { + const chunks = db.collection(bucket + '.chunks'); + const files = db.collection(bucket + '.files'); + if (sourceStatus === 'activeAppend') { + const nextChunk = await contiguousChunkPrefix(db, bucket, id, doc); + const changed = await uploads.updateOne(held, { + $set: { status: 'pending', nextChunk, updatedAt: new Date(), + expiresAt: new Date(Date.now() + TTL_MS) }, + $unset: { activeAppend: '', recovery: '' }, + }); + if (changed.matchedCount !== 1) fail('Lost recovery claim while restoring append'); + } else if (sourceStatus === 'finalizing' && !visible) { + const changed = await uploads.updateOne(held, { + $set: { status: 'pending', updatedAt: new Date(), + expiresAt: new Date(Date.now() + TTL_MS) }, + $unset: { recovery: '' }, + }); + if (changed.matchedCount !== 1) fail('Lost recovery claim while restoring finalize'); + } else { + // Verify before ANY deletion. Never delete chunks of a visible file + // unless this is an explicitly quiesced, retained "deleting" owner. + if (sourceStatus === 'finalizing') { + await verifyVisibleFile(db, bucket, id, doc); + } else if ((sourceStatus === 'uploading' || sourceStatus === 'renaming') && visible) { + await verifyVisibleFile(db, bucket, id, null); + } else if (sourceStatus === 'renaming' && !visible) { + fail('Renaming owner has no visible file; preserve ownership for manual inspection'); + } + if (sourceStatus === 'deleting') { + await files.deleteOne({ _id: id }); + await chunks.deleteMany({ files_id: id }); + } else if ((sourceStatus === 'uploading' || sourceStatus === 'cleaning' || + sourceStatus === 'aborting') && !visible) { + await chunks.deleteMany({ files_id: id }); + } + const removed = await uploads.deleteOne(held); + if (removed.deletedCount !== 1) fail('Lost recovery claim while releasing owner'); + } + await audit.updateOne({ _id: attemptId }, { $set: { outcome: 'completed', finishedAt: new Date() } }); + return { outcome: 'completed', modified: true, plan, attemptId }; + } catch (error) { + // Deliberately retain recovering state; retry using its exact token and + // updatedAt only after re-verifying process quiescence. + try { + await audit.updateOne({ _id: attemptId }, + { $set: { outcome: 'blocked', error: String(error), finishedAt: new Date() } }); + } catch { /* The recovery claim remains; preserve original error. */ } + throw error; + } +} + +function readArg(args, name) { + const prefix = '--' + name + '='; + const found = args.find((a) => a.startsWith(prefix)); + return found && found.slice(prefix.length); +} + +async function main() { + const args = process.argv.slice(2); + const uri = process.env.MONSQLIZE_RECOVERY_MONGODB_URI; + if (!uri) fail('MONSQLIZE_RECOVERY_MONGODB_URI is required (do not put credentials in argv)'); + const dbName = readArg(args, 'db'); + const bucket = readArg(args, 'bucket'); + const idString = readArg(args, 'id'); + if (!dbName || !bucket || !idString || !ObjectId.isValid(idString)) fail('--db, --bucket and --id are required'); + const client = new MongoClient(uri); + await client.connect(); + try { + const report = await recoverGridFSOwnership({ + db: client.db(dbName), bucket, id: new ObjectId(idString), + expectedStatus: readArg(args, 'expected-status'), + expectedUpdatedAt: readArg(args, 'expected-updated-at'), + expectedToken: readArg(args, 'expected-token') || null, + operator: readArg(args, 'operator'), reason: readArg(args, 'reason'), + quiesced: args.includes('--quiesced'), + resumeQuiesced: args.includes('--prior-recovery-stopped'), + apply: args.includes('--apply'), + }); + process.stdout.write(JSON.stringify(report, null, 2) + '\n'); + } finally { + await client.close(); + } +} + +module.exports = { recoverGridFSOwnership, verifyVisibleFile }; +if (require.main === module) { + main().catch((error) => { + process.stderr.write(String(error.code || 'RECOVERY_ERROR') + ': ' + error.message + '\n'); + process.exitCode = 1; + }); +} diff --git a/scripts/validation/run-memory-server-matrix.cjs b/scripts/validation/run-memory-server-matrix.cjs index 959aad4..f6a7071 100644 --- a/scripts/validation/run-memory-server-matrix.cjs +++ b/scripts/validation/run-memory-server-matrix.cjs @@ -45,6 +45,10 @@ const integrationSourceSuites = [ 'test/integration/runtime/review-regressions.test.js', 'test/integration/data-tasks/data-task-job-facade.test.js', 'test/integration/sync/sync.test.js', + 'test/integration/gridfs/gridfs.test.js', + 'test/integration/gridfs/gridfs-scan-regressions.test.js', + 'test/integration/gridfs/gridfs-ownership-regressions.test.js', + 'test/integration/gridfs/gridfs-stale-recovery.test.js', ]; const integrationSuites = integrationSourceSuites.map((suite) => path.join(testDistRoot, suite)); const compatibilitySuites = [ diff --git a/src/capabilities/gridfs/errors.ts b/src/capabilities/gridfs/errors.ts index f1c25f8..2d5801e 100644 --- a/src/capabilities/gridfs/errors.ts +++ b/src/capabilities/gridfs/errors.ts @@ -26,6 +26,7 @@ const GRIDFS_CODES = new Set([ ErrorCodes.GRIDFS_UPLOAD_INCOMPLETE, ErrorCodes.GRIDFS_RESUME_MISMATCH, ErrorCodes.GRIDFS_RANGE_INVALID, + ErrorCodes.GRIDFS_UPLOAD_BUSY, ErrorCodes.INVALID_ARGUMENT, ErrorCodes.INVALID_OPERATION, ErrorCodes.NOT_CONNECTED, @@ -85,6 +86,12 @@ export function integrityMismatch(operation: string, bucket: string, fields: { u ); } +export function uploadBusy(operation: string, ctx: Pick, uploadId: ObjectId): MonSQLizeError { + return gridfsError(ErrorCodes.GRIDFS_UPLOAD_BUSY, 'GridFS file id is owned by another operation', { + operation, bucket: ctx.bucketName, uploadId, + }); +} + export function uploadIncomplete(bucket: string, uploadId: ObjectId, missingChunks: number[]): MonSQLizeError { return gridfsError( ErrorCodes.GRIDFS_UPLOAD_INCOMPLETE, diff --git a/src/capabilities/gridfs/indexes.ts b/src/capabilities/gridfs/indexes.ts index 12eaf6a..6beaaf9 100644 --- a/src/capabilities/gridfs/indexes.ts +++ b/src/capabilities/gridfs/indexes.ts @@ -1,5 +1,5 @@ import type { Db } from 'mongodb'; -import { mapDriverError } from './errors'; +import { mapDriverError, isNamespaceNotFound } from './errors'; import { chunksCollectionName, filesCollectionName, uploadsCollectionName } from './normalize'; import { UPLOADS_TTL_SECONDS } from './types'; @@ -42,7 +42,20 @@ export class GridFSIndexLatch { await files.createIndex({ filename: 1, uploadDate: 1 }); await chunks.createIndex({ files_id: 1, n: 1 }, { unique: true }); await files.createIndex({ 'metadata.sha256': 1 }); - await uploads.createIndex({ createdAt: 1 }, { expireAfterSeconds: UPLOADS_TTL_SECONDS }); + // Legacy 3.5.3 sessions expired seven days after creation, even during active uploads. + // Migrate pending sessions before removing the legacy TTL index. The new index + // applies only to explicitly expiring pending sessions, not active ownership claims. + await uploads.updateMany( + { status: 'pending', activeAppend: { $exists: false }, createdAt: { $type: 'date' }, expiresAt: { $exists: false } }, + [{ $set: { expiresAt: { $add: [{ $ifNull: ['$updatedAt', '$createdAt'] }, UPLOADS_TTL_SECONDS * 1000] } } }], + ); + try { + await uploads.dropIndex('createdAt_1'); + } catch (error) { + const code = (error as { code?: number }).code; + if (code !== 27 && !isNamespaceNotFound(error)) throw error; + } + await uploads.createIndex({ expiresAt: 1 }, { expireAfterSeconds: 0 }); } catch (error) { throw mapDriverError(error, 'ensureIndexes', { bucketName: this.bucketName }); } diff --git a/src/capabilities/gridfs/operations.ts b/src/capabilities/gridfs/operations.ts index cf6ece1..0847055 100644 --- a/src/capabilities/gridfs/operations.ts +++ b/src/capabilities/gridfs/operations.ts @@ -13,6 +13,7 @@ import type { GridFSUploadStreamOptions, } from './types'; import { readReadable, releaseReadable } from './bytes'; +import { claimOwner, cleanupFailedDirect, releaseOwner } from './ownership'; import { fileExists, fileNotFound, integrityMismatch, invalidArgument, isDuplicateKey, isNamespaceNotFound, mapDriverError, targetOf } from './errors'; import { chunksCollectionName, filesCollectionName, toFileInfo, uploadsCollectionName } from './normalize'; import { @@ -81,12 +82,43 @@ export async function uploadFile( } const id = options.id ?? new ObjectId(); - const writeOptions = { chunkSizeBytes, metadata, contentType: options.contentType }; - if (payload) await writeBuffer(ctx, id, filename, payload, writeOptions); - else if (source) await writeReadable(ctx, id, filename, source, writeOptions); - const info = await getFile(ctx, id); - if (!info) throw fileNotFound('upload', ctx, { id, filename }); - return info; + // One atomic reservation excludes all other GridFS writers/deleters under this ID. + // The visible-file preflight above is not sufficient across collections or processes. + let owner: Awaited>; + try { + owner = await claimOwner(ctx, id, 'direct', 'upload'); + } catch (error) { + if (source) releaseReadable(source); + throw error; + } + try { + if (await getFile(ctx, id)) { + if (source) releaseReadable(source); + throw fileExists('upload', ctx, { id, filename, sha256: options.sha256 }); + } + const writeOptions = { chunkSizeBytes, metadata, contentType: options.contentType }; + if (payload) await writeBuffer(ctx, id, filename, payload, writeOptions); + else if (source) await writeReadable(ctx, id, filename, source, writeOptions); + const info = await getFile(ctx, id); + if (!info) throw fileNotFound('upload', ctx, { id, filename }); + // A successful upload is committed even if removing its reservation fails. + // Keep the durable reservation for explicit recovery rather than deleting the file. + try { + await releaseOwner(ctx, owner, 'upload'); + } catch (error) { + ctx.logger?.warn?.('[gridfs] upload committed but reservation cleanup failed', { bucket: ctx.bucketName, id: String(id), error }); + } + return info; + } catch (error) { + // Never release ownership before cleaning this upload's incomplete chunks. + // A failed cleanup intentionally leaves a durable reservation behind. + try { + await cleanupFailedDirect(ctx, owner); + } catch (cleanupError) { + ctx.logger?.warn?.('[gridfs] upload reservation retained after cleanup failure', { bucket: ctx.bucketName, id: String(id), error: cleanupError }); + } + throw error; + } } export function openUploadStream(ctx: GridFSContext, filename: string, opts?: GridFSUploadStreamOptions): Writable & { id: ObjectId; done: Promise } { @@ -104,10 +136,10 @@ export function openUploadStream(ctx: GridFSContext, filename: string, opts?: Gr input.on('error', () => undefined); const done = (async (): Promise => { let target: GridFSDriverStream | undefined; + let owner: Awaited> | undefined; try { - await ctx.indexes.ensure(); - // Refuse an id that already belongs to a file before writing anything (same rule as upload()), - // so failure cleanup can never delete data this call did not create. + owner = await claimOwner(ctx, id, 'direct', 'uploadStream'); + // Check again after the ID is reserved, in case an older unreserved file exists. if (await getFile(ctx, id)) { throw fileExists('uploadStream', ctx, { id, filename, sha256: options.sha256 }); } @@ -120,11 +152,22 @@ export function openUploadStream(ctx: GridFSContext, filename: string, opts?: Gr await pipeline(input, target); const info = await getFile(ctx, id); if (!info) throw fileNotFound('uploadStream', ctx, { id, filename }); + try { + await releaseOwner(ctx, owner, 'uploadStream'); + } catch (error) { + ctx.logger?.warn?.('[gridfs] uploadStream committed but reservation cleanup failed', { bucket: ctx.bucketName, id: String(id), error }); + } return info; } catch (error) { if (target) await cleanupFailedWrite(ctx, id, target, error); + if (owner) { + try { + await cleanupFailedDirect(ctx, owner); + } catch (cleanupError) { + ctx.logger?.warn?.('[gridfs] uploadStream reservation retained after cleanup failure', { bucket: ctx.bucketName, id: String(id), error: cleanupError }); + } + } const mapped = mapDriverError(error, 'uploadStream', ctx, { id, filename }); - // Fail the returned Writable too, so an upstream pipe() stops instead of hanging on backpressure. if (!input.destroyed) input.destroy(mapped); throw mapped; } @@ -198,27 +241,49 @@ export async function findFiles(ctx: GridFSContext, filter?: GridFSFindFilesFilt export async function deleteFile(ctx: GridFSContext, id: ObjectId, opts?: GridFSDeleteOptions): Promise { assertObjectId(id, 'delete', 'id'); - const existing = await getFile(ctx, id); - if (!existing) { - if (opts?.ignoreMissing) return; - throw fileNotFound('delete', ctx, { id }); - } + const owner = await claimOwner(ctx, id, 'delete', 'delete'); + let driverInvoked = false; try { - await ctx.driver.delete(id); + const existing = await getFile(ctx, id); + if (!existing) { + await releaseOwner(ctx, owner, 'delete'); + if (opts?.ignoreMissing) return; + throw fileNotFound('delete', ctx, { id }); + } + // Driver deletes files first, then chunks. Keep ownership until both finish; + // otherwise a replacement upload can lose its chunks to this old delete. + driverInvoked = true; + try { + await ctx.driver.delete(id); + } catch (error) { + throw mapDriverError(error, 'delete', ctx, { id, filename: existing.filename }); + } + await releaseOwner(ctx, owner, 'delete'); } catch (error) { - throw mapDriverError(error, 'delete', ctx, { id, filename: existing.filename }); + // After driver invocation, an ambiguous partial deletion must retain the + // durable reservation for explicit recovery, never unlock a replacement. + if (!driverInvoked) { + try { await releaseOwner(ctx, owner, 'delete'); } catch { /* Preserve initial error. */ } + } + throw error; } } export async function renameFile(ctx: GridFSContext, id: ObjectId, filename: string): Promise { assertObjectId(id, 'rename', 'id'); assertFilename(filename, 'rename'); - const existing = await getFile(ctx, id); - if (!existing) throw fileNotFound('rename', ctx, { id, filename }); + const owner = await claimOwner(ctx, id, 'rename', 'rename'); try { - await ctx.driver.rename(id, filename); - } catch (error) { - throw mapDriverError(error, 'rename', ctx, { id, filename }); + const existing = await getFile(ctx, id); + if (!existing) throw fileNotFound('rename', ctx, { id, filename }); + try { + await ctx.driver.rename(id, filename); + } catch (error) { + throw mapDriverError(error, 'rename', ctx, { id, filename }); + } + } finally { + // Rename never removes chunks and is safe to unlock after driver settles. + await releaseOwner(ctx, owner, 'rename'); } } diff --git a/src/capabilities/gridfs/ownership.ts b/src/capabilities/gridfs/ownership.ts new file mode 100644 index 0000000..caac89b --- /dev/null +++ b/src/capabilities/gridfs/ownership.ts @@ -0,0 +1,83 @@ +import { randomUUID } from 'node:crypto'; +import type { ObjectId } from 'mongodb'; +import type { GridFSContext } from './types'; +import { fileExists, isDuplicateKey, mapDriverError, uploadBusy } from './errors'; +import { chunksCollectionName, filesCollectionName, uploadsCollectionName } from './normalize'; + +/** + * One cross-operation, per-file-ID ownership registry. + * All writers, deleters and orphan cleaners must claim the same unique _id before + * touching chunks. Do not auto-expire active owners: a paused writer could resume. + */ +export type GridFSOwnerKind = 'direct' | 'delete' | 'rename' | 'cleanup'; +export interface GridFSOwner { id: ObjectId; token: string; kind: GridFSOwnerKind } + +export async function claimOwner( + ctx: GridFSContext, id: ObjectId, kind: GridFSOwnerKind, operation: string, +): Promise { + await ctx.indexes.ensure(); + const token = randomUUID(); + try { + await ctx.db.collection(uploadsCollectionName(ctx.bucketName)).insertOne({ + _id: id, kind, status: kind === 'direct' ? 'uploading' : kind === 'delete' ? 'deleting' : kind === 'rename' ? 'renaming' : 'cleaning', + ownerToken: token, createdAt: new Date(), updatedAt: new Date(), + }); + } catch (error) { + if (isDuplicateKey(error)) throw uploadBusy(operation, ctx, id); + throw mapDriverError(error, operation, ctx, { uploadId: id }); + } + return { id, token, kind }; +} + +export async function releaseOwner(ctx: GridFSContext, owner: GridFSOwner, operation: string): Promise { + const status = owner.kind === 'direct' ? 'uploading' + : owner.kind === 'delete' ? 'deleting' + : owner.kind === 'rename' ? 'renaming' : 'cleaning'; + try { + const released = await ctx.db.collection(uploadsCollectionName(ctx.bucketName)).deleteOne({ + _id: owner.id, kind: owner.kind, ownerToken: owner.token, status, + }); + // Recovery intentionally retains the original kind/ownerToken while + // changing status to recovering. A late original writer must not be + // able to remove that new recovery fence with its stale ownerToken. + if (released.deletedCount !== 1) throw uploadBusy(operation, ctx, owner.id); + } catch (error) { + throw mapDriverError(error, operation, ctx, { uploadId: owner.id }); + } +} + +/** Return an ID to the pool only after its chunks are known to have been cleaned. */ +export async function cleanupFailedDirect(ctx: GridFSContext, owner: GridFSOwner): Promise { + try { + const visible = await ctx.db.collection(filesCollectionName(ctx.bucketName)) + .findOne({ _id: owner.id }, { projection: { _id: 1 } }); + if (!visible) { + await ctx.db.collection(chunksCollectionName(ctx.bucketName)) + .deleteMany({ files_id: owner.id }); + } + await releaseOwner(ctx, owner, 'upload'); + } catch (error) { + // Keep the durable reservation on failure: another upload must not reuse + // the id while an earlier write or its cleanup is ambiguous. + throw mapDriverError(error, 'upload', ctx, { uploadId: owner.id }); + } +} + +/** + * A known orphan may only be cleaned after claiming the same id that uploads use. + * The visible-file check must happen while this claim excludes all new writers. + */ +export async function cleanupOrphanById(ctx: GridFSContext, id: ObjectId): Promise { + const owner = await claimOwner(ctx, id, 'cleanup', 'abortResumableUpload'); + let safeToRelease = false; + try { + const visible = await ctx.db.collection(filesCollectionName(ctx.bucketName)) + .findOne({ _id: id }, { projection: { _id: 1 } }); + if (!visible) await ctx.db.collection(chunksCollectionName(ctx.bucketName)).deleteMany({ files_id: id }); + safeToRelease = true; + } catch (error) { + throw mapDriverError(error, 'abortResumableUpload', ctx, { uploadId: id }); + } finally { + if (safeToRelease) await releaseOwner(ctx, owner, 'abortResumableUpload'); + } +} diff --git a/src/capabilities/gridfs/resumable.ts b/src/capabilities/gridfs/resumable.ts index 54be7ef..bb3245a 100644 --- a/src/capabilities/gridfs/resumable.ts +++ b/src/capabilities/gridfs/resumable.ts @@ -1,8 +1,9 @@ -import { createHash } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; import type { Readable } from 'node:stream'; import { ObjectId } from 'mongodb'; import type { AppendChunksResult, BeginResumableUploadOptions, BeginResumableUploadResult, GridFSContext, GridFSFileInfo, GridFSUploadSession } from './types'; -import { fileExists, fileNotFound, integrityMismatch, invalidArgument, isDuplicateKey, mapDriverError, resumeMismatch, uploadIncomplete } from './errors'; +import { fileExists, fileNotFound, integrityMismatch, invalidArgument, isDuplicateKey, mapDriverError, resumeMismatch, uploadBusy, uploadIncomplete } from './errors'; +import { cleanupOrphanById } from './ownership'; import { binaryToBuffer, chunksCollectionName, filesCollectionName, sameBytes, toFileInfo, uploadsCollectionName } from './normalize'; import { assertChunkSize, @@ -55,10 +56,12 @@ export async function beginResumableUpload(ctx: GridFSContext, filename: string, chunkSizeBytes, ...(metadata ? { metadata } : {}), ...(opts.contentType !== undefined ? { contentType: opts.contentType } : {}), + kind: 'resumable', status: 'pending', nextChunk: 0, createdAt: now, updatedAt: now, + expiresAt: new Date(now.getTime() + 604800 * 1000), }); } catch (error) { throw mapDriverError(error, 'beginResumableUpload', ctx, { uploadId, filename }); @@ -83,88 +86,140 @@ export async function appendChunks( assertNonNegativeInteger(fromChunk, 'appendChunks', 'fromChunk'); const payload = await payloadBytes(data); if (payload.length === 0) throw badChunkSize(); - const session = await requirePending(ctx, uploadId, 'appendChunks'); - const nextChunk = await contiguousNext(ctx, uploadId, session.nextChunk); - if (fromChunk > nextChunk) { - throw resumeMismatch('appendChunks', ctx.bucketName, { uploadId, fromChunk, expectedNextChunk: nextChunk }); - } - - const slices = planChunkSlices(payload, fromChunk, session.length, session.chunkSizeBytes); - const lastIndex = slices[slices.length - 1].index; - if (lastIndex >= nextChunk && fromChunk !== nextChunk) { - throw resumeMismatch('appendChunks', ctx.bucketName, { uploadId, fromChunk, expectedNextChunk: nextChunk }); - } - for (const slice of slices) { - if (slice.index < nextChunk) await assertStoredChunk(ctx, uploadId, slice.index, slice.bytes, nextChunk); - else await writeChunk(ctx, uploadId, slice.index, slice.bytes, nextChunk); - } - - const advanced = await contiguousNext(ctx, uploadId, nextChunk); - let matched: number; + const uploads = ctx.db.collection(uploadsCollectionName(ctx.bucketName)); + const token = randomUUID(); + // A durable single-append claim excludes finalize, abort, and concurrent + // appenders. Remove the expiry while this process is actively writing. + let claimed: number; try { - const updated = await ctx.db.collection(uploadsCollectionName(ctx.bucketName)).updateOne( - { _id: uploadId, status: 'pending' }, - { $max: { nextChunk: advanced }, $set: { updatedAt: new Date() } }, + const update = await uploads.updateOne( + { _id: uploadId, status: 'pending', activeAppend: { $exists: false } }, + { $set: { activeAppend: { token, startedAt: new Date() }, updatedAt: new Date() }, $unset: { expiresAt: '' } }, ); - matched = updated.matchedCount; + claimed = update.matchedCount; } catch (error) { throw mapDriverError(error, 'appendChunks', ctx, { uploadId }); } - if (matched === 0) { - // The session was aborted (or expired) while this call was writing; do not leave its chunks behind. - await removeInvisibleChunks(ctx, uploadId, 'appendChunks'); - throw fileNotFound('appendChunks', ctx, { id: uploadId }); + if (!claimed) { + const existing = await uploads.findOne({ _id: uploadId }); + if (!existing) throw fileNotFound('appendChunks', ctx, { id: uploadId }); + throw uploadBusy('appendChunks', ctx, uploadId); + } + try { + const session = await requireSession(ctx, uploadId, 'appendChunks'); + const nextChunk = await contiguousNext(ctx, uploadId, session.nextChunk); + if (fromChunk > nextChunk) { + throw resumeMismatch('appendChunks', ctx.bucketName, { uploadId, fromChunk, expectedNextChunk: nextChunk }); + } + const slices = planChunkSlices(payload, fromChunk, session.length, session.chunkSizeBytes); + const lastIndex = slices[slices.length - 1].index; + if (lastIndex >= nextChunk && fromChunk !== nextChunk) { + throw resumeMismatch('appendChunks', ctx.bucketName, { uploadId, fromChunk, expectedNextChunk: nextChunk }); + } + for (const slice of slices) { + if (slice.index < nextChunk) await assertStoredChunk(ctx, uploadId, slice.index, slice.bytes, nextChunk); + else await writeChunk(ctx, uploadId, slice.index, slice.bytes, nextChunk); + } + const advanced = await contiguousNext(ctx, uploadId, nextChunk); + const completed = await uploads.updateOne( + { _id: uploadId, status: 'pending', 'activeAppend.token': token }, + { + $max: { nextChunk: advanced }, + $set: { updatedAt: new Date(), expiresAt: new Date(Date.now() + 604800 * 1000) }, + $unset: { activeAppend: '' }, + }, + ); + if (!completed.matchedCount) throw uploadBusy('appendChunks', ctx, uploadId); + return { nextChunk: advanced, bytesAccepted: payload.length }; + } catch (error) { + // The worker has stopped writing. Releasing only its own token allows + // the next append to reconcile partial chunks idempotently. If release + // itself fails the persistent claim remains for explicit recovery. + try { + await uploads.updateOne( + { _id: uploadId, status: 'pending', 'activeAppend.token': token }, + { $unset: { activeAppend: '' }, $set: { updatedAt: new Date(), expiresAt: new Date(Date.now() + 604800 * 1000) } }, + ); + } catch { /* Preserve the original failure; do not remove ownership by ID alone. */ } + throw error; } - return { nextChunk: advanced, bytesAccepted: payload.length }; } export async function finalizeResumableUpload(ctx: GridFSContext, uploadId: ObjectId): Promise { assertObjectId(uploadId, 'finalizeResumableUpload', 'uploadId'); - const session = await requirePending(ctx, uploadId, 'finalizeResumableUpload'); + const uploads = ctx.db.collection(uploadsCollectionName(ctx.bucketName)); + let session = await uploads.findOne({ _id: uploadId }) as unknown as GridFSUploadSession | null; + if (!session || (session.kind && session.kind !== 'resumable')) { + throw fileNotFound('finalizeResumableUpload', ctx, { id: uploadId }); + } + // An interrupted finalize can be retried when its file is already visible. + // A second in-flight finalize is not allowed to race the first. + if (session.status === 'finalizing') { + const committed = await committedFile(ctx, uploadId, session); + if (committed) return committed; + throw uploadBusy('finalizeResumableUpload', ctx, uploadId); + } + if (session.status !== 'pending') throw uploadBusy('finalizeResumableUpload', ctx, uploadId); + const transitioned = await uploads.updateOne( + { _id: uploadId, status: 'pending', activeAppend: { $exists: false } }, + { $set: { status: 'finalizing', updatedAt: new Date() }, $unset: { expiresAt: '' } }, + ); + if (!transitioned.matchedCount) throw uploadBusy('finalizeResumableUpload', ctx, uploadId); + // Only the winner of this CAS may inspect/commit files. Abort will now + // fail its pending -> aborting CAS instead of deleting validated chunks. + session = await requireSession(ctx, uploadId, 'finalizeResumableUpload'); const files = ctx.db.collection(filesCollectionName(ctx.bucketName)); const committed = await committedFile(ctx, uploadId, session); if (committed) return committed; - const expected = chunkCount(session.length, session.chunkSizeBytes); - const hash = createHash('sha256'); - let total = 0; - const missing: number[] = []; - const cursor = ctx.db.collection(chunksCollectionName(ctx.bucketName)).find({ files_id: uploadId }).sort({ n: 1 }); - let seen = 0; - for await (const doc of cursor) { - const n = typeof doc.n === 'number' ? doc.n : -1; - while (seen < n && seen < expected) { + try { + const expected = chunkCount(session.length, session.chunkSizeBytes); + const hash = createHash('sha256'); + let total = 0; + const missing: number[] = []; + const cursor = ctx.db.collection(chunksCollectionName(ctx.bucketName)).find({ files_id: uploadId }).sort({ n: 1 }); + let seen = 0; + for await (const doc of cursor) { + const n = typeof doc.n === 'number' ? doc.n : -1; + while (seen < n && seen < expected) { + missing.push(seen); + seen += 1; + } + if (n !== seen || n < 0 || n >= expected) { + if (n >= 0) missing.push(n); + continue; + } + const bytes = binaryToBuffer(doc.data, 'finalizeResumableUpload'); + const want = chunkLengthAt(n, session.length, session.chunkSizeBytes); + if (bytes.length !== want) { + throw integrityMismatch('finalizeResumableUpload', ctx.bucketName, { uploadId, expected: want, actual: bytes.length }); + } + hash.update(bytes); + total += bytes.length; + seen += 1; + } + while (seen < expected) { missing.push(seen); seen += 1; } - if (n !== seen || n < 0 || n >= expected) { - if (n >= 0) missing.push(n); - continue; + if (missing.length > 0) throw uploadIncomplete(ctx.bucketName, uploadId, missing); + if (total !== session.length) { + throw integrityMismatch('finalizeResumableUpload', ctx.bucketName, { uploadId, expected: session.length, actual: total }); } - const bytes = binaryToBuffer(doc.data, 'finalizeResumableUpload'); - const want = chunkLengthAt(n, session.length, session.chunkSizeBytes); - if (bytes.length !== want) { - throw integrityMismatch('finalizeResumableUpload', ctx.bucketName, { - uploadId, - expected: want, - actual: bytes.length, - }); + const actual = hash.digest('hex'); + if (actual !== session.sha256) { + throw integrityMismatch('finalizeResumableUpload', ctx.bucketName, { uploadId, expected: session.sha256, actual }); } - hash.update(bytes); - total += bytes.length; - seen += 1; - } - while (seen < expected) { - missing.push(seen); - seen += 1; - } - if (missing.length > 0) throw uploadIncomplete(ctx.bucketName, uploadId, missing); - if (total !== session.length) { - throw integrityMismatch('finalizeResumableUpload', ctx.bucketName, { uploadId, expected: session.length, actual: total }); - } - const actual = hash.digest('hex'); - if (actual !== session.sha256) { - throw integrityMismatch('finalizeResumableUpload', ctx.bucketName, { uploadId, expected: session.sha256, actual }); + } catch (error) { + // Validation failed before any files insert. Return to pending so the + // caller can supply missing/correct chunks; do not erase those chunks. + try { + await uploads.updateOne( + { _id: uploadId, status: 'finalizing' }, + { $set: { status: 'pending', updatedAt: new Date(), expiresAt: new Date(Date.now() + 604800 * 1000) } }, + ); + } catch { /* The retained finalizing state must be recovered explicitly. */ } + throw error; } const metadata = withContentType( @@ -185,13 +240,13 @@ export async function finalizeResumableUpload(ctx: GridFSContext, uploadId: Obje const raced = await committedFile(ctx, uploadId, session); if (raced) return raced; } + // Commit outcome may be ambiguous; hold finalizing ownership rather than + // allowing an abort or another writer to delete/reuse the same chunks. throw mapDriverError(error, 'finalizeResumableUpload', ctx, { uploadId, filename: session.filename }); } try { - await ctx.db.collection(uploadsCollectionName(ctx.bucketName)).deleteOne({ _id: uploadId }); + await uploads.deleteOne({ _id: uploadId, status: 'finalizing' }); } catch (error) { - // The files document is committed, so the upload succeeded. A leftover session cannot add chunks past - // the file length, abort keeps a visible file's chunks, and the TTL index removes it later. ctx.logger?.warn?.('[gridfs] finalizeResumableUpload could not delete the upload session', { bucket: ctx.bucketName, uploadId: String(uploadId), error }); } let info; @@ -206,14 +261,32 @@ export async function finalizeResumableUpload(ctx: GridFSContext, uploadId: Obje export async function abortResumableUpload(ctx: GridFSContext, uploadId: ObjectId): Promise { assertObjectId(uploadId, 'abortResumableUpload', 'uploadId'); + const uploads = ctx.db.collection(uploadsCollectionName(ctx.bucketName)); + let session: GridFSUploadSession | null; + try { + session = await uploads.findOne({ _id: uploadId }) as unknown as GridFSUploadSession | null; + } catch (error) { + throw mapDriverError(error, 'abortResumableUpload', ctx, { uploadId }); + } + if (!session) { + // A TTL-expired session can leave chunks. Take the same exclusive ID + // claim as all writers before deleting an orphan. + await cleanupOrphanById(ctx, uploadId); + return; + } + if (session.kind && session.kind !== 'resumable') throw uploadBusy('abortResumableUpload', ctx, uploadId); + if (session.status === 'finalized') return; + if (session.status !== 'pending') throw uploadBusy('abortResumableUpload', ctx, uploadId); + const transitioned = await uploads.updateOne( + { _id: uploadId, status: 'pending', activeAppend: { $exists: false } }, + { $set: { status: 'aborting', updatedAt: new Date() }, $unset: { expiresAt: '' } }, + ); + if (!transitioned.matchedCount) throw uploadBusy('abortResumableUpload', ctx, uploadId); + // Keep the ID locked during both cleanup steps. A failed delete leaves an + // aborting record that prevents a second writer from claiming this ID. try { - const uploads = ctx.db.collection(uploadsCollectionName(ctx.bucketName)); - const session = await uploads.findOne({ _id: uploadId }); - if (session?.status === 'finalized') return; - // A missing session (expired by the TTL, or aborted while an append was in flight) can still leave - // chunks; clear them as long as no file is visible under this id. await removeInvisibleChunks(ctx, uploadId, 'abortResumableUpload'); - if (session) await uploads.deleteOne({ _id: uploadId }); + await uploads.deleteOne({ _id: uploadId, status: 'aborting' }); } catch (error) { throw mapDriverError(error, 'abortResumableUpload', ctx, { uploadId }); } @@ -233,7 +306,26 @@ async function committedFile(ctx: GridFSContext, uploadId: ObjectId, session: Gr if (info.metadata?.sha256 !== session.sha256 || info.length !== session.length) { throw fileExists('finalizeResumableUpload', ctx, { id: uploadId, filename: session.filename, sha256: session.sha256 }); } - await ctx.db.collection(uploadsCollectionName(ctx.bucketName)).deleteOne({ _id: uploadId, status: 'pending' }); + // Metadata-only SHA can be forged by uploadStream(). Never treat it as + // proof of a previous resumable commit without verifying actual chunks. + const hash = createHash('sha256'); + let total = 0; + let index = 0; + for await (const chunk of ctx.db.collection(chunksCollectionName(ctx.bucketName)) + .find({ files_id: uploadId }).sort({ n: 1 })) { + const bytes = binaryToBuffer(chunk.data, 'finalizeResumableUpload'); + const expectedSize = chunkLengthAt(index, session.length, session.chunkSizeBytes); + if (chunk.n !== index || bytes.length !== expectedSize) { + throw integrityMismatch('finalizeResumableUpload', ctx.bucketName, { uploadId, expected: expectedSize, actual: bytes.length }); + } + hash.update(bytes); + total += bytes.length; + index += 1; + } + if (index !== chunkCount(session.length, session.chunkSizeBytes) || total !== session.length || hash.digest('hex') !== session.sha256) { + throw integrityMismatch('finalizeResumableUpload', ctx.bucketName, { uploadId, expected: session.sha256, actual: 'committed chunks do not match' }); + } + await ctx.db.collection(uploadsCollectionName(ctx.bucketName)).deleteOne({ _id: uploadId, status: 'finalizing' }); return info; } catch (error) { // GridFS domain errors (the conflict above) pass through unchanged. @@ -280,9 +372,9 @@ function planChunkSlices(payload: Buffer, fromChunk: number, length: number, chu return slices; } -async function requirePending(ctx: GridFSContext, uploadId: ObjectId, operation: string): Promise { +async function requireSession(ctx: GridFSContext, uploadId: ObjectId, operation: string): Promise { const doc = await ctx.db.collection(uploadsCollectionName(ctx.bucketName)).findOne({ _id: uploadId }); - if (!doc || doc.status !== 'pending') throw fileNotFound(operation, ctx, { id: uploadId }); + if (!doc || (doc.kind && doc.kind !== 'resumable')) throw fileNotFound(operation, ctx, { id: uploadId }); return doc as unknown as GridFSUploadSession; } diff --git a/src/capabilities/gridfs/types.ts b/src/capabilities/gridfs/types.ts index 8ee48f2..cbc6321 100644 --- a/src/capabilities/gridfs/types.ts +++ b/src/capabilities/gridfs/types.ts @@ -82,7 +82,10 @@ export interface GridFSUploadSession { chunkSizeBytes: number; metadata?: Record; contentType?: string; - status: 'pending' | 'aborted'; + kind?: 'resumable'; + status: 'pending' | 'finalizing' | 'aborting' | 'finalized' | 'aborted'; + activeAppend?: { token: string; startedAt: Date }; + expiresAt?: Date; nextChunk: number; createdAt: Date; updatedAt: Date; diff --git a/src/core/errors/index.ts b/src/core/errors/index.ts index 0cfac1a..d0142f9 100644 --- a/src/core/errors/index.ts +++ b/src/core/errors/index.ts @@ -38,6 +38,7 @@ export const ErrorCodes = { GRIDFS_UPLOAD_INCOMPLETE: 'GRIDFS_UPLOAD_INCOMPLETE', GRIDFS_RESUME_MISMATCH: 'GRIDFS_RESUME_MISMATCH', GRIDFS_RANGE_INVALID: 'GRIDFS_RANGE_INVALID', + GRIDFS_UPLOAD_BUSY: 'GRIDFS_UPLOAD_BUSY', /** Relation select is outside the supported inclusion-only top-level contract. */ INVALID_RELATION_SELECT: 'INVALID_RELATION_SELECT', /** A protected delete found declared Model relations that still reference its target. */ diff --git a/test/integration/gridfs/gridfs-ownership-regressions.test.ts b/test/integration/gridfs/gridfs-ownership-regressions.test.ts new file mode 100644 index 0000000..1cf3a9c --- /dev/null +++ b/test/integration/gridfs/gridfs-ownership-regressions.test.ts @@ -0,0 +1,373 @@ +import { createHash } from 'node:crypto'; +import { Readable } from 'node:stream'; +import { after, before, describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { Collection, MongoClient, ObjectId, type Db } from 'mongodb'; +import { createMemoryServerBootstrap } from '../../bootstrap/memory-server'; + +const MonSQLize = require('../../../dist/cjs/index.cjs'); + +function sha(data: Buffer): string { + return createHash('sha256').update(data).digest('hex'); +} + + +/** + * Deterministic regression tests for MZ-15 / MZ-16 / MZ-18. + * Each test uses a collection-operation gate to force the previously unsafe + * interleaving and asserts that a shared per-id ownership claim prevents damage. + */ +describe('GridFS MZ-15 / MZ-16 / MZ-18 ownership regressions', () => { + const bootstrap = createMemoryServerBootstrap(); + let uri = ''; + let raw: MongoClient; + + before(async () => { + uri = (await bootstrap.setup()).uri; + raw = new MongoClient(uri); + await raw.connect(); + }); + + after(async () => { + await raw.close(); + await bootstrap.teardown(); + MonSQLize.Model._clear(); + }); + + async function withRuntime(databaseName: string, fn: (runtime: any, db: Db) => Promise): Promise { + const runtime = new MonSQLize({ type: 'mongodb', databaseName, config: { uri } }); + await runtime.connect(); + try { + await fn(runtime, raw.db(databaseName)); + } finally { + await runtime.close(); + await raw.db(databaseName).dropDatabase(); + } + } + + it('MZ-16 blocks uploadStream claiming a pending resumable id and finalize trusting metadata SHA', { timeout: 30_000 }, async () => { + await withRuntime('gridfs_mz16_stream_id', async (runtime, db) => { + const bucket = runtime.gridfs('b', { chunkSizeBytes: 4 }); + const id = new ObjectId(); + const expected = Buffer.from('GOOD'); + const actual = Buffer.from('EVIL'); + const expectedSha = sha(expected); + + await bucket.beginResumableUpload('expected.bin', { + id, + length: expected.length, + sha256: expectedSha, + }); + + const writable = bucket.uploadStream('actual.bin', { id, sha256: expectedSha }); + Readable.from([actual]).pipe(writable); + await assert.rejects(writable.done, (error: unknown) => + (error as { code?: string }).code === 'GRIDFS_UPLOAD_BUSY'); + assert.equal(await bucket.getFile(id), null); + assert.equal(await db.collection('b.uploads').countDocuments({ _id: id, status: 'pending' }), 1); + + await bucket.appendChunks(id, 0, expected); + const finalized = await bucket.finalizeResumableUpload(id); + assert.equal(finalized.metadata?.sha256, expectedSha); + assert.deepEqual(await bucket.download(id), expected); + assert.equal(await db.collection('b.uploads').countDocuments({ _id: id }), 0); + }); + }); + + it('MZ-16 blocks beginResumableUpload deleting chunks from an in-flight ordinary upload', { timeout: 30_000 }, async () => { + await withRuntime('gridfs_mz16_begin_race', async (runtime, db) => { + const bucket = runtime.gridfs('b', { chunkSizeBytes: 4 }); + const id = new ObjectId(); + const payload = Buffer.from('AAAABBBB'); + const digest = sha(payload); + + const original = Collection.prototype.insertOne; + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + let reached!: () => void; + const hit = new Promise((resolve) => { reached = resolve; }); + let armed = true; + + (Collection.prototype as any).insertOne = async function (this: Collection, ...args: unknown[]) { + const document = args[0] as Record; + const filesId = document.files_id as ObjectId | undefined; + if (armed && this.collectionName === 'b.chunks' && filesId?.equals(id) && document.n === 1) { + armed = false; + reached(); + await gate; + } + return (original as any).apply(this, args); + }; + + let uploading: Promise | undefined; + try { + // Chunk 0 is inserted; chunk 1 is held before its insert. + uploading = bucket.upload('ordinary.bin', payload, { id, sha256: digest }); + await hit; + + // The direct upload holds the ID even before its files document is visible. + await assert.rejects(bucket.beginResumableUpload('resumable.bin', { + id, length: payload.length, sha256: digest, + }), (error: unknown) => (error as { code?: string }).code === 'GRIDFS_FILE_EXISTS'); + + release(); + const uploaded = await uploading; + assert.equal(String((uploaded as { id: ObjectId }).id), String(id)); + const chunks = await db.collection('b.chunks') + .find({ files_id: id }).sort({ n: 1 }).toArray(); + assert.deepEqual(chunks.map((chunk) => chunk.n), [0, 1]); + assert.deepEqual(await bucket.download(id), payload); + assert.equal(await db.collection('b.uploads').countDocuments({ _id: id }), 0); + } finally { + release(); + Collection.prototype.insertOne = original; + if (uploading) await uploading.catch(() => undefined); + } + }); + }); + + it('MZ-15 prevents abort deleting chunks while finalize is paused before files insert', { timeout: 30_000 }, async () => { + await withRuntime('gridfs_mz15_finalize_abort_race', async (runtime, db) => { + const bucket = runtime.gridfs('b', { chunkSizeBytes: 4 }); + const payload = Buffer.from('AAAABBBB'); + const session = await bucket.beginResumableUpload('race.bin', { + length: payload.length, + sha256: sha(payload), + }); + await bucket.appendChunks(session.uploadId, 0, payload); + + const original = Collection.prototype.insertOne; + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + let reached!: () => void; + const hit = new Promise((resolve) => { reached = resolve; }); + let armed = true; + + (Collection.prototype as any).insertOne = async function (this: Collection, ...args: unknown[]) { + const document = args[0] as Record; + const fileId = document._id as ObjectId | undefined; + if (armed && this.collectionName === 'b.files' && fileId?.equals(session.uploadId)) { + armed = false; + reached(); + await gate; + } + return (original as any).apply(this, args); + }; + + let finalizing: Promise | undefined; + try { + // finalize has already read the session, validated every chunk, and + // computed the expected SHA when it reaches this blocked files insert. + finalizing = bucket.finalizeResumableUpload(session.uploadId); + await hit; + + // Finalize has already atomically entered 'finalizing'. Abort must + // reject rather than delete the chunks needed for the commit. + await assert.rejects(bucket.abortResumableUpload(session.uploadId), + (error: unknown) => (error as { code?: string }).code === 'GRIDFS_UPLOAD_BUSY'); + + release(); + const finalized = await finalizing as { id: ObjectId }; + assert.equal(String(finalized.id), String(session.uploadId)); + assert.equal(await db.collection('b.files').countDocuments({ _id: session.uploadId }), 1); + assert.equal(await db.collection('b.uploads').countDocuments({ _id: session.uploadId }), 0); + assert.equal(await db.collection('b.chunks').countDocuments({ files_id: session.uploadId }), 2); + assert.deepEqual(await bucket.download(session.uploadId), payload); + } finally { + release(); + Collection.prototype.insertOne = original; + if (finalizing) await finalizing.catch(() => undefined); + } + }); + }); + + it('MZ-18 prevents delete removing chunks from a replacement upload with the same id', { timeout: 30_000 }, async () => { + await withRuntime('gridfs_mz18_delete_upload_race', async (runtime, db) => { + const bucket = runtime.gridfs('b', { chunkSizeBytes: 4 }); + const id = new ObjectId(); + + // An empty file has no old chunks, so a replacement upload can write + // its first chunk after delete removes the files document. + const empty = await bucket.upload('empty.bin', Buffer.alloc(0), { id }); + assert.equal(empty.length, 0); + assert.equal(await db.collection('b.chunks').countDocuments({ files_id: id }), 0); + + const original = Collection.prototype.deleteMany; + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + let reached!: () => void; + const hit = new Promise((resolve) => { reached = resolve; }); + let armed = true; + + (Collection.prototype as any).deleteMany = async function (this: Collection, ...args: unknown[]) { + const filter = args[0] as Record; + const filesId = filter.files_id as ObjectId | undefined; + if (armed && this.collectionName === 'b.chunks' && filesId?.equals(id)) { + armed = false; + reached(); + await gate; + } + return (original as any).apply(this, args); + }; + + let deleting: Promise | undefined; + try { + // GridFSBucket.delete removes files first, then deletes chunks. + deleting = bucket.delete(id); + await hit; + assert.equal(await bucket.getFile(id), null); + + // The delete owns the ID until both files and chunks have gone. + const payload = Buffer.from('NEW!'); + await assert.rejects(bucket.upload('replacement.bin', payload, { + id, sha256: sha(payload), + }), (error: unknown) => (error as { code?: string }).code === 'GRIDFS_UPLOAD_BUSY'); + + release(); + await deleting; + + // Reuse is safe only after delete has finished its chunk cleanup. + const replacement = await bucket.upload('replacement.bin', payload, { + id, sha256: sha(payload), + }); + assert.equal(String(replacement.id), String(id)); + assert.deepEqual(await bucket.download(id), payload); + assert.equal(await db.collection('b.chunks').countDocuments({ files_id: id }), 1); + } finally { + release(); + Collection.prototype.deleteMany = original; + if (deleting) await deleting.catch(() => undefined); + } + }); + }); + + + it('MZ-04 migrates legacy createdAt TTL to seven-day idle expiresAt', { timeout: 30_000 }, async () => { + await withRuntime('gridfs_mz04_migrate', async (runtime, db) => { + const id = new ObjectId(); + const createdAt = new Date(Date.now() - 2 * 86400_000); + const updatedAt = new Date(Date.now() - 30 * 60_000); + await db.collection('b.uploads').insertOne({ + _id: id, filename: 'legacy.bin', length: 4, + sha256: sha(Buffer.from('GOOD')), chunkSizeBytes: 4, + status: 'pending', nextChunk: 0, createdAt, updatedAt, + }); + await db.collection('b.uploads').createIndex({ createdAt: 1 }, { expireAfterSeconds: 604800 }); + const bucket = runtime.gridfs('b', { chunkSizeBytes: 4 }); + await bucket.ensureIndexes(); + const indexes = await db.collection('b.uploads').indexes(); + assert.equal(indexes.some((i) => i.name === 'createdAt_1'), false); + assert.equal(indexes.find((i) => i.name === 'expiresAt_1')?.expireAfterSeconds, 0); + const migrated = await db.collection('b.uploads').findOne({ _id: id }); + assert.ok(migrated?.expiresAt instanceof Date); + assert.ok(Math.abs(migrated.expiresAt.getTime() - (updatedAt.getTime() + 604800_000)) < 1000); + await bucket.abortResumableUpload(id); + }); + }); + + it('MZ-04 does not reinstall TTL while an append is actively writing', { timeout: 30_000 }, async () => { + await withRuntime('gridfs_mz04_active_append', async (runtime, db) => { + const bucket = runtime.gridfs('b', { chunkSizeBytes: 4 }); + const id = (await bucket.beginResumableUpload('active.bin', { + length: 4, sha256: sha(Buffer.from('GOOD')), + })).uploadId; + const original = Collection.prototype.insertOne; + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + let reached!: () => void; + const hit = new Promise((resolve) => { reached = resolve; }); + let armed = true; + (Collection.prototype as any).insertOne = async function (this: Collection, ...args: unknown[]) { + const doc = args[0] as { files_id?: ObjectId }; + if (armed && this.collectionName === 'b.chunks' && doc.files_id?.equals(id)) { + armed = false; + reached(); + await gate; + } + return (original as any).apply(this, args); + }; + let appending: Promise | undefined; + try { + appending = bucket.appendChunks(id, 0, Buffer.from('GOOD')); + await hit; + const active = await db.collection('b.uploads').findOne({ _id: id }); + assert.ok(active?.activeAppend); + assert.equal(active?.expiresAt, undefined); + await runtime.gridfs('b').ensureIndexes(); + const during = await db.collection('b.uploads').findOne({ _id: id }); + assert.equal(during?.expiresAt, undefined, 'index initialization must not re-arm expiry during append'); + release(); + await appending; + const finished = await db.collection('b.uploads').findOne({ _id: id }); + assert.equal(finished?.activeAppend, undefined); + assert.ok(finished?.expiresAt instanceof Date); + await bucket.abortResumableUpload(id); + } finally { + release(); + Collection.prototype.insertOne = original; + if (appending) await appending.catch(() => undefined); + } + }); + }); + + it('MZ-15 blocks finalize while append owns the session', { timeout: 30_000 }, async () => { + await withRuntime('gridfs_mz15_append_finalize', async (runtime) => { + const bucket = runtime.gridfs('b', { chunkSizeBytes: 4 }); + const payload = Buffer.from('AAAABBBB'); + const id = (await bucket.beginResumableUpload('append.bin', { + length: payload.length, sha256: sha(payload), + })).uploadId; + const original = Collection.prototype.insertOne; + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + let reached!: () => void; + const hit = new Promise((resolve) => { reached = resolve; }); + let armed = true; + (Collection.prototype as any).insertOne = async function (this: Collection, ...args: unknown[]) { + const doc = args[0] as { files_id?: ObjectId }; + if (armed && this.collectionName === 'b.chunks' && doc.files_id?.equals(id)) { + armed = false; + reached(); + await gate; + } + return (original as any).apply(this, args); + }; + let appending: Promise | undefined; + try { + appending = bucket.appendChunks(id, 0, payload); + await hit; + await assert.rejects(bucket.finalizeResumableUpload(id), + (error: unknown) => (error as { code?: string }).code === 'GRIDFS_UPLOAD_BUSY'); + release(); + await appending; + await bucket.finalizeResumableUpload(id); + assert.deepEqual(await bucket.download(id), payload); + } finally { + release(); + Collection.prototype.insertOne = original; + if (appending) await appending.catch(() => undefined); + } + }); + }); + + it('MZ-16 rechecks stored chunk bytes before accepting an ambiguous legacy finalize retry', { timeout: 30_000 }, async () => { + await withRuntime('gridfs_mz16_bad_legacy_hash', async (runtime, db) => { + const bucket = runtime.gridfs('b', { chunkSizeBytes: 4 }); + const id = new ObjectId(); + const actual = Buffer.from('EVIL'); + const expected = Buffer.from('GOOD'); + await bucket.upload('actual.bin', actual, { id, sha256: sha(actual) }); + await db.collection('b.files').updateOne({ _id: id }, { $set: { 'metadata.sha256': sha(expected) } }); + await db.collection('b.uploads').insertOne({ + _id: id, filename: 'expected.bin', length: expected.length, + sha256: sha(expected), chunkSizeBytes: 4, + status: 'pending', nextChunk: 1, + createdAt: new Date(), updatedAt: new Date(), + }); + await assert.rejects(bucket.finalizeResumableUpload(id), + (error: unknown) => (error as { code?: string }).code === 'GRIDFS_INTEGRITY_MISMATCH'); + assert.deepEqual(await bucket.download(id), actual); + }); + }); + +}); diff --git a/test/integration/gridfs/gridfs-scan-regressions.test.ts b/test/integration/gridfs/gridfs-scan-regressions.test.ts index 7fb7e7c..34ded73 100644 --- a/test/integration/gridfs/gridfs-scan-regressions.test.ts +++ b/test/integration/gridfs/gridfs-scan-regressions.test.ts @@ -99,9 +99,12 @@ describe('gridfs scan regressions (2026-11-02)', () => { try { const appending = bucket.appendChunks(session.uploadId, 0, data); await hit; - await bucket.abortResumableUpload(session.uploadId); + // Abort cannot delete chunks while append owns the upload. + await assert.rejects(bucket.abortResumableUpload(session.uploadId), + (error: unknown) => codeOf(error) === 'GRIDFS_UPLOAD_BUSY'); release(); - await assert.rejects(appending, (error: unknown) => codeOf(error) === 'GRIDFS_FILE_NOT_FOUND'); + await appending; + await bucket.abortResumableUpload(session.uploadId); } finally { Collection.prototype.insertOne = original; } @@ -117,14 +120,14 @@ describe('gridfs scan regressions (2026-11-02)', () => { await bucket.drop(); await bucket.ensureIndexes(); assert.deepEqual(await indexNames(db, 'b.chunks'), ['_id_', 'files_id_1_n_1']); - assert.deepEqual(await indexNames(db, 'b.uploads'), ['_id_', 'createdAt_1']); + assert.deepEqual(await indexNames(db, 'b.uploads'), ['_id_', 'expiresAt_1']); }); }); it('fails the uploadStream Writable when it fails before the driver stream opens', { timeout: 30_000 }, async () => { await withRuntime('gridfs_scan_preopen', async (runtime, db) => { // A conflicting TTL index makes ensureIndexes fail before any driver stream exists. - await db.collection('b.uploads').createIndex({ createdAt: 1 }, { expireAfterSeconds: 60 }); + await db.collection('b.uploads').createIndex({ expiresAt: 1 }, { expireAfterSeconds: 60 }); const bucket = runtime.gridfs('b'); const writable = bucket.uploadStream('x.bin'); const errored = new Promise((resolve) => writable.once('error', resolve)); diff --git a/test/integration/gridfs/gridfs-stale-recovery.test.ts b/test/integration/gridfs/gridfs-stale-recovery.test.ts new file mode 100644 index 0000000..4a19ad8 --- /dev/null +++ b/test/integration/gridfs/gridfs-stale-recovery.test.ts @@ -0,0 +1,437 @@ +import { createHash } from 'node:crypto'; +import { after, before, describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { Collection, MongoClient, ObjectId, type Db } from 'mongodb'; +import { createMemoryServerBootstrap } from '../../bootstrap/memory-server'; +import { releaseOwner } from '../../../src/capabilities/gridfs/ownership'; + +const MonSQLize = require('../../../dist/cjs/index.cjs'); +const { recoverGridFSOwnership } = require(process.cwd() + '/scripts/maintenance/gridfs-recover.cjs'); + +function digest(data: Buffer): string { + return createHash('sha256').update(data).digest('hex'); +} + +describe('GridFS operator recovery: deterministic post-crash state and chunk safety', () => { + const bootstrap = createMemoryServerBootstrap(); + let raw: MongoClient; + let uri: string; + let counter = 0; + + before(async () => { + uri = (await bootstrap.setup()).uri; + raw = new MongoClient(uri); + await raw.connect(); + }); + after(async () => { + await raw.close(); + await bootstrap.teardown(); + MonSQLize.Model._clear(); + }); + + async function withRuntime(fn: (bucket: any, db: Db) => Promise): Promise { + const dbName = 'gridfs_recovery_' + (++counter); + const runtime = new MonSQLize({ type: 'mongodb', databaseName: dbName, config: { uri } }); + await runtime.connect(); + try { + await fn(runtime.gridfs('b', { chunkSizeBytes: 4 }), raw.db(dbName)); + } finally { + await runtime.close(); + await raw.db(dbName).dropDatabase(); + } + } + + async function recover(db: Db, id: ObjectId, overrides: Record = {}) { + const doc = await db.collection('b.uploads').findOne({ _id: id }); + assert.ok(doc, 'recovery owner must exist'); + return recoverGridFSOwnership({ + db, bucket: 'b', id, + expectedStatus: doc.status, + expectedUpdatedAt: doc.updatedAt.toISOString(), + expectedToken: doc.recovery?.token || doc.ownerToken || doc.activeAppend?.token || null, + operator: 'integration-test', reason: 'deterministic process quiescence simulation', + quiesced: true, apply: true, ...overrides, + }); + } + + it('is read-only by default and rejects apply without explicit quiescence', async () => { + await withRuntime(async (_, db) => { + const id = new ObjectId(); + const now = new Date(); + await db.collection('b.uploads').insertOne({ + _id: id, kind: 'direct', status: 'uploading', ownerToken: 'direct-1', + createdAt: now, updatedAt: now, + }); + await db.collection('b.chunks').insertOne({ files_id: id, n: 0, data: Buffer.from('ABCD') }); + const snapshot = await db.collection('b.uploads').findOne({ _id: id }); + const dry = await recover(db, id, { apply: false, quiesced: false }); + assert.equal(dry.outcome, 'dry_run'); + assert.equal(await db.collection('b.chunks').countDocuments({ files_id: id }), 1); + await assert.rejects(recover(db, id, { quiesced: false }), + /external writer quiescence/); + await assert.rejects(recover(db, id, { expectedToken: 'incorrect' }), + /Owner token changed/); + await assert.rejects(recover(db, id, { expectedUpdatedAt: '2020-01-01T00:00:00.000Z' }), + /updatedAt changed/); + assert.deepEqual(await db.collection('b.uploads').findOne({ _id: id }), snapshot); + await recover(db, id); + assert.equal(await db.collection('b.uploads').countDocuments({ _id: id }), 0); + assert.equal(await db.collection('b.chunks').countDocuments({ files_id: id }), 0); + assert.equal(await db.collection('b.ownership_recovery_audit').countDocuments({ outcome: 'completed' }), 1); + }); + }); + + it('preserves committed direct upload bytes while releasing a stranded owner', async () => { + await withRuntime(async (bucket, db) => { + const id = new ObjectId(); + const data = Buffer.from('GOODGOOD'); + await bucket.upload('committed.bin', data, { id, sha256: digest(data) }); + const now = new Date(); + await db.collection('b.uploads').insertOne({ + _id: id, kind: 'direct', status: 'uploading', ownerToken: 'post-commit', + createdAt: now, updatedAt: now, + }); + await recover(db, id); + assert.equal(await db.collection('b.uploads').countDocuments({ _id: id }), 0); + assert.equal(await db.collection('b.chunks').countDocuments({ files_id: id }), 2); + assert.deepEqual(await bucket.download(id), data); + }); + }); + + it('completes interrupted delete without exposing replacement chunks', async () => { + await withRuntime(async (bucket, db) => { + const id = new ObjectId(); + await bucket.upload('old.bin', Buffer.from('ABCD'), { id }); + const now = new Date(); + await db.collection('b.uploads').insertOne({ + _id: id, kind: 'delete', status: 'deleting', ownerToken: 'delete-1', + createdAt: now, updatedAt: now, + }); + await db.collection('b.files').deleteOne({ _id: id }); // failure after files deletion + await recover(db, id); + assert.equal(await db.collection('b.chunks').countDocuments({ files_id: id }), 0); + assert.equal(await db.collection('b.uploads').countDocuments({ _id: id }), 0); + await bucket.upload('replacement.bin', Buffer.from('NEW!'), { id }); + assert.deepEqual(await bucket.download(id), Buffer.from('NEW!')); + }); + }); + + it('restores stale activeAppend with a reconciled contiguous nextChunk and expiry', async () => { + await withRuntime(async (bucket, db) => { + const body = Buffer.from('AAAABBBB'); + const id = (await bucket.beginResumableUpload('append.bin', + { length: body.length, sha256: digest(body) })).uploadId; + await db.collection('b.chunks').insertOne({ files_id: id, n: 0, data: Buffer.from('AAAA') }); + await db.collection('b.uploads').updateOne({ _id: id }, { + $set: { activeAppend: { token: 'append-crash', startedAt: new Date() }, updatedAt: new Date() }, + $unset: { expiresAt: '' }, + }); + await recover(db, id); + const session = await db.collection('b.uploads').findOne({ _id: id }); + assert.equal(session?.status, 'pending'); + assert.equal(session?.nextChunk, 1); + assert.equal(session?.activeAppend, undefined); + assert.ok(session?.expiresAt instanceof Date); + await bucket.appendChunks(id, 1, Buffer.from('BBBB')); + await bucket.finalizeResumableUpload(id); + assert.deepEqual(await bucket.download(id), body); + }); + }); + + it('keeps interrupted append fenced when an existing chunk is malformed', async () => { + await withRuntime(async (bucket, db) => { + const body = Buffer.from('AAAABBBB'); + const id = (await bucket.beginResumableUpload('corrupt-append.bin', + { length: body.length, sha256: digest(body) })).uploadId; + await db.collection('b.chunks').insertOne({ + files_id: id, n: 0, data: Buffer.from('A'), + }); + await db.collection('b.uploads').updateOne({ _id: id }, { + $set: { activeAppend: { token: 'corrupt-append', startedAt: new Date() }, + updatedAt: new Date() }, + $unset: { expiresAt: '' }, + }); + await assert.rejects(recover(db, id), /malformed chunk/); + const owner = await db.collection('b.uploads').findOne({ _id: id }); + assert.equal(owner?.status, 'recovering'); + assert.equal(await db.collection('b.chunks').countDocuments({ files_id: id }), 1); + assert.equal((await db.collection('b.chunks').findOne({ files_id: id }))?.n, 0); + }); + }); + + it('recovers pre-commit finalizing without deleting verified chunks', async () => { + await withRuntime(async (bucket, db) => { + const body = Buffer.from('AAAABBBB'); + const id = (await bucket.beginResumableUpload('finalize.bin', + { length: body.length, sha256: digest(body) })).uploadId; + await bucket.appendChunks(id, 0, body); + await db.collection('b.uploads').updateOne({ _id: id }, { + $set: { status: 'finalizing', updatedAt: new Date() }, + $unset: { expiresAt: '' }, + }); + await recover(db, id); + const doc = await db.collection('b.uploads').findOne({ _id: id }); + assert.equal(doc?.status, 'pending'); + assert.equal(await db.collection('b.chunks').countDocuments({ files_id: id }), 2); + await bucket.finalizeResumableUpload(id); + assert.deepEqual(await bucket.download(id), body); + }); + }); + + it('recovers post-commit finalizing only after rehash of actual visible bytes', async () => { + await withRuntime(async (bucket, db) => { + const body = Buffer.from('GOOD'); + const hash = digest(body); + const id = (await bucket.beginResumableUpload('done.bin', + { length: body.length, sha256: hash })).uploadId; + await bucket.appendChunks(id, 0, body); + await db.collection('b.uploads').updateOne({ _id: id }, { + $set: { status: 'finalizing', updatedAt: new Date() }, + $unset: { expiresAt: '' }, + }); + await db.collection('b.files').insertOne({ + _id: id, filename: 'done.bin', length: 4, chunkSize: 4, + uploadDate: new Date(), metadata: { sha256: hash }, + }); + await recover(db, id); + assert.equal(await db.collection('b.uploads').countDocuments({ _id: id }), 0); + assert.deepEqual(await bucket.download(id), body); + }); + }); + + it('fails closed on forged metadata SHA and leaves chunks/ownership untouched', async () => { + await withRuntime(async (bucket, db) => { + const body = Buffer.from('GOOD'); + const id = (await bucket.beginResumableUpload('wrong.bin', + { length: body.length, sha256: digest(body) })).uploadId; + await bucket.appendChunks(id, 0, Buffer.from('EVIL')); + await db.collection('b.uploads').updateOne({ _id: id }, { + $set: { status: 'finalizing', updatedAt: new Date() }, + $unset: { expiresAt: '' }, + }); + await db.collection('b.files').insertOne({ + _id: id, filename: 'wrong.bin', length: 4, chunkSize: 4, + uploadDate: new Date(), metadata: { sha256: digest(body) }, + }); + await assert.rejects(recover(db, id), /visible bytes mismatch/); + assert.equal(await db.collection('b.chunks').countDocuments({ files_id: id }), 1); + const owner = await db.collection('b.uploads').findOne({ _id: id }); + assert.equal(owner?.status, 'recovering', 'failed recovery remains fenced'); + assert.ok(owner?.recovery?.token); + assert.equal(await db.collection('b.ownership_recovery_audit').countDocuments({ outcome: 'blocked' }), 1); + }); + }); + + it('completes aborting and is idempotent when the prior owner is gone', async () => { + await withRuntime(async (bucket, db) => { + const body = Buffer.from('GOOD'); + const id = (await bucket.beginResumableUpload('aborting.bin', + { length: body.length, sha256: digest(body) })).uploadId; + await bucket.appendChunks(id, 0, body); + await db.collection('b.uploads').updateOne({ _id: id }, { + $set: { status: 'aborting', updatedAt: new Date() }, + $unset: { expiresAt: '' }, + }); + const receipt = await recover(db, id); + assert.equal(receipt.outcome, 'completed'); + assert.equal(await db.collection('b.chunks').countDocuments({ files_id: id }), 0); + assert.equal((await recoverGridFSOwnership({ + db, bucket: 'b', id, expectedStatus: 'aborting', + expectedUpdatedAt: new Date().toISOString(), + })).outcome, 'no_owner'); + await bucket.beginResumableUpload('new.bin', { id, length: 4, sha256: digest(body) }); + }); + }); + + it('can resume a recovery crash using the persistent recovery token', async () => { + await withRuntime(async (_, db) => { + const id = new ObjectId(); + const now = new Date(); + await db.collection('b.uploads').insertOne({ + _id: id, kind: 'cleanup', status: 'recovering', + ownerToken: 'old-claim', createdAt: now, updatedAt: now, + recovery: { token: 'recovery-claim', fromStatus: 'cleaning', operator: 'original' }, + }); + await db.collection('b.chunks').insertOne({ files_id: id, n: 0, data: Buffer.from('ABCD') }); + await recover(db, id, { resumeQuiesced: true }); + assert.equal(await db.collection('b.chunks').countDocuments({ files_id: id }), 0); + assert.equal(await db.collection('b.uploads').countDocuments({ _id: id }), 0); + }); + }); + + it('rejects corrupt recovery origin metadata rather than unlocking the ID', async () => { + await withRuntime(async (_, db) => { + const id = new ObjectId(); + const now = new Date(); + await db.collection('b.uploads').insertOne({ + _id: id, kind: 'delete', status: 'recovering', ownerToken: 'old', + createdAt: now, updatedAt: now, + recovery: { token: 'recovery-token', fromStatus: 'uploading' }, + }); + await db.collection('b.chunks').insertOne({ + files_id: id, n: 0, data: Buffer.from('GOOD'), + }); + await assert.rejects(recover(db, id), /Inconsistent or unsupported owner origin/); + assert.equal(await db.collection('b.uploads').countDocuments({ _id: id }), 1); + assert.equal(await db.collection('b.chunks').countDocuments({ files_id: id }), 1); + }); + }); + + it('refuses to clear aborting ownership if a visible file already owns the chunks', async () => { + await withRuntime(async (bucket, db) => { + const id = new ObjectId(); + await bucket.upload('visible.bin', Buffer.from('GOOD'), { id }); + const now = new Date(); + await db.collection('b.uploads').insertOne({ + _id: id, kind: 'resumable', status: 'aborting', + filename: 'bad.bin', length: 4, chunkSizeBytes: 4, sha256: digest(Buffer.from('GOOD')), + createdAt: now, updatedAt: now, + }); + await assert.rejects(recover(db, id), /Visible file conflicts/); + assert.equal(await db.collection('b.chunks').countDocuments({ files_id: id }), 1); + assert.equal(await db.collection('b.uploads').countDocuments({ _id: id }), 1); + }); + }); + + it('requires proof the previous recovery process stopped before resuming', async () => { + await withRuntime(async (_, db) => { + const id = new ObjectId(); + const now = new Date(); + await db.collection('b.uploads').insertOne({ + _id: id, kind: 'cleanup', status: 'recovering', + ownerToken: 'original', createdAt: now, updatedAt: now, + recovery: { token: 'old-recovery', fromStatus: 'cleaning' }, + }); + await db.collection('b.chunks').insertOne({ files_id: id, n: 0, data: Buffer.from('GOOD') }); + await assert.rejects(recover(db, id), /verified shutdown of the prior recovery executor/); + assert.equal(await db.collection('b.chunks').countDocuments({ files_id: id }), 1); + assert.equal(await db.collection('b.uploads').countDocuments({ _id: id }), 1); + }); + }); + + it('atomically rejects two recovery resumptions holding the same snapshot', { timeout: 30_000 }, async () => { + await withRuntime(async (bucket, db) => { + const id = new ObjectId(); + const now = new Date(); + await db.collection('b.uploads').insertOne({ + _id: id, kind: 'cleanup', status: 'recovering', + ownerToken: 'original', createdAt: now, updatedAt: now, + recovery: { token: 'old-recovery', fromStatus: 'cleaning' }, + }); + await db.collection('b.chunks').insertOne({ files_id: id, n: 0, data: Buffer.from('GOOD') }); + const original = Collection.prototype.updateOne; + let ready!: () => void; + const bothReached = new Promise((resolve) => { ready = resolve; }); + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + let claims = 0; + (Collection.prototype as any).updateOne = async function (this: Collection, ...args: unknown[]) { + const filter = args[0] as Record; + if (this.collectionName === 'b.uploads' && filter.status === 'recovering' + && filter['recovery.token'] === 'old-recovery') { + claims += 1; + if (claims === 2) ready(); + await gate; + } + return (original as any).apply(this, args); + }; + try { + const input = { + db, bucket: 'b', id, + expectedStatus: 'recovering', expectedUpdatedAt: now.toISOString(), + expectedToken: 'old-recovery', quiesced: true, resumeQuiesced: true, + operator: 'integration-test', reason: 'verified prior executor stop', + apply: true, + }; + const first = recoverGridFSOwnership(input); + const second = recoverGridFSOwnership(input); + await bothReached; + release(); + const outcomes = await Promise.allSettled([first, second]); + assert.equal(outcomes.filter((x) => x.status === 'fulfilled').length, 1); + assert.equal(outcomes.filter((x) => x.status === 'rejected').length, 1); + assert.equal(await db.collection('b.ownership_recovery_audit').countDocuments({ outcome: 'completed' }), 1); + assert.equal(await db.collection('b.ownership_recovery_audit').countDocuments({ outcome: 'claim_conflict' }), 1); + assert.equal(await db.collection('b.uploads').countDocuments({ _id: id }), 0); + assert.equal(await db.collection('b.chunks').countDocuments({ files_id: id }), 0); + const payload = Buffer.from('NEW!'); + await bucket.upload('replacement.bin', payload, { id }); + assert.deepEqual(await bucket.download(id), payload); + } finally { + release(); + Collection.prototype.updateOne = original; + } + }); + }); + + + it('does not release a visible direct upload with inconsistent claimed SHA', async () => { + await withRuntime(async (bucket, db) => { + const id = new ObjectId(); + const actual = Buffer.from('EVIL'); + await bucket.upload('direct.bin', actual, { id }); + await db.collection('b.files').updateOne({ _id: id }, { + $set: { 'metadata.sha256': digest(Buffer.from('GOOD')) }, + }); + const now = new Date(); + await db.collection('b.uploads').insertOne({ + _id: id, kind: 'direct', status: 'uploading', ownerToken: 'crash', + createdAt: now, updatedAt: now, + }); + await assert.rejects(recover(db, id), /metadata SHA mismatch/); + assert.equal((await db.collection('b.uploads').findOne({ _id: id }))?.status, 'recovering'); + assert.deepEqual(await bucket.download(id), actual); + }); + }); + + it('releases a quiesced rename only after checking a visible complete file', async () => { + await withRuntime(async (bucket, db) => { + const id = new ObjectId(); + const payload = Buffer.from('GOODGOOD'); + await bucket.upload('already-renamed.bin', payload, { id }); + const now = new Date(); + await db.collection('b.uploads').insertOne({ + _id: id, kind: 'rename', status: 'renaming', ownerToken: 'rename-crash', + createdAt: now, updatedAt: now, + }); + await recover(db, id); + assert.equal(await db.collection('b.uploads').countDocuments({ _id: id }), 0); + assert.deepEqual(await bucket.download(id), payload); + }); + }); + + it('keeps a missing-file rename fenced pending manual inspection', async () => { + await withRuntime(async (_, db) => { + const id = new ObjectId(); + const now = new Date(); + await db.collection('b.uploads').insertOne({ + _id: id, kind: 'rename', status: 'renaming', ownerToken: 'rename-crash', + createdAt: now, updatedAt: now, + }); + await assert.rejects(recover(db, id), /no visible file/); + assert.equal((await db.collection('b.uploads').findOne({ _id: id }))?.status, 'recovering'); + }); + }); + + + it('does not let a stale original writer release an active recovery claim', async () => { + await withRuntime(async (_, db) => { + const id = new ObjectId(); + const now = new Date(); + await db.collection('b.uploads').insertOne({ + _id: id, kind: 'direct', status: 'recovering', + ownerToken: 'original-writer', createdAt: now, updatedAt: now, + recovery: { token: 'active-recovery', fromStatus: 'uploading' }, + }); + const ctx = { db, bucketName: 'b' } as any; + await assert.rejects( + releaseOwner(ctx, { id, kind: 'direct', token: 'original-writer' }, 'upload'), + (error: unknown) => (error as { code?: string }).code === 'GRIDFS_UPLOAD_BUSY', + ); + const owner = await db.collection('b.uploads').findOne({ _id: id }); + assert.equal(owner?.status, 'recovering'); + assert.equal(owner?.recovery?.token, 'active-recovery'); + }); + }); + +}); diff --git a/test/run-tests.cjs b/test/run-tests.cjs index 5cbf376..0627adf 100644 --- a/test/run-tests.cjs +++ b/test/run-tests.cjs @@ -137,6 +137,8 @@ const INTEGRATION = [ 'test/integration/search-indexes/unsupported.test.js', 'test/integration/gridfs/gridfs.test.js', 'test/integration/gridfs/gridfs-scan-regressions.test.js', + 'test/integration/gridfs/gridfs-ownership-regressions.test.js', + 'test/integration/gridfs/gridfs-stale-recovery.test.js', 'test/integration/mongodb/review-repair-regressions.test.js', 'test/integration/model/review-model-regressions.test.js', 'test/integration/slow-query-log/review-storage-regressions.test.js', diff --git a/test/unit/gridfs/gridfs.test.ts b/test/unit/gridfs/gridfs.test.ts index 5222487..2810bcf 100644 --- a/test/unit/gridfs/gridfs.test.ts +++ b/test/unit/gridfs/gridfs.test.ts @@ -40,6 +40,12 @@ function matches(doc: Doc, filter: Doc): boolean { if (expected && typeof expected === 'object' && '$gte' in (expected as Doc)) { return typeof actual === 'number' && actual >= (expected as { $gte: number }).$gte; } + if (expected && typeof expected === 'object' && '$exists' in (expected as Doc)) { + return (actual !== undefined) === (expected as { $exists: boolean }).$exists; + } + if (expected && typeof expected === 'object' && '$type' in (expected as Doc)) { + return (expected as { $type: string }).$type === 'date' && actual instanceof Date; + } return actual === expected; }); } @@ -101,16 +107,30 @@ class MemoryCollection { return { insertedId: doc._id }; } - async updateOne(filter: Doc, update: { $set?: Doc; $max?: Doc }) { + async updateOne(filter: Doc, update: { $set?: Doc; $max?: Doc; $unset?: Doc }) { const row = this.docs.find((doc) => matches(doc, filter)); if (!row) return { matchedCount: 0 }; Object.assign(row, update.$set ?? {}); + for (const key of Object.keys(update.$unset ?? {})) delete row[key]; for (const [key, value] of Object.entries(update.$max ?? {})) { if (row[key] === undefined || (value as number) > row[key]) row[key] = value; } return { matchedCount: 1 }; } + async updateMany(filter: Doc, update: Doc | Doc[]) { + const rows = this.docs.filter((doc) => matches(doc, filter)); + for (const row of rows) { + if (Array.isArray(update)) { + const start = row.updatedAt instanceof Date ? row.updatedAt : row.createdAt as Date; + row.expiresAt = new Date(start.getTime() + 604800 * 1000); + } else { + Object.assign(row, update.$set ?? {}); + } + } + return { matchedCount: rows.length }; + } + async deleteOne(filter: Doc) { const index = this.docs.findIndex((doc) => matches(doc, filter)); if (index < 0) return { deletedCount: 0 }; @@ -125,8 +145,15 @@ class MemoryCollection { } async createIndex(keys: Doc, options?: Doc) { - this.indexes.push({ key: keys, ...(options ?? {}) }); - return 'index'; + const name = Object.entries(keys).map(([key, direction]) => `${key}_${direction}`).join('_'); + this.indexes.push({ name, key: keys, ...(options ?? {}) }); + return name; + } + + async dropIndex(name: string) { + const index = this.indexes.findIndex((entry) => entry.name === name); + if (index < 0) throw Object.assign(new Error('index not found'), { code: 27 }); + this.indexes.splice(index, 1); } async drop() { @@ -418,7 +445,7 @@ describe('gridfs bucket', () => { await bucket.ensureIndexes(); const indexes = db.collection('fs.files').indexes.concat(db.collection('fs.chunks').indexes, db.collection('fs.uploads').indexes); assert.equal(indexes.length, 4); - assert.ok(indexes.some((index) => index.expireAfterSeconds === 604800)); + assert.ok(indexes.some((index) => index.expireAfterSeconds === 0)); assert.ok(indexes.some((index) => index.key?.['metadata.sha256'] === 1)); }); diff --git a/types/gridfs.d.ts b/types/gridfs.d.ts index b12812a..e412588 100644 --- a/types/gridfs.d.ts +++ b/types/gridfs.d.ts @@ -56,7 +56,7 @@ export interface GridFSUploadOptions { chunkSizeBytes?: number; } -/** `uploadStream` writes `sha256` into metadata and does not support `ifExists`. */ +/** `uploadStream` writes `sha256` into metadata without validating bytes, and does not support `ifExists`. */ export interface GridFSUploadStreamOptions extends Omit {} /** @@ -139,8 +139,8 @@ export interface MonSQLizeGridFSBucket { beginResumableUpload(filename: string, opts: BeginResumableUploadOptions): Promise; appendChunks(uploadId: ObjectId, fromChunk: number, data: Buffer | Uint8Array | Readable): Promise; finalizeResumableUpload(uploadId: ObjectId): Promise; - /** Deletes the session and any chunks. A missing or already aborted session resolves. */ + /** Aborts pending sessions only; an active append/finalize may reject with GRIDFS_UPLOAD_BUSY. */ abortResumableUpload(uploadId: ObjectId): Promise; - /** Creates the standard files/chunks indexes, `metadata.sha256`, and the uploads TTL index. */ + /** Creates files/chunks indexes, `metadata.sha256` and expiresAt TTL; migrates the legacy createdAt TTL. */ ensureIndexes(): Promise; }