diff --git a/packages/blob/package.json b/packages/blob/package.json index c4a88de98..236eb0ec5 100644 --- a/packages/blob/package.json +++ b/packages/blob/package.json @@ -60,7 +60,7 @@ "is-buffer": "^2.0.5", "is-node-process": "^1.2.0", "throttleit": "^2.1.0", - "undici": "^6.23.0" + "undici": "^6.27.0" }, "devDependencies": { "@edge-runtime/jest-environment": "4.0.0", diff --git a/packages/blob/src/index.node.test.ts b/packages/blob/src/index.node.test.ts index ee8a35b5d..ff9c425a2 100644 --- a/packages/blob/src/index.node.test.ts +++ b/packages/blob/src/index.node.test.ts @@ -17,6 +17,7 @@ import { rename, uploadPart, } from './index'; +import { getPartSizeInBytes, uploadAllParts } from './multipart/upload'; const BLOB_API_URL_AGENT = 'https://vercel.com'; const BLOB_STORE_BASE_URL = 'https://storeId.public.blob.vercel-storage.com'; @@ -1226,6 +1227,110 @@ describe('blob client', () => { }); }); + describe('multipart part size scaling', () => { + const mebibyte = 1024 * 1024; + const gibibyte = 1024 * mebibyte; + const tebibyte = 1024 * gibibyte; + // Vercel Blob enforces a maximum of 10,000 parts per multipart upload + const maxParts = 10_000; + + it('keeps the 8 MiB default part size for bodies at or under the 80 GiB ceiling', () => { + expect(getPartSizeInBytes(0)).toBe(8 * mebibyte); + expect(getPartSizeInBytes(1 * gibibyte)).toBe(8 * mebibyte); + expect(getPartSizeInBytes(maxParts * 8 * mebibyte)).toBe(8 * mebibyte); + }); + + it('scales the part size up for bodies above 80 GiB so the part count stays under the 10,000 limit', () => { + const totalToLoad = 100 * gibibyte; + const partSize = getPartSizeInBytes(totalToLoad); + + expect(partSize).toBeGreaterThan(8 * mebibyte); + expect(Math.ceil(totalToLoad / partSize)).toBeLessThanOrEqual(maxParts); + + // 5 TB is the maximum supported by the changelog (5TB file transfers) + const fiveTebibyte = 5 * tebibyte; + const fiveTebibytePartSize = getPartSizeInBytes(fiveTebibyte); + expect( + Math.ceil(fiveTebibyte / fiveTebibytePartSize), + ).toBeLessThanOrEqual(maxParts); + }); + + it('uploads one scaled part instead of many 8 MiB parts for a body above 80 GiB', async () => { + const totalToLoad = 100 * gibibyte; + const partSize = getPartSizeInBytes(totalToLoad); + const uploadedPartNumbers: number[] = []; + + mockClient + .intercept({ + path: () => true, + method: 'POST', + }) + .reply(200, (req) => { + uploadedPartNumbers.push( + Number( + (req.headers as Record)['x-mpu-part-number'], + ), + ); + return { etag: `etag-${uploadedPartNumbers.length}` }; + }) + .persist(); + + const stream = new Blob([ + new Uint8Array(partSize), + ]).stream() as unknown as ReadableStream; + + const parts = await uploadAllParts({ + uploadId: 'upload-123', + key: 'big.bin', + pathname: 'big.bin', + stream, + headers: {}, + options: { access: 'public' }, + totalToLoad, + }); + + expect(parts).toHaveLength(1); + expect(uploadedPartNumbers).toEqual([1]); + }); + + it('keeps splitting small bodies into 8 MiB parts', async () => { + const totalToLoad = 16 * mebibyte; + const uploadedPartNumbers: number[] = []; + + mockClient + .intercept({ + path: () => true, + method: 'POST', + }) + .reply(200, (req) => { + uploadedPartNumbers.push( + Number( + (req.headers as Record)['x-mpu-part-number'], + ), + ); + return { etag: `etag-${uploadedPartNumbers.length}` }; + }) + .persist(); + + const stream = new Blob([ + new Uint8Array(totalToLoad), + ]).stream() as unknown as ReadableStream; + + const parts = await uploadAllParts({ + uploadId: 'upload-123', + key: 'small.bin', + pathname: 'small.bin', + stream, + headers: {}, + options: { access: 'public' }, + totalToLoad, + }); + + expect(parts).toHaveLength(2); + expect(uploadedPartNumbers).toEqual([1, 2]); + }); + }); + describe('copy', () => { it('throws when filepath is too long', async () => { await expect( diff --git a/packages/blob/src/multipart/upload.ts b/packages/blob/src/multipart/upload.ts index 023e18fd4..1c8d388eb 100644 --- a/packages/blob/src/multipart/upload.ts +++ b/packages/blob/src/multipart/upload.ts @@ -135,9 +135,36 @@ export async function uploadPart({ const maxConcurrentUploads = typeof window !== 'undefined' ? 6 : 8; // 5MB is the minimum part size accepted by Vercel Blob, but we set our default part size to 8mb like the aws cli -const partSizeInBytes = 8 * 1024 * 1024; +const defaultPartSizeInBytes = 8 * 1024 * 1024; -const maxBytesInMemory = maxConcurrentUploads * partSizeInBytes * 2; +// Vercel Blob enforces a maximum of 10,000 parts per multipart upload +const maxPartsPerUpload = 10_000; + +// With the default part size, 10,000 parts cap uploads at 80 GiB. Bodies larger +// than that need bigger parts to stay within the part limit. +const maxDefaultPartSizeUploadBytes = + maxPartsPerUpload * defaultPartSizeInBytes; + +/** + * Returns the part size to use for a multipart upload of `totalToLoad` bytes. + * + * The Vercel Blob API rejects uploads with more than 10,000 parts, so bodies + * larger than 80 GiB (10,000 × 8 MiB) scale the part size up to keep the part + * count within the limit. Bodies with an unknown size (streams) keep the + * 8 MiB default. + */ +export function getPartSizeInBytes(totalToLoad: number): number { + if (totalToLoad <= maxDefaultPartSizeUploadBytes) { + return defaultPartSizeInBytes; + } + + return Math.ceil(totalToLoad / maxPartsPerUpload); +} + +// Bound read-ahead memory regardless of part size. Tying this to part size +// (as it used to be) would buffer gigabytes in memory for very large uploads +// whose part size is scaled up. +const maxBytesInMemory = maxConcurrentUploads * defaultPartSizeInBytes * 2; interface UploadPartApiResponse { etag: string; @@ -168,6 +195,7 @@ export function uploadAllParts({ }): Promise { debug('mpu: upload init', 'key:', key); const internalAbortController = new AbortController(); + const partSizeInBytes = getPartSizeInBytes(totalToLoad); return new Promise((resolve, reject) => { const partsToUpload: BlobUploadPart[] = []; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1b18ff93f..7f89223d0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -54,8 +54,8 @@ importers: specifier: ^2.1.0 version: 2.1.0 undici: - specifier: ^6.23.0 - version: 6.23.0 + specifier: ^6.27.0 + version: 6.28.0 devDependencies: '@edge-runtime/jest-environment': specifier: 4.0.0 @@ -3485,8 +3485,8 @@ packages: undici-types@7.16.0: resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} - undici@6.23.0: - resolution: {integrity: sha512-VfQPToRA5FZs/qJxLIinmU59u0r7LXqoJkCzinq3ckNJp3vKEh7jTWN589YQ5+aoAC/TGRLyJLCPKcLQbM8r9g==} + undici@6.28.0: + resolution: {integrity: sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==} engines: {node: '>=18.17'} undici@7.19.0: @@ -6889,7 +6889,7 @@ snapshots: undici-types@7.16.0: {} - undici@6.23.0: {} + undici@6.28.0: {} undici@7.19.0: {}