fix(blob): scale multipart part size for uploads above 80 GiB

packages/blob/package.json

@@ -60,7 +60,7 @@
     "is-buffer": "^2.0.5",
     "is-node-process": "^1.2.0",
     "throttleit": "^2.1.0",
-    "undici": "^6.23.0"
+    "undici": "^6.27.0"
   },
   "devDependencies": {
     "@edge-runtime/jest-environment": "4.0.0",

packages/blob/src/index.node.test.ts

@@ -17,6 +17,7 @@ import {
   rename,
   uploadPart,
 } from './index';
+import { getPartSizeInBytes, uploadAllParts } from './multipart/upload';
 
 const BLOB_API_URL_AGENT = 'https://vercel.com';
 const BLOB_STORE_BASE_URL = 'https://storeId.public.blob.vercel-storage.com';
@@ -1226,6 +1227,110 @@ describe('blob client', () => {
     });
   });
 
+  describe('multipart part size scaling', () => {
+    const mebibyte = 1024 * 1024;
+    const gibibyte = 1024 * mebibyte;
+    const tebibyte = 1024 * gibibyte;
+    // Vercel Blob enforces a maximum of 10,000 parts per multipart upload
+    const maxParts = 10_000;
+
+    it('keeps the 8 MiB default part size for bodies at or under the 80 GiB ceiling', () => {
+      expect(getPartSizeInBytes(0)).toBe(8 * mebibyte);
+      expect(getPartSizeInBytes(1 * gibibyte)).toBe(8 * mebibyte);
+      expect(getPartSizeInBytes(maxParts * 8 * mebibyte)).toBe(8 * mebibyte);
+    });
+
+    it('scales the part size up for bodies above 80 GiB so the part count stays under the 10,000 limit', () => {
+      const totalToLoad = 100 * gibibyte;
+      const partSize = getPartSizeInBytes(totalToLoad);
+
+      expect(partSize).toBeGreaterThan(8 * mebibyte);
+      expect(Math.ceil(totalToLoad / partSize)).toBeLessThanOrEqual(maxParts);
+
+      // 5 TB is the maximum supported by the changelog (5TB file transfers)
+      const fiveTebibyte = 5 * tebibyte;
+      const fiveTebibytePartSize = getPartSizeInBytes(fiveTebibyte);
+      expect(
+        Math.ceil(fiveTebibyte / fiveTebibytePartSize),
+      ).toBeLessThanOrEqual(maxParts);
+    });
+
+    it('uploads one scaled part instead of many 8 MiB parts for a body above 80 GiB', async () => {
+      const totalToLoad = 100 * gibibyte;
+      const partSize = getPartSizeInBytes(totalToLoad);
+      const uploadedPartNumbers: number[] = [];
+
+      mockClient
+        .intercept({
+          path: () => true,
+          method: 'POST',
+        })
+        .reply(200, (req) => {
+          uploadedPartNumbers.push(
+            Number(
+              (req.headers as Record<string, string>)['x-mpu-part-number'],
+            ),
+          );
+          return { etag: `etag-${uploadedPartNumbers.length}` };
+        })
+        .persist();
+
+      const stream = new Blob([
+        new Uint8Array(partSize),
+      ]).stream() as unknown as ReadableStream<ArrayBuffer>;
+
+      const parts = await uploadAllParts({
+        uploadId: 'upload-123',
+        key: 'big.bin',
+        pathname: 'big.bin',
+        stream,
+        headers: {},
+        options: { access: 'public' },
+        totalToLoad,
+      });
+
+      expect(parts).toHaveLength(1);
+      expect(uploadedPartNumbers).toEqual([1]);
+    });
+
+    it('keeps splitting small bodies into 8 MiB parts', async () => {
+      const totalToLoad = 16 * mebibyte;
+      const uploadedPartNumbers: number[] = [];
+
+      mockClient
+        .intercept({
+          path: () => true,
+          method: 'POST',
+        })
+        .reply(200, (req) => {
+          uploadedPartNumbers.push(
+            Number(
+              (req.headers as Record<string, string>)['x-mpu-part-number'],
+            ),
+          );
+          return { etag: `etag-${uploadedPartNumbers.length}` };
+        })
+        .persist();
+
+      const stream = new Blob([
+        new Uint8Array(totalToLoad),
+      ]).stream() as unknown as ReadableStream<ArrayBuffer>;
+
+      const parts = await uploadAllParts({
+        uploadId: 'upload-123',
+        key: 'small.bin',
+        pathname: 'small.bin',
+        stream,
+        headers: {},
+        options: { access: 'public' },
+        totalToLoad,
+      });
+
+      expect(parts).toHaveLength(2);
+      expect(uploadedPartNumbers).toEqual([1, 2]);
+    });
+  });
+
   describe('copy', () => {
     it('throws when filepath is too long', async () => {
       await expect(

packages/blob/src/multipart/upload.ts

@@ -135,9 +135,36 @@ export async function uploadPart({
 const maxConcurrentUploads = typeof window !== 'undefined' ? 6 : 8;
 
 // 5MB is the minimum part size accepted by Vercel Blob, but we set our default part size to 8mb like the aws cli
-const partSizeInBytes = 8 * 1024 * 1024;
+const defaultPartSizeInBytes = 8 * 1024 * 1024;
 
-const maxBytesInMemory = maxConcurrentUploads * partSizeInBytes * 2;
+// Vercel Blob enforces a maximum of 10,000 parts per multipart upload
+const maxPartsPerUpload = 10_000;
+
+// With the default part size, 10,000 parts cap uploads at 80 GiB. Bodies larger
+// than that need bigger parts to stay within the part limit.
+const maxDefaultPartSizeUploadBytes =
+  maxPartsPerUpload * defaultPartSizeInBytes;
+
+/**
+ * Returns the part size to use for a multipart upload of `totalToLoad` bytes.
+ *
+ * The Vercel Blob API rejects uploads with more than 10,000 parts, so bodies
+ * larger than 80 GiB (10,000 × 8 MiB) scale the part size up to keep the part
+ * count within the limit. Bodies with an unknown size (streams) keep the
+ * 8 MiB default.
+ */
+export function getPartSizeInBytes(totalToLoad: number): number {
+  if (totalToLoad <= maxDefaultPartSizeUploadBytes) {
+    return defaultPartSizeInBytes;
+  }
+
+  return Math.ceil(totalToLoad / maxPartsPerUpload);
+}
+
+// Bound read-ahead memory regardless of part size. Tying this to part size
+// (as it used to be) would buffer gigabytes in memory for very large uploads
+// whose part size is scaled up.
+const maxBytesInMemory = maxConcurrentUploads * defaultPartSizeInBytes * 2;
 
 interface UploadPartApiResponse {
   etag: string;
@@ -168,6 +195,7 @@ export function uploadAllParts({
 }): Promise<Part[]> {
   debug('mpu: upload init', 'key:', key);
   const internalAbortController = new AbortController();
+  const partSizeInBytes = getPartSizeInBytes(totalToLoad);
 
   return new Promise((resolve, reject) => {
     const partsToUpload: BlobUploadPart[] = [];