From 883870181c7302c669089e9fc95bdb59a7c09ff8 Mon Sep 17 00:00:00 2001 From: Nic Polumeyv <162764842+Nic-Polumeyv@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:31:32 -0400 Subject: [PATCH 1/3] compress each file from one read, a few files at a time --- .changeset/calm-files-compress.md | 5 +++ packages/kit/src/core/adapt/builder.js | 55 ++++++++++++-------------- 2 files changed, 31 insertions(+), 29 deletions(-) create mode 100644 .changeset/calm-files-compress.md diff --git a/.changeset/calm-files-compress.md b/.changeset/calm-files-compress.md new file mode 100644 index 000000000000..565fda9070d8 --- /dev/null +++ b/.changeset/calm-files-compress.md @@ -0,0 +1,5 @@ +--- +'@sveltejs/kit': patch +--- + +fix: keep memory flat when precompressing many files diff --git a/packages/kit/src/core/adapt/builder.js b/packages/kit/src/core/adapt/builder.js index 28973a91c79f..493ef0d491a8 100644 --- a/packages/kit/src/core/adapt/builder.js +++ b/packages/kit/src/core/adapt/builder.js @@ -5,16 +5,9 @@ /** @import { RouteData, ValidatedConfig, BuildData, ServerMetadata, ServerMetadataRoute, Prerendered, PrerenderMap, Logger, RemoteChunk } from 'types' */ import { loadEnv } from 'vite'; import * as devalue from 'devalue'; -import { - createReadStream, - createWriteStream, - existsSync, - mkdirSync, - rmSync, - statSync -} from 'node:fs'; +import { existsSync, mkdirSync, rmSync, statSync } from 'node:fs'; +import { readFile, writeFile } from 'node:fs/promises'; import path from 'node:path'; -import { pipeline } from 'node:stream'; import { promisify } from 'node:util'; import zlib from 'node:zlib'; import { copy, relative_path, walk } from '../../utils/filesystem.js'; @@ -29,7 +22,8 @@ import { handle_issues, validate } from '../../exports/internal/env.js'; import { get_mime_lookup } from '../utils.js'; import { lookup as mime_lookup } from '../../utils/mime.js'; -const pipe = promisify(pipeline); +const gzip = promisify(zlib.gzip); +const brotli = promisify(zlib.brotliCompress); const extensions = [ '.html', '.js', @@ -151,10 +145,14 @@ export function create_builder({ const files = [...walk(directory)].filter((file) => extensions.includes(path.extname(file))); + // zlib work is serialised on the threadpool and each brotli encoder is allocated up front, + // so a handful of files in flight is as fast as all of them and keeps memory flat + const queue = [...files]; await Promise.all( - files.flatMap((file) => { - const abs = path.resolve(directory, file); - return [compress_file(abs, 'gz'), compress_file(abs, 'br')]; + Array.from({ length: 16 }, async () => { + /** @type {string | undefined} */ + let file; + while ((file = queue.shift())) await compress_file(path.resolve(directory, file)); }) ); @@ -367,25 +365,24 @@ export function create_builder({ } /** + * Writes gzip and brotli variants next to `file` * @param {string} file - * @param {'gz' | 'br'} format */ -async function compress_file(file, format = 'gz') { - const compress = - format == 'br' - ? zlib.createBrotliCompress({ - params: { - [zlib.constants.BROTLI_PARAM_MODE]: zlib.constants.BROTLI_MODE_TEXT, - [zlib.constants.BROTLI_PARAM_QUALITY]: zlib.constants.BROTLI_MAX_QUALITY, - [zlib.constants.BROTLI_PARAM_SIZE_HINT]: statSync(file).size - } - }) - : zlib.createGzip({ level: zlib.constants.Z_BEST_COMPRESSION }); - - const source = createReadStream(file); - const destination = createWriteStream(`${file}.${format}`); +async function compress_file(file) { + const contents = await readFile(file); + + const [gz, br] = await Promise.all([ + gzip(contents, { level: zlib.constants.Z_BEST_COMPRESSION }), + brotli(contents, { + params: { + [zlib.constants.BROTLI_PARAM_MODE]: zlib.constants.BROTLI_MODE_TEXT, + [zlib.constants.BROTLI_PARAM_QUALITY]: zlib.constants.BROTLI_MAX_QUALITY, + [zlib.constants.BROTLI_PARAM_SIZE_HINT]: contents.length + } + }) + ]); - await pipe(source, compress, destination); + await Promise.all([writeFile(`${file}.gz`, gz), writeFile(`${file}.br`, br)]); } /** From 7c345383d1958d9d516fbdda6d373289941e66e3 Mon Sep 17 00:00:00 2001 From: Nic Polumeyv <162764842+Nic-Polumeyv@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:44:07 -0400 Subject: [PATCH 2/3] use the default fs import, walk the file list by index --- packages/kit/src/core/adapt/builder.js | 36 +++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/kit/src/core/adapt/builder.js b/packages/kit/src/core/adapt/builder.js index 493ef0d491a8..0d193c9cfcae 100644 --- a/packages/kit/src/core/adapt/builder.js +++ b/packages/kit/src/core/adapt/builder.js @@ -5,8 +5,7 @@ /** @import { RouteData, ValidatedConfig, BuildData, ServerMetadata, ServerMetadataRoute, Prerendered, PrerenderMap, Logger, RemoteChunk } from 'types' */ import { loadEnv } from 'vite'; import * as devalue from 'devalue'; -import { existsSync, mkdirSync, rmSync, statSync } from 'node:fs'; -import { readFile, writeFile } from 'node:fs/promises'; +import fs from 'node:fs'; import path from 'node:path'; import { promisify } from 'node:util'; import zlib from 'node:zlib'; @@ -109,8 +108,8 @@ export function create_builder({ return { log, - rimraf: (dir) => rmSync(dir, { force: true, recursive: true }), - mkdirp: (dir) => mkdirSync(dir, { recursive: true }), + rimraf: (dir) => fs.rmSync(dir, { force: true, recursive: true }), + mkdirp: (dir) => fs.mkdirSync(dir, { recursive: true }), copy, config, @@ -139,7 +138,7 @@ export function create_builder({ }, async compress(directory) { - if (!existsSync(directory)) { + if (!fs.existsSync(directory)) { return []; } @@ -147,12 +146,10 @@ export function create_builder({ // zlib work is serialised on the threadpool and each brotli encoder is allocated up front, // so a handful of files in flight is as fast as all of them and keeps memory flat - const queue = [...files]; + let i = 0; await Promise.all( Array.from({ length: 16 }, async () => { - /** @type {string | undefined} */ - let file; - while ((file = queue.shift())) await compress_file(path.resolve(directory, file)); + while (i < files.length) await compress_file(path.resolve(directory, files[i++])); }) ); @@ -179,7 +176,7 @@ export function create_builder({ assets: config.files.assets }); - if (existsSync(dest)) { + if (fs.existsSync(dest)) { log.warn( `\nOverwriting ${dest} with fallback page. Consider using a different name for the fallback.\n` ); @@ -305,7 +302,7 @@ export function create_builder({ }, hasServerInstrumentationFile() { - return existsSync(`${config.outDir}/output/server/instrumentation.server.js`); + return fs.existsSync(`${config.outDir}/output/server/instrumentation.server.js`); }, instrument({ @@ -317,24 +314,24 @@ export function create_builder({ exports: ['default'] } }) { - if (!existsSync(instrumentation)) { + if (!fs.existsSync(instrumentation)) { throw new Error( `Instrumentation file ${instrumentation} not found. This is probably a bug in your adapter.` ); } - if (!existsSync(entrypoint)) { + if (!fs.existsSync(entrypoint)) { throw new Error( `Entrypoint file ${entrypoint} not found. This is probably a bug in your adapter.` ); } - if (!existsSync(initializer)) { + if (!fs.existsSync(initializer)) { throw new Error( `Instrumentation initializer ${initializer} not found. This is probably a bug in your adapter.` ); } copy(entrypoint, start); - if (existsSync(`${entrypoint}.map`)) { + if (fs.existsSync(`${entrypoint}.map`)) { copy(`${entrypoint}.map`, `${start}.map`); } @@ -358,7 +355,7 @@ export function create_builder({ initializer: relative_initializer }); - rmSync(entrypoint, { force: true, recursive: true }); + fs.rmSync(entrypoint, { force: true, recursive: true }); write(entrypoint, facade); } }; @@ -369,7 +366,7 @@ export function create_builder({ * @param {string} file */ async function compress_file(file) { - const contents = await readFile(file); + const contents = await fs.promises.readFile(file); const [gz, br] = await Promise.all([ gzip(contents, { level: zlib.constants.Z_BEST_COMPRESSION }), @@ -382,7 +379,10 @@ async function compress_file(file) { }) ]); - await Promise.all([writeFile(`${file}.gz`, gz), writeFile(`${file}.br`, br)]); + await Promise.all([ + fs.promises.writeFile(`${file}.gz`, gz), + fs.promises.writeFile(`${file}.br`, br) + ]); } /** From 6a992b2f24123592ab81e3c77d5ee56bc4017db3 Mon Sep 17 00:00:00 2001 From: Nic Polumeyv <162764842+Nic-Polumeyv@users.noreply.github.com> Date: Thu, 17 Sep 2026 15:27:20 -0400 Subject: [PATCH 3/3] drop brotli params that do not change the output --- packages/kit/src/core/adapt/builder.js | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/packages/kit/src/core/adapt/builder.js b/packages/kit/src/core/adapt/builder.js index 0d193c9cfcae..89298891f535 100644 --- a/packages/kit/src/core/adapt/builder.js +++ b/packages/kit/src/core/adapt/builder.js @@ -370,13 +370,7 @@ async function compress_file(file) { const [gz, br] = await Promise.all([ gzip(contents, { level: zlib.constants.Z_BEST_COMPRESSION }), - brotli(contents, { - params: { - [zlib.constants.BROTLI_PARAM_MODE]: zlib.constants.BROTLI_MODE_TEXT, - [zlib.constants.BROTLI_PARAM_QUALITY]: zlib.constants.BROTLI_MAX_QUALITY, - [zlib.constants.BROTLI_PARAM_SIZE_HINT]: contents.length - } - }) + brotli(contents) ]); await Promise.all([