Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .changeset/fix-doc-kit-1201.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
---
'@doc-kit/core': patch
'@doc-kit/generator-react': patch
---

Fix custom HTML bundler configurations failing with DataCloneError
during parallel page generation by excluding main-thread-only
bundler functions from worker messages.
3 changes: 3 additions & 0 deletions packages/core/src/generators/types.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,9 @@ declare global {

hasParallelProcessor?: boolean;

/** Configuration keys excluded from worker-thread messages. */
workerConfigOmit?: string[];

/**
* The unified plugins this generator processes Markdown with, listed as
* in the `markdown` option, with paths relative to its module. Each list
Expand Down
45 changes: 45 additions & 0 deletions packages/core/src/threading/__tests__/parallel.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,51 @@ describe('createParallelWorker', () => {
deepStrictEqual(await result, [0, 1, 2, 3, 4]);
});

it('does not clone main-thread-only HTML bundler functions', async () => {
const htmlGenerator = await loadGenerator('html');
const transferred = [];
const pool = {
run(task) {
const copy = structuredClone(task);
transferred.push(copy);
return Promise.resolve(copy.input);
},
};

const bundler = {
buildServer: async () => 'file:///library.mjs',
buildClient: async () => ({}),
compile: async code => code,
};

const configuration = {
threads: 2,
chunkSize: 1,
html: {
title: 'Worker regression test',
bundler,
},
};

const worker = createParallelWorker(
'html',
htmlGenerator,
pool,
configuration
);

const results = await collectStream(worker.stream([1, 2, 3], {}));

deepStrictEqual(results, [1, 2, 3]);
strictEqual(transferred.length, 3);
for (const task of transferred) {
deepStrictEqual(task.configuration.html, {
title: 'Worker regression test',
});
}
strictEqual(configuration.html.bundler, bundler);
});

it('propagates a later worker failure without an unhandled rejection', async () => {
const tasks = [];
const pool = {
Expand Down
18 changes: 16 additions & 2 deletions packages/core/src/threading/parallel.mjs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
'use strict';

import logger from '#logger/index.mjs';
import { omitKeys } from '#utils/misc.mjs';

const parallelLogger = logger.child('parallel');

Expand Down Expand Up @@ -73,7 +74,13 @@ export default function createParallelWorker(
) {
const { threads, chunkSize } = configuration;

const { name } = generator;
const { name, workerConfigOmit } = generator;

// Main-thread-only options may contain functions that cannot be cloned.
const workerConfiguration = { ...configuration };
if (configuration[name]) {
workerConfiguration[name] = omitKeys(configuration[name], workerConfigOmit);
}

return {
/**
Expand Down Expand Up @@ -110,7 +117,14 @@ export default function createParallelWorker(
// unhandled rejection while earlier chunks are still running.
const tasks = chunks.map(indices => {
const task = pool.run(
createTask(items, indices, extra, configuration, specifier, name)
createTask(
items,
indices,
extra,
workerConfiguration,
specifier,
name
)
);

return task.then(
Expand Down
3 changes: 3 additions & 0 deletions packages/react/src/html/index.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -96,5 +96,8 @@ export default {
// page count, so it is farmed out to the worker pool.
hasParallelProcessor: true,

// The bundler runs on the main thread and contains non-cloneable functions.
workerConfigOmit: ['bundler'],
Comment thread
ovflowd marked this conversation as resolved.
Resolved

processChunk,
};
Loading