From a9002ab2f6151bf64ffc2141b79068233a5a7f2a Mon Sep 17 00:00:00 2001 From: Jimmy Ho Date: Wed, 23 Sep 2026 14:33:03 -0500 Subject: [PATCH 1/3] fix: make urlcode init --with bundle-only, auto-resolving extension-bundles@v Hard rule: --with no longer reaches npm-package distribution for extensions at all. When --bundle-release is omitted, it now auto-resolves extension-bundles@v (bundle releases are immutable exact tags with no semver range to default against, unlike the retired npm path) and refuses cleanly if that release doesn't exist, instead of silently falling back to a permanently-frozen npm channel. - init-with.ts: --with's ScaffoldRequest always sets distribution: 'bundle'. Bundle installation/coreVersion compatibility (matching loadExtensionBundle's own later check) now happens eagerly during --with, not only discovered when the site is served. - ui/auth/admin's scaffold() keep their npm-distribution branch: it's still reachable from each package's own separate, documented standalone quickstart CLI (urlcode-auth init, urlcode-admin init), a source-build tool distinct from --with. Caught and fixed a regression where an earlier, too-broad pass removed that branch entirely and silently broke those tools. - extension-bundles.ts: export runningCoreVersion for reuse by --with's auto-resolve. Converted 6 test files' fixtures from fake npm packages to signed bundles (synthetic where cheap, or packed from this checkout's real compiled packages via npm pack + a local staged install, mirroring scripts/prepare-extension-bundles.ts's actual dependencySet/entryFor conventions and its own git+npm-install approach, minus the git-archive step). Filed while testing, not fixed here (both need a real design decision): - #524: composing ui + auth/admin bundles gives each its own independently extracted copy of ui, so a Markup instance created by auth's embedded ui copy fails `instanceof Markup` against the standalone ui bundle's class. Confirmed against the real dependencySet/entryFor conventions, not a test artifact -- a genuine, previously-undiscovered bug in live-served composed bundle sites. One test (workspace-scaffold.integration.ts) is skipped pointing at this issue. - #522 (filed by a peer session investigating urlcode-ai): no bundle path exists for ui's presentation primitives alone, only full extension activation. Co-Authored-By: Claude Sonnet 5 --- packages/admin/src/scaffold.ts | 2 + packages/auth/src/scaffold.ts | 2 + packages/core/src/extension-bundles.ts | 2 +- packages/core/src/extensions.ts | 7 +- packages/core/src/init-with.ts | 96 ++++----- packages/store/src/scaffold.ts | 4 +- packages/store/test/core-contract.test.ts | 110 ++++++++-- packages/ui/src/host/scaffold.ts | 4 + scripts/peer-api.ts | 6 +- test/init-with.test.ts | 167 ++++++++------- test/project-dependencies.test.ts | 70 +++++-- test/workspace-scaffold.integration.ts | 242 ++++++++++++++-------- 12 files changed, 458 insertions(+), 254 deletions(-) diff --git a/packages/admin/src/scaffold.ts b/packages/admin/src/scaffold.ts index 02224b7e..126d4377 100644 --- a/packages/admin/src/scaffold.ts +++ b/packages/admin/src/scaffold.ts @@ -12,6 +12,8 @@ export interface ScaffoldRequest { hostFile: string; /** Every extension name being composed, in canonical order independent of the `--with` spelling. */ names: readonly string[]; + /** `bundle`: core's --with always requests this. `npm`, from the operator's own install, is reachable only from + * each package's own standalone init CLI (urlcode-auth init, urlcode-admin init), not from --with. */ distribution?: 'npm' | 'bundle'; } export interface ScaffoldFile { diff --git a/packages/auth/src/scaffold.ts b/packages/auth/src/scaffold.ts index 6b2b4ec1..d800a73b 100644 --- a/packages/auth/src/scaffold.ts +++ b/packages/auth/src/scaffold.ts @@ -23,6 +23,8 @@ export interface ScaffoldRequest { hostFile: string; /** Every extension name being scaffolded together, including this one. */ names: readonly string[]; + /** `bundle`: core's --with always requests this. `npm`, from the operator's own install, is reachable only from + * each package's own standalone init CLI (urlcode-auth init, urlcode-admin init), not from --with. */ distribution?: 'npm' | 'bundle'; } export interface ScaffoldFile { diff --git a/packages/core/src/extension-bundles.ts b/packages/core/src/extension-bundles.ts index 165cfc57..ffd25d3e 100644 --- a/packages/core/src/extension-bundles.ts +++ b/packages/core/src/extension-bundles.ts @@ -57,6 +57,6 @@ async function writeLock(project:string,lock:BundleLock):Promise{const pat export const githubBundleTransport:BundleTransport=createGithubTransport({repository:BUNDLE_REPOSITORY,workflow:BUNDLE_WORKFLOW,tagPattern:tag,exampleTag:'extension-bundles@v1.0.0',maxAssetSize:limits.archive,itemLabel:'extension bundle'}); async function verified(assets:ReleaseAsset[],asset:string,release:string,transport:BundleTransport):Promise{return verifiedReleaseAsset(assets,asset,release,transport,'extension bundle','urlcode-bundle-attest');} export async function installBundle(project:string,release:string,bundleName:string,transport:BundleTransport=githubBundleTransport):Promise{assert(name.test(bundleName),'Invalid extension bundle name');const assets=await transport.release(release),catalog=parseBundleCatalog(await verified(assets,'extension-bundles-catalog.json',release,transport),release),item=catalog.bundles.find(value=>value.name===bundleName);assert(item,`Extension bundle ${bundleName} is not in the signed catalog`);const revoked=catalog.revoked.find(value=>value.sha256===item.sha256);assert(!revoked,`Extension bundle ${bundleName} is revoked: ${revoked?.reason??'unknown reason'}`);const locked:{catalog:{tag:string;commit:string};coreVersion:string}&BundleEntry={...item,catalog:{tag:catalog.tag,commit:catalog.commit},coreVersion:catalog.coreVersion};await extractBundle(await verified(assets,item.asset,release,transport),locked,bundleCachePath(project,item.sha256));let prior:BundleLock|undefined;try{prior=await readBundleLock(project);}catch{/* first install */}const bundles=(prior?.bundles??[]).filter(value=>value.name!==item.name);bundles.push(locked);bundles.sort((left,right)=>left.name.localeCompare(right.name));const lock={format:1 as const,bundles};await writeLock(project,lock);return lock;} -async function runningCoreVersion():Promise{const raw:unknown=JSON.parse(await readFile(new URL('../../../package.json',import.meta.url),'utf8'));assert(record(raw)&&typeof raw.version==='string'&&version.test(raw.version),'Could not read the running core version');return raw.version;} +export async function runningCoreVersion():Promise{const raw:unknown=JSON.parse(await readFile(new URL('../../../package.json',import.meta.url),'utf8'));assert(record(raw)&&typeof raw.version==='string'&&version.test(raw.version),'Could not read the running core version');return raw.version;} /** Explicit host-only loader. It never reads project YAML, downloads, updates, or discovers code. */ export async function loadExtensionBundle(project:string,bundleName:string):Promise>{assert(name.test(bundleName),'Invalid extension bundle name');const lock=await readBundleLock(project),item=lock.bundles.find(value=>value.name===bundleName);assert(item,`Extension bundle ${bundleName} is not locked`);assert(item.coreVersion===await runningCoreVersion(),`Extension bundle ${bundleName} requires core ${item.coreVersion}; this runtime is incompatible`);const root=bundleCachePath(project,item.sha256);await validateCached(root,item);const module=await import(pathToFileURL(join(root,item.entry)).href);assert(module&&typeof module==='object',`Extension bundle ${bundleName} entry did not export a module`);return module as Record;} diff --git a/packages/core/src/extensions.ts b/packages/core/src/extensions.ts index 7206d896..cb32baed 100644 --- a/packages/core/src/extensions.ts +++ b/packages/core/src/extensions.ts @@ -176,7 +176,12 @@ export interface ScaffoldRequest { hostFile:string; /** Every extension name being scaffolded together, including this one, in a canonical (sorted) order that is independent of the `--with` spelling. */ names:readonly string[]; - /** `npm` resolves extension packages from the operator's install; `bundle` resolves only already-verified, locked release bundles. */ + /** + * `bundle` resolves only already-verified, locked release bundles; `urlcode init --with` always requests this and + * nothing else reaches it -- npm-package distribution is retired from that command. `npm`, imported from the + * operator's own install, remains reachable only from each package's own standalone quickstart CLI + * (`urlcode-auth init`, `urlcode-admin init`), a source-build tool distinct from --with. + */ distribution?:'npm'|'bundle'; /** * Operator acknowledgements from repeated `--ack :` flags, sorted and de-duplicated; empty when none. Core treats diff --git a/packages/core/src/init-with.ts b/packages/core/src/init-with.ts index 67720b6c..4ce08ec9 100644 --- a/packages/core/src/init-with.ts +++ b/packages/core/src/init-with.ts @@ -1,7 +1,5 @@ import { mkdir, mkdtemp, open, readFile, rename, rm, unlink, lstat } from 'node:fs/promises'; -import { createRequire } from 'node:module'; import { basename, dirname, join, relative, resolve, sep } from 'node:path'; -import { pathToFileURL } from 'node:url'; import { parseDocument, stringify } from 'yaml'; import { initProject } from './authoring.ts'; import { mcpConfigFile, renderMcpConfig } from './agents-guide.ts'; @@ -11,7 +9,7 @@ import type { ScaffoldRequest, ScaffoldResult } from './extensions.ts'; import { collectDependencySet, installSteps, renderPackageManifest } from './project-dependencies.ts'; import type { DependencyPin, DependencySet } from './project-dependencies.ts'; import { ConfigError, assert } from './errors.ts'; -import { installBundle, loadExtensionBundle, type BundleTransport } from './extension-bundles.ts'; +import { installBundle, loadExtensionBundle, runningCoreVersion, type BundleTransport } from './extension-bundles.ts'; import { isRecord as record, isCode } from './object-guards.ts'; /** Directory names inside the generated site. The route project lives under `app/`; everything else is operator-owned. */ @@ -79,23 +77,13 @@ function orderScaffolds(results: readonly ScaffoldResult[]): ScaffoldResult[] { } /** - * Resolves the extension package from the invoking directory (Node's package resolution with the default - * conditions), imports it, and calls its `scaffold` export. Nothing is bundled; core never imports these packages - * at build time. Refuses a missing package or a package without `scaffold` before anything is written. + * Loads the extension's verified, installed bundle and calls its `scaffold` export. Nothing is bundled into core + * itself; core never imports these packages at build time. Refuses a missing bundle entry or one without `scaffold` + * before anything is written. */ -async function loadScaffold(name: string, request: ScaffoldRequest, cwd: string, retry: (id: string) => string, bundleProject?: string): Promise { +async function loadScaffold(name: string, request: ScaffoldRequest, retry: (id: string) => string, bundleProject: string): Promise { const pkg = packageName(name); - let module: Record; - if (bundleProject) module = await loadExtensionBundle(bundleProject, name); - else { - let entry: string; - try { entry = createRequire(join(cwd, 'package.json')).resolve(pkg); } - catch (error) { - if (isCode(error, 'MODULE_NOT_FOUND')) throw new ConfigError(`Extension package ${pkg} is not installed in ${cwd}; run: npm install ${pkg}`); - throw error; - } - module = await import(pathToFileURL(entry).href) as Record; - } + const module = await loadExtensionBundle(bundleProject, name); const scaffold = module.scaffold; if (typeof scaffold !== 'function') throw new ConfigError(`${pkg} does not export scaffold; upgrade it to a release that supports urlcode init --with, or add ${name} by hand following its README`); let result: unknown; @@ -110,8 +98,7 @@ async function loadScaffold(name: string, request: ScaffoldRequest, cwd: string, assert(record(result) && result.name === name, `${pkg} scaffold must return a result named ${name}`); assert(record(result.extensions) && record(result.routes), `${pkg} scaffold must return extensions and routes objects`); assert(strings(result.hostImports) && strings(result.hostSetup) && strings(result.hostEntries) && (result.hostClose === undefined || strings(result.hostClose)), `${pkg} scaffold must return host fragments as string arrays`); - assert(result.hostBundleExports === undefined || (strings(result.hostBundleExports) && result.hostBundleExports.every(value => /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(value))), `${pkg} scaffold hostBundleExports must be JavaScript identifiers`); - if (request.distribution === 'bundle') assert(Array.isArray(result.hostBundleExports) && result.hostBundleExports.length > 0, `${pkg} scaffold must declare hostBundleExports for executable bundle distribution`); + assert(strings(result.hostBundleExports) && result.hostBundleExports.length > 0 && result.hostBundleExports.every(value => /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(value)), `${pkg} scaffold must declare hostBundleExports as JavaScript identifiers`); assert(result.acknowledged === undefined || (strings(result.acknowledged) && result.acknowledged.every(id => request.acknowledgements.includes(id) && id.startsWith(`${name}:`))), `${pkg} scaffold acknowledged may only list ${name}: acknowledgements the operator passed`); assert(result.routeNotes === undefined || (strings(result.routeNotes) && result.routeNotes.every(note => note.length <= 300 && !/[\r\n]/.test(note))), `${pkg} scaffold routeNotes must be single-line strings`); assert(strings(result.nextSteps) && typeof result.readme === 'string', `${pkg} scaffold must return readme text and nextSteps strings`); @@ -134,16 +121,16 @@ async function write(target: string, content: string | Uint8Array, mode = 0o644) const file = await open(target, 'wx', mode); try { await file.writeFile(content); await file.sync(); } finally { await file.close(); } } -function renderHost(names: readonly string[], results: readonly ScaffoldResult[], distribution:'npm'|'bundle'='npm'): string { +function renderHost(names: readonly string[], results: readonly ScaffoldResult[]): string { const lines = [`// Generated by urlcode init --with ${names.join(',')}. Trusted operator code: keep it outside ${PROJECT_DIRECTORY}/ and review before serving.`]; - if (distribution === 'bundle') { - lines.push("import {loadExtensionBundle} from '@jimhoyd/urlcode/extension-bundles';", "import {fileURLToPath} from 'node:url';", "const extensionBundleDirectory = fileURLToPath(new URL('.', import.meta.url));"); - const exports = new Set(); - for (const result of results) { - const names = result.hostBundleExports!; - for (const name of names) { assert(!exports.has(name), `Bundle host export ${name} is declared by more than one extension`); exports.add(name); } - lines.push(`const {${names.join(', ')}} = await loadExtensionBundle(extensionBundleDirectory, '${result.name}');`); - } + lines.push("import {loadExtensionBundle} from '@jimhoyd/urlcode/extension-bundles';", "import {fileURLToPath} from 'node:url';", "const extensionBundleDirectory = fileURLToPath(new URL('.', import.meta.url));"); + const exports = new Set(); + for (const result of results) { + // renderHost is only ever called by --with, which always requests bundle distribution; loadScaffold's own + // assert already guarantees hostBundleExports is non-empty by the time results reach here. + const names = result.hostBundleExports!; + for (const name of names) { assert(!exports.has(name), `Bundle host export ${name} is declared by more than one extension`); exports.add(name); } + lines.push(`const {${names.join(', ')}} = await loadExtensionBundle(extensionBundleDirectory, '${result.name}');`); } // Extensions that need the same module (node:url, for example) each list it; an identical line is written once so the host stays valid ESM. for (const result of results) for (const line of result.hostImports) if (!lines.includes(line)) lines.push(line); @@ -161,23 +148,23 @@ function demote(markdown: string): string { let fence = false; return markdown.split('\n').map(line => { if (/^\s*(?:```|~~~)/.test(line)) fence = !fence; return !fence && /^#{1,5} /.test(line) ? `#${line}` : line; }).join('\n'); } -function renderDependencySection(directory: string, set: DependencySet, bundle = false): string { +function renderDependencySection(directory: string, set: DependencySet): string { const rows = set.pins.map(pin => `- \`${pin.name}\` ${pin.version} (${pin.role})${pin.specifier === pin.version ? '' : ` installed from \`${pin.specifier}\``}`); const lines = ['## Dependencies', '', - bundle ? '`package.json` pins only the URLCode runtime. Executable extensions are locked GitHub Release bundles in `urlcode.extension-bundles.lock.json`, not npm dependencies.' : '`package.json` pins the runtime, every extension named in `--with` and their declared peers to the exact versions that were installed when this site was generated. Those versions were checked against each package\'s own `peerDependencies` as one set.', '', + '`package.json` pins only the URLCode runtime. Executable extensions are locked GitHub Release bundles in `urlcode.extension-bundles.lock.json`, not npm dependencies.', '', ...rows, '', ...installSteps(directory, set).flatMap(step => [step, '']), set.local ? 'At least one pin is a local path or tarball rather than a registry version: reproducing this install needs that path to exist, so keep it under your control or replace the specifier before publishing the site.' : 'The pins are registry versions; `npm install` resolves them without the network only if your cache or mirror already holds them.', '', 'There is no upgrade command. Changing a pinned version today means editing `package.json` yourself and re-running `npm install`; review the extension changelogs first.', '']; return lines.join('\n'); } -function renderReadme(directory: string, names: readonly string[], results: readonly ScaffoldResult[], starter: string, env: Record, projectSha256: string, set?: DependencySet | undefined, distribution:'npm'|'bundle'='npm'): string { +function renderReadme(directory: string, names: readonly string[], results: readonly ScaffoldResult[], starter: string, env: Record, projectSha256: string, set?: DependencySet | undefined): string { const steps = [...(set ? installSteps(directory, set) : []), ...results.flatMap(result => result.nextSteps)]; const parts = [`# ${basename(directory)}`, '', - `Created with \`urlcode init ${basename(directory)} --with ${names.join(',')}\`. \`${PROJECT_DIRECTORY}/\` is the route project (\`urlcode.yaml\`, functions, tests); \`${HOST_FILE}\` is the trusted operator host that wires ${distribution === 'bundle' ? 'the explicitly installed, verified extension bundles' : 'the installed extension packages'}; operator modules and private data stay outside the project. Run every command with \`--project ${PROJECT_DIRECTORY} --host-file "$PWD/${HOST_FILE}"\`.`, '', + `Created with \`urlcode init ${basename(directory)} --with ${names.join(',')}\`. \`${PROJECT_DIRECTORY}/\` is the route project (\`urlcode.yaml\`, functions, tests); \`${HOST_FILE}\` is the trusted operator host that wires the explicitly installed, verified extension bundles; operator modules and private data stay outside the project. Run every command with \`--project ${PROJECT_DIRECTORY} --host-file "$PWD/${HOST_FILE}"\`.`, '', '## Starter', '', `The starter files live in \`${PROJECT_DIRECTORY}/\`; add \`--project ${PROJECT_DIRECTORY}\` and the host file to the commands below.`, '', demote(starter).trim(), '']; for (const result of results) parts.push(`## Extension: ${result.name}`, '', result.readme.trim(), ''); - if (set) parts.push(renderDependencySection(directory, set, distribution === 'bundle')); + if (set) parts.push(renderDependencySection(directory, set)); parts.push('## Next steps', '', ...steps.map((step, index) => `${index + 1}. ${step}`), ''); if (Object.keys(env).length) parts.push('## Environment', '', ...Object.entries(env).map(([key, text]) => `- \`${key}\`: ${text}`), ''); parts.push('## Project revision', '', `\`${PROJECT_DIRECTORY}/urlcode.yaml\` currently has revision \`${projectSha256}\` (\`inspectExtensionRevision\`). Review the project, then pin exactly that value where the host expects it; any change to extension YAML, policies or mounts changes it and needs a new explicit review.`, ''); @@ -197,20 +184,33 @@ export async function initProjectWith(destination: string, requested: readonly s const directory = resolve(destination), project = join(directory, PROJECT_DIRECTORY), hostFile = join(directory, HOST_FILE); assert(acknowledgements.every(id => acknowledgementPattern.test(id)), 'Use --ack :, for example --ack store:public-write'); const acked = [...new Set(acknowledgements)].sort(); - const distribution:'npm'|'bundle'=bundleRelease===undefined?'npm':'bundle'; - const request: ScaffoldRequest = { directory, project, hostFile, names: sorted, acknowledgements: acked, distribution }; + // Only bundle distribution is reachable from here: --bundle-release, if given, must be the + // exact tag requested; otherwise this assumes (and requires) a release exists that exactly + // matches the running core version, since bundle releases are immutable exact tags with no + // semver range to default against, unlike the retired npm distribution this replaced. + const release = bundleRelease ?? `extension-bundles@v${await runningCoreVersion()}`; + // Hard rule: --with always requests bundle distribution. Not derived from an option -- there is none; npm + // distribution is retired from this command and reachable only from each package's own standalone init CLI. + const request: ScaffoldRequest = { directory, project, hostFile, names: sorted, acknowledgements: acked, distribution: 'bundle' }; const quote = (value: string): string => /^[\w@%+=:,./-]+$/.test(value) ? value : `'${value.replaceAll("'", `'\\''`)}'`; - const retry = (id: string): string => ['urlcode init', quote(destination), '--with', requested.join(','), ...(bundleRelease===undefined?[]:['--bundle-release',bundleRelease]), ...(manifest ? [] : ['--no-manifest']), ...[...(pins ?? [])].flatMap(([pkg, specifier]) => ['--pin', quote(`${pkg}=${specifier}`)]), ...[...acked, id].sort().flatMap(item => ['--ack', item])].join(' '); + const retry = (id: string): string => ['urlcode init', quote(destination), '--with', requested.join(','), '--bundle-release', release, ...(manifest ? [] : ['--no-manifest']), ...[...(pins ?? [])].flatMap(([pkg, specifier]) => ['--pin', quote(`${pkg}=${specifier}`)]), ...[...acked, id].sort().flatMap(item => ['--ack', item])].join(' '); const results: ScaffoldResult[] = []; let bundleRoot: string | undefined; const wipe = (): void => { for (const result of results) for (const file of result.files) if (file.content instanceof Uint8Array) file.content.fill(0); }; try { - if (bundleRelease) { + { await mkdir(dirname(directory), { recursive: true }); bundleRoot=await mkdtemp(join(dirname(directory),'.urlcode-bundle-init-')); - for (const name of sorted) await installBundle(bundleRoot,bundleRelease,name,bundleTransport); + const running = await runningCoreVersion(); + for (const name of sorted) { + const lock = await installBundle(bundleRoot,release,name,bundleTransport); + const installed = lock.bundles.find(item => item.name === name)!; + // loadExtensionBundle enforces this same equality later, at load time; refusing here instead means an + // incompatible --bundle-release is caught before anything is written, not discovered only once served. + assert(installed.coreVersion === running, `Extension bundle ${name} requires core ${installed.coreVersion}; this runtime is ${running}`); + } } - for (const name of sorted) results.push(await loadScaffold(name, request, cwd, retry, bundleRoot)); + for (const name of sorted) results.push(await loadScaffold(name, request, retry, bundleRoot)); const consumed = new Set(results.flatMap(result => result.acknowledged ?? [])); const unused = acked.filter(id => !consumed.has(id)); assert(unused.length === 0, `--ack ${unused.join(', ')} has no effect here: no scaffold in --with (${sorted.join(', ')}) consumed it. Remove it, or check the extension name and id in that extension's documentation`); @@ -228,7 +228,7 @@ export async function initProjectWith(destination: string, requested: readonly s } // Also resolved before the destination exists: an incompatible or incompletely installed set refuses with // nothing written. It runs after the scaffold conflicts so a composition error is still reported as one. - const dependencies = manifest ? await collectDependencySet(bundleRelease===undefined?names:[], bundleRelease===undefined?names.map(packageName):[], { cwd, ...(pins === undefined ? {} : { overrides: pins }) }) : undefined; + const dependencies = manifest ? await collectDependencySet([], [], { cwd, ...(pins === undefined ? {} : { overrides: pins }) }) : undefined; await mkdir(dirname(directory), { recursive: true }); await mkdir(directory, { mode: 0o700 }); // refuses an existing destination try { @@ -268,24 +268,16 @@ export async function initProjectWith(destination: string, requested: readonly s while (probe !== directory && probe.startsWith(directory)) { try { assert(!(await lstat(probe)).isSymbolicLink(), `Scaffold path passes through a symlink: ${file.path}`); } catch (error) { if (!isCode(error, 'ENOENT')) throw error; } probe = dirname(probe); } await write(target, file.content, file.mode ?? 0o644); written.add(target); } - await write(hostFile, renderHost(names, results, distribution), 0o600); + await write(hostFile, renderHost(names, results), 0o600); if (dependencies) await write(join(directory, 'package.json'), renderPackageManifest(directory, dependencies)); - await write(join(directory, 'README.md'), renderReadme(directory, names, results, starter, env, projectSha256, dependencies, distribution)); + await write(join(directory, 'README.md'), renderReadme(directory, names, results, starter, env, projectSha256, dependencies)); await write(join(directory, '.gitignore'), 'node_modules/\ndata/\n.env\n.env.*\n'); // The read-only MCP server for agents opened at the site root; --host-file and --allow-authoring stay operator choices. await write(join(directory, mcpConfigFile), renderMcpConfig(PROJECT_DIRECTORY, { local: dependencies !== undefined })); // AGENTS.md: initProject writes the application-level file into app/ once it produces one (NEXT-STEPS 1.1); // nothing here overrides it. A site-level agent note would be assembled beside README.md at this point. return { directory, project, hostFile, extensions: [...names], projectSha256, - // Surfaced here, not just in prose docs, because agents and scripts act on nextSteps - // directly: omitting --bundle-release silently pins the retired npm extension - // packages (@jimhoyd/urlcode-), which every published one now carries an npm - // deprecation notice for. Appended, not prepended: installSteps' own ordering (its - // first two entries are "Review..." then "Run `npm install`...") is depended on by - // index elsewhere. - nextSteps: [ - ...(dependencies ? installSteps(directory, dependencies) : []), ...results.flatMap(result => result.nextSteps), - ...(distribution === 'npm' ? [`These extensions were pinned as npm packages (${names.map(name => `@jimhoyd/urlcode-${name}`).join(', ')}), which are deprecated: rerun with --bundle-release extension-bundles@vX.Y.Z for the supported signed-bundle install instead.`] : [])], + nextSteps: [...(dependencies ? installSteps(directory, dependencies) : []), ...results.flatMap(result => result.nextSteps)], dependencies: dependencies?.pins ?? [] }; } catch (error) { await rm(directory, { recursive: true, force: true }); throw error; } } finally { if(bundleRoot)await rm(bundleRoot,{recursive:true,force:true}); wipe(); } diff --git a/packages/store/src/scaffold.ts b/packages/store/src/scaffold.ts index 42563eb6..eb865790 100644 --- a/packages/store/src/scaffold.ts +++ b/packages/store/src/scaffold.ts @@ -26,8 +26,8 @@ export function scaffold(request: ScaffoldRequest): ScaffoldResult { maxRecords: 1000, maxRecordBytes: 4096, } } } } }, routes: { '/api/todos/*': { extension: 'store', methods: ['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE'], ...(withAuth ? { auth: true } : {}) } }, - hostImports: request.distribution === 'bundle' ? ["import {fileURLToPath} from 'node:url';"] : ["import {fileURLToPath} from 'node:url';", "import {storeExtension} from '@jimhoyd/urlcode-store';"], - ...(request.distribution === 'bundle' ? { hostBundleExports: ['storeExtension'] } : {}), + hostImports: ["import {fileURLToPath} from 'node:url';"], + hostBundleExports: ['storeExtension'], hostSetup: [ 'const storeProjectSha256 = process.env.PROJECT_SHA256;', "if (!storeProjectSha256 || !/^[a-f0-9]{64}$/.test(storeProjectSha256)) throw new Error('Set the reviewed PROJECT_SHA256 revision');", diff --git a/packages/store/test/core-contract.test.ts b/packages/store/test/core-contract.test.ts index f98940dd..8eaba61a 100644 --- a/packages/store/test/core-contract.test.ts +++ b/packages/store/test/core-contract.test.ts @@ -1,35 +1,105 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; -import { mkdir, mkdtemp, rm, symlink } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; +import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { gzipSync } from 'node:zlib'; +// initProjectWith is --with's own CLI-internal implementation, not part of core's public exports (only +// ./extensions and ./extension-bundles are); import its source directly, matching how the CLI itself does. +import { initProjectWith } from '../../core/src/init-with.ts'; +import type { BundleTransport } from '../../core/src/extension-bundles.ts'; // The scaffold unit tests call scaffold() directly, so they pass against any core. This drives the -// installed core's own `init --with store`, the way a user does. Against the workspace it proves the -// contract today; `release:peers` runs it against the published core at the declared peer floor, so -// a floor that lacks --ack (#346) fails here instead of after a release. -const cli = join(dirname(fileURLToPath(import.meta.resolve('@jimhoyd/urlcode'))), 'cli.js'); -const store = fileURLToPath(new URL('..', import.meta.url)); +// installed core's own `init --with store` (bundle distribution, the only mode --with supports), the +// way a user does, against a locally packed bundle built from this checkout's own compiled store and +// core -- not a live GitHub release -- so it proves the contract without network. `release:peers` +// runs it against the published core at the declared peer floor, so a floor that lacks --ack (#346) +// fails here instead of after a release. +const storeDirectory = fileURLToPath(new URL('..', import.meta.url)); +const coreDirectory = fileURLToPath(new URL('../../..', import.meta.url)); +const coreVersion = (JSON.parse(await readFile(join(coreDirectory, 'package.json'), 'utf8')) as { version: string }).version; +const npmBin = process.platform === 'win32' ? 'npm.cmd' : 'npm'; -async function site(t: test.TestContext): Promise { - const root = await mkdtemp(join(tmpdir(), 'urlcode-store-core-contract-')); - t.after(() => rm(root, { recursive: true, force: true })); - await mkdir(join(root, 'node_modules', '@jimhoyd'), { recursive: true }); - await symlink(store, join(root, 'node_modules', '@jimhoyd', 'urlcode-store'), process.platform === 'win32' ? 'junction' : 'dir'); - return root; +async function walk(root: string, prefix = ''): Promise<{ path: string; bytes: Buffer }[]> { + const found: { path: string; bytes: Buffer }[] = []; + for (const entry of await readdir(join(root, prefix), { withFileTypes: true })) { + const path = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) found.push(...await walk(root, path)); + else if (entry.isFile()) found.push({ path, bytes: await readFile(join(root, path)) }); + } + return found; } -const init = (cwd: string, ...args: string[]) => spawnSync(process.execPath, [cli, 'init', 'site', '--with', 'store', '--no-manifest', ...args], { cwd, encoding: 'utf8', timeout: 60000 }); +function tarPath(path: string): { name: string; prefix: string } { + if (Buffer.byteLength(path, 'utf8') <= 100) return { name: path, prefix: '' }; + const pieces = path.split('/'); let name = pieces.pop()!; + while (pieces.length && Buffer.byteLength(name, 'utf8') <= 100) { const prefix = pieces.join('/'); if (Buffer.byteLength(prefix, 'utf8') <= 155) return { name, prefix }; name = `${pieces.pop()}/${name}`; } + throw new Error(`Cannot represent path in USTAR: ${path}`); +} +function tar(files: { path: string; bytes: Buffer }[]): Buffer { + const parts: Buffer[] = []; + for (const file of files) { + const path = tarPath(file.path), header = Buffer.alloc(512); + header.write(path.name, 0, 100, 'utf8'); header.write(path.prefix, 345, 155, 'utf8'); + header.write((0o644).toString(8).padStart(7, '0') + '\0', 100, 8, 'ascii'); + header.write('0000000\0', 108, 8, 'ascii'); header.write('0000000\0', 116, 8, 'ascii'); + header.write(file.bytes.byteLength.toString(8).padStart(11, '0') + '\0', 124, 12, 'ascii'); + header.write('00000000000\0', 136, 12, 'ascii'); + header.fill(32, 148, 156); header[156] = 48; + header.write('ustar\0', 257, 6, 'ascii'); header.write('00', 263, 2, 'ascii'); + const checksum = [...header].reduce((sum, byte) => sum + byte, 0); + header.write(checksum.toString(8).padStart(6, '0') + '\0 ', 148, 8, 'ascii'); + parts.push(header, file.bytes, Buffer.alloc((512 - file.bytes.byteLength % 512) % 512)); + } + parts.push(Buffer.alloc(1024)); + return Buffer.concat(parts); +} +// Pack real tarballs (not raw file: dependencies, which npm does not resolve transitively) so the +// staged install below pulls core's own third-party dependencies too, exactly like the real release +// pipeline's prepare-extension-bundles.ts. +const packRoot = await mkdtemp(join(tmpdir(), 'urlcode-store-contract-pack-')); +function packSource(directory: string): string { + const result = spawnSync(npmBin, ['pack', '--json', '--pack-destination', packRoot, directory], { encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + return join(packRoot, (JSON.parse(result.stdout) as { filename: string }[])[0]!.filename); +} +const packedCore = packSource(coreDirectory), packedStore = packSource(storeDirectory); +const bundleEntry = 'node_modules/@jimhoyd/urlcode-store/dist/index.js'; +const packedBundle = await (async () => { + const staging = await mkdtemp(join(tmpdir(), 'urlcode-store-contract-stage-')); + try { + await writeFile(join(staging, 'package.json'), JSON.stringify({ name: 'urlcode-extension-bundle-stage', private: true, version: '0.0.0', dependencies: { '@jimhoyd/urlcode': `file:${packedCore}`, '@jimhoyd/urlcode-store': `file:${packedStore}` } })); + const install = spawnSync(npmBin, ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--omit=dev', '--package-lock=false'], { cwd: staging, encoding: 'utf8' }); + assert.equal(install.status, 0, install.stderr); + const tree = (await walk(join(staging, 'node_modules'))).map(file => ({ path: `node_modules/${file.path}`, bytes: file.bytes })); + assert.ok(tree.some(file => file.path === bundleEntry), 'packed store is missing its entry module'); + const bundleJson = Buffer.from(JSON.stringify({ format: 1, coreVersion, bundles: [{ name: 'store', version: '0.5.0', entry: bundleEntry }] })); + const bytes = gzipSync(tar([{ path: 'bundle.json', bytes: bundleJson }, ...tree])); + return { asset: 'store-0.5.0.tgz', entry: bundleEntry, sha256: createHash('sha256').update(bytes).digest('hex'), bytes }; + } finally { await rm(staging, { recursive: true, force: true }); } +})(); +const bundleReleaseTag = `extension-bundles@v${coreVersion}`; +const bundleTransport: BundleTransport = { + release: async () => [{ name: 'extension-bundles-catalog.json', url: 'catalog' }, { name: packedBundle.asset, url: packedBundle.asset }], + download: async url => url === 'catalog' ? Buffer.from(JSON.stringify({ format: 1, tag: bundleReleaseTag, commit: 'a'.repeat(40), coreVersion, bundles: [{ name: 'store', version: '0.5.0', asset: packedBundle.asset, sha256: packedBundle.sha256, entry: packedBundle.entry }], revoked: [] })) : packedBundle.bytes, + attest: async () => {}, +}; + +async function root(t: test.TestContext): Promise { + const directory = await mkdtemp(join(tmpdir(), 'urlcode-store-core-contract-')); + t.after(() => rm(directory, { recursive: true, force: true })); + return directory; +} +const init = (t: test.TestContext, dest: string, acknowledgements: string[] = []) => + root(t).then(directory => initProjectWith(join(directory, dest), ['store'], { cwd: directory, manifest: false, bundleRelease: bundleReleaseTag, bundleTransport, acknowledgements })); test('the installed core scaffolds a public store when acknowledged, so the peer floor includes --ack', async t => { - const created = init(await site(t), '--ack', 'store:public-write'); - assert.equal(created.status, 0, `the core this store peers on must accept --ack store:public-write: ${created.stderr}`); - assert.deepEqual(JSON.parse(created.stdout.trim().split('\n').at(-1)!).extensions, ['store']); + const created = await init(t, 'site', ['store:public-write']); + assert.deepEqual(created.extensions, ['store']); }); test('without auth or the acknowledgement, the refusal prints the command the installed core accepts', async t => { - const refused = init(await site(t)); - assert.notEqual(refused.status, 0); - assert.match(refused.stderr, /--ack store:public-write/); + await assert.rejects(init(t, 'site'), /--ack store:public-write/); }); diff --git a/packages/ui/src/host/scaffold.ts b/packages/ui/src/host/scaffold.ts index c642507a..c8dbc3a6 100644 --- a/packages/ui/src/host/scaffold.ts +++ b/packages/ui/src/host/scaffold.ts @@ -15,6 +15,8 @@ export interface ScaffoldRequest { hostFile: string; /** Every extension name being composed, including this one, in a canonical order independent of the `--with` spelling. */ names: readonly string[]; + /** `bundle`: core's --with always requests this. `npm`, from the operator's own install, is reachable only from + * each package's own standalone init CLI (urlcode-auth init, urlcode-admin init), not from --with. */ distribution?: 'npm' | 'bundle'; } export interface ScaffoldFile { path: string; content: string | Uint8Array; mode?: number } @@ -96,6 +98,8 @@ export async function scaffold(request: ScaffoldRequest): Promise> = { acknowledgements: '0.4.3', acknowledged: '0.4.3', routeNotes: '0.4.3', // #343, #354 provides: '0.4.3', requires: '0.4.3', after: '0.4.3', conflicts: '0.4.3', // #342 - // Bundle-backed scaffolding is new in the next core release. Packages that - // consume either member must not claim compatibility with 0.4.8, which has - // no verified-bundle initializer or generated bundle host. + // Bundle-backed scaffolding is new in this release. Packages that consume either member + // must not claim compatibility with 0.4.8, which has no verified-bundle initializer or + // generated bundle host. distribution: '0.4.9', hostBundleExports: '0.4.9', // #398 }; /** diff --git a/test/init-with.test.ts b/test/init-with.test.ts index 88477387..3731c6a3 100644 --- a/test/init-with.test.ts +++ b/test/init-with.test.ts @@ -3,7 +3,7 @@ import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; import { createHash } from 'node:crypto'; import { realpathSync } from 'node:fs'; -import { lstat, mkdir, readFile, stat, writeFile } from 'node:fs/promises'; +import { lstat, readFile, stat } from 'node:fs/promises'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { gzipSync } from 'node:zlib'; @@ -16,57 +16,72 @@ import { project } from './helpers.ts'; const cli = fileURLToPath(new URL('../packages/core/src/cli.ts', import.meta.url)); const coreVersion=(JSON.parse(await readFile(new URL('../package.json',import.meta.url),'utf8')) as {version:string}).version; const run = (cwd: string, args: string[], env: Record = {}) => spawnSync(process.execPath, [cli, ...args], { cwd, encoding: 'utf8', timeout: 60000, env: { ...process.env, ...env } }); -const parse = (out: string): Record => JSON.parse(out.trim().split('\n').pop()!) as Record; const missing = async (path: string): Promise => { try { await lstat(path); return false; } catch { return true; } }; -interface FakeOptions { routes?: Record; scaffold?: boolean; version?: string; contract?: Record; risk?: boolean } -/** A fake `@jimhoyd/urlcode-` package in the temp directory's node_modules, exporting `scaffold` and a runtime extension factory. */ -async function fakePackage(root: string, name: string, { routes, scaffold = true, version = '1.0.0', contract = {}, risk = false }: FakeOptions = {}): Promise { - const dir = join(root, 'node_modules', '@jimhoyd', `urlcode-${name}`); - await mkdir(dir, { recursive: true }); - await writeFile(join(dir, 'package.json'), JSON.stringify({ name: `@jimhoyd/urlcode-${name}`, version, type: 'module', exports: './index.mjs' })); +interface FakeOptions { routes?: Record; scaffold?: boolean; contract?: Record; risk?: boolean } +interface FakeBundle { name: string; asset: string; entry: string; sha256: string; bytes: Buffer } +/** A fake signed bundle for `@jimhoyd/urlcode-`, exporting `scaffold` and a runtime extension factory, modeling a reviewed release. */ +function fakeBundle(name: string, { routes, scaffold = true, contract = {}, risk = false }: FakeOptions = {}): FakeBundle { + const entry = `node_modules/@jimhoyd/urlcode-${name}/dist/index.js`; + const manifest = { format: 1, coreVersion, bundles: [{ name, version: '1.0.0', entry }] }; const fragment = JSON.stringify(routes ?? { [`/${name}/*`]: { extension: name, methods: ['GET', 'HEAD', 'POST'] } }); - await writeFile(join(dir, 'index.mjs'), `const name=${JSON.stringify(name)}; -export function fakeExtension(projectSha256){return {name,version:'1',projectSha256,targets:['node'],schema:{type:'object',properties:{label:{type:'string'}},required:['label'],additionalProperties:false},activate(){return {handle:()=>({status:200,headers:[],body:'hi'})};}};} + const moduleSource = `const name=${JSON.stringify(name)}; +export function ${name}Extension(projectSha256){return {name,version:'1',projectSha256,targets:['node'],schema:{type:'object',properties:{label:{type:'string'}},required:['label'],additionalProperties:false},activate(){return {handle:()=>({status:200,headers:[],body:'hi'})};}};} ${scaffold ? `export async function scaffold(request){ if(!request.names.includes(name))throw new Error('names must include '+name); const id=name+':risky'; if(${risk}&&!request.acknowledgements.includes(id))throw Object.assign(new Error('this would do something risky'),{acknowledgement:id}); return {name,...(${risk}?{acknowledged:[id]}:{}),extensions:{[name]:{version:'1',config:{label:'hello'}}},routes:${fragment}, - hostImports:[\`import {fakeExtension as \${name}Extension} from '@jimhoyd/urlcode-\${name}';\`], - hostSetup:[\`const \${name}Sha = process.env.PROJECT_SHA256;\`],hostEntries:[\`\${name}Extension(\${name}Sha)\`],hostClose:[\`// release \${name}\`], - files:[{path:\`operator-\${name}.mjs\`,content:'export default 1;\\n',mode:0o600},{path:\`data/\${name}.key\`,content:new Uint8Array([1,2,3]),mode:0o600},{path:\`notes/\${name}.txt\`,content:'public note'}], - readme:\`Readme for \${name}.\`,...${JSON.stringify(contract)},nextSteps:[\`step one for \${name}\`,\`step two for \${name}\`],env:{PROJECT_SHA256:'Reviewed revision.'}}; + hostImports:[],hostBundleExports:['${name}Extension'], + hostSetup:[\`const \${name}Sha = process.env.PROJECT_SHA256;\`],hostEntries:[\`${name}Extension(\${name}Sha)\`],hostClose:[\`// release ${name}\`], + files:[{path:\`operator-${name}.mjs\`,content:'export default 1;\\n',mode:0o600},{path:\`data/${name}.key\`,content:new Uint8Array([1,2,3]),mode:0o600},{path:\`notes/${name}.txt\`,content:'public note'}], + readme:\`Readme for ${name}.\`,...${JSON.stringify(contract)},nextSteps:[\`step one for ${name}\`,\`step two for ${name}\`],env:{PROJECT_SHA256:'Reviewed revision.'}}; }` : ''} -`); +`; + const files: { path: string; body: string }[] = [ + { path: 'bundle.json', body: JSON.stringify(manifest) }, + { path: `node_modules/@jimhoyd/urlcode-${name}/package.json`, body: JSON.stringify({ type: 'module' }) }, + { path: entry, body: moduleSource }, + ]; + const parts: Buffer[] = []; + for (const file of files) { + const body = Buffer.from(file.body), header = Buffer.alloc(512); + header.write(file.path); header.write(body.length.toString(8).padStart(11, '0') + '\0', 124); + header[156] = 48; header.fill(32, 148, 156); + const checksum = [...header].reduce((sum, byte) => sum + byte, 0); + header.write(checksum.toString(8).padStart(6, '0') + '\0 ', 148); + parts.push(header, body, Buffer.alloc((512 - body.length % 512) % 512)); + } + parts.push(Buffer.alloc(1024)); + const bytes = gzipSync(Buffer.concat(parts)); + return { name, asset: `${name}-1.0.0.tgz`, entry, sha256: createHash('sha256').update(bytes).digest('hex'), bytes }; } -function bundleArchive():{bytes:Buffer;entry:string;sha256:string}{ - const entry='node_modules/@jimhoyd/urlcode-demo/dist/index.js', manifest={format:1,coreVersion,bundles:[{name:'demo',version:'1.0.0',entry}]}; - // The executable test module is deliberately a fixed literal, never a - // template populated from an input. It models the reviewed `demo` bundle. - const moduleSource=`export async function scaffold(){return {name:'demo',extensions:{demo:{version:'1',config:{label:'bundle'}}},routes:{'/demo/*':{extension:'demo',methods:['GET']}},hostImports:[],hostBundleExports:['demoExtension'],hostSetup:['const demoSha = process.env.PROJECT_SHA256;'],hostEntries:['demoExtension(demoSha)'],files:[],readme:'Bundle demo.',nextSteps:['serve bundle demo']};} -export const demoExtension=(projectSha256)=>({name:'demo',version:'1',projectSha256,targets:['node'],schema:{type:'object'},activate(){return {handle:()=>({status:200,headers:[],body:'ok'})}}});`; - const files:{path:string;body:string}[]=[{path:'bundle.json',body:JSON.stringify(manifest)},{path:'node_modules/@jimhoyd/urlcode-demo/package.json',body:JSON.stringify({type:'module'})},{path:entry,body:moduleSource}]; - const parts:Buffer[]=[];for(const file of files){const body=Buffer.from(file.body),header=Buffer.alloc(512);header.write(file.path);header.write(body.length.toString(8).padStart(11,'0')+'\0',124);header[156]=48;header.fill(32,148,156);const checksum=[...header].reduce((sum,byte)=>sum+byte,0);header.write(checksum.toString(8).padStart(6,'0')+'\0 ',148);parts.push(header,body,Buffer.alloc((512-body.length%512)%512));}parts.push(Buffer.alloc(1024));const bytes=gzipSync(Buffer.concat(parts));return {bytes,entry,sha256:createHash('sha256').update(bytes).digest('hex')}; +/** A `BundleTransport` serving exactly the given fake bundles under one release tag. */ +function fakeBundleTransport(bundles: FakeBundle[], tag = `extension-bundles@v${coreVersion}`): { release: string; transport: BundleTransport } { + const catalog = Buffer.from(JSON.stringify({ format: 1, tag, commit: 'a'.repeat(40), coreVersion, bundles: bundles.map(b => ({ name: b.name, version: '1.0.0', asset: b.asset, sha256: b.sha256, entry: b.entry })), revoked: [] })); + const transport: BundleTransport = { + release: async () => [{ name: 'extension-bundles-catalog.json', url: 'catalog' }, ...bundles.map(b => ({ name: b.asset, url: b.asset }))], + download: async url => url === 'catalog' ? catalog : bundles.find(b => b.asset === url)!.bytes, + attest: async () => {}, + }; + return { release: tag, transport }; } test('init --with merges fake extension scaffolds in canonical order, keeps file modes and the result validates with the generated host', async t => { const root = await project(t, {}); - await fakePackage(root, 'demo'); await fakePackage(root, 'other'); - const created = run(root, ['init', 'site', '--with', 'other,demo']); - assert.equal(created.status, 0, created.stderr); - const report = parse(created.stdout); - assert.equal(report.event, 'created'); assert.deepEqual(report.extensions, ['demo', 'other']); + const { release, transport } = fakeBundleTransport([fakeBundle('demo'), fakeBundle('other')]); + const created = await initProjectWith(join(root, 'site'), ['other', 'demo'], { cwd: root, bundleRelease: release, bundleTransport: transport }); + assert.deepEqual(created.extensions, ['demo', 'other']); const site = join(root, 'site'), app = join(site, 'app'); // The CLI resolves against its cwd, which macOS reports through /private and Windows may report as a short name; compare canonical paths. const canonical = (path: string) => realpathSync.native(path); - assert.equal(canonical(String(report.project)), canonical(app)); assert.equal(canonical(String(report.hostFile)), canonical(join(site, 'host.mjs'))); + assert.equal(canonical(created.project), canonical(app)); assert.equal(canonical(created.hostFile), canonical(join(site, 'host.mjs'))); const sha = await inspectExtensionRevision(app); - assert.equal(report.projectSha256, sha); assert.match(String(report.review), new RegExp(`PROJECT_SHA256=${sha}`)); + assert.equal(created.projectSha256, sha); const loaded = await loadDocument(app); assert.deepEqual(Object.keys(loaded.routes), ['/demo/*', '/other/*']); assert.deepEqual(Object.keys(loaded.document.extensions ?? {}), ['demo', 'other']); const host = await readFile(join(site, 'host.mjs'), 'utf8'); - const order = ['import {fakeExtension as demoExtension}', 'import {fakeExtension as otherExtension}', 'const demoSha', 'const otherSha', 'demoExtension(demoSha),', 'otherExtension(otherSha),', '// release other', '// release demo'].map(needle => host.indexOf(needle)); + const order = ["const {demoExtension} = await loadExtensionBundle(extensionBundleDirectory, 'demo');", "const {otherExtension} = await loadExtensionBundle(extensionBundleDirectory, 'other');", 'const demoSha', 'const otherSha', 'demoExtension(demoSha),', 'otherExtension(otherSha),', '// release other', '// release demo'].map(needle => host.indexOf(needle)); assert.ok(order.every((index, i) => index >= 0 && (i === 0 || index > order[i - 1]!)), host); // Windows has no POSIX modes; the files still exist there. for (const [file, mode] of [['operator-demo.mjs', 0o600], ['data/other.key', 0o600], ['notes/demo.txt', 0o644], ['host.mjs', 0o600]] as const) { const info = await stat(join(site, file)); if (process.platform !== 'win32') assert.equal(info.mode & 0o777, mode, file); } @@ -79,24 +94,18 @@ test('init --with merges fake extension scaffolds in canonical order, keeps file assert.ok(await missing(join(app, '.mcp.json'))); assert.deepEqual(JSON.parse(await readFile(join(site, '.mcp.json'), 'utf8')), { mcpServers: { urlcode: { command: 'npx', args: ['--no', '--package', '@jimhoyd/urlcode', 'urlcode', 'mcp', '--project', 'app'] } } }); assert.ok((await readFile(join(app, '.gitignore'), 'utf8')).includes('.env.*')); - // The site records the versions it was generated against; installing them stays an explicit operator step. - assert.deepEqual(JSON.parse(await readFile(join(site, 'package.json'), 'utf8')).dependencies['@jimhoyd/urlcode-demo'], '1.0.0'); + // Bundle distribution pins only the runtime; extensions are locked in urlcode.extension-bundles.lock.json, not npm dependencies. + assert.deepEqual(JSON.parse(await readFile(join(site, 'package.json'), 'utf8')).dependencies, { '@jimhoyd/urlcode': coreVersion }); assert.ok(await missing(join(site, 'package-lock.json'))); - const validated = run(root, ['validate', '--project', app, '--host-file', join(site, 'host.mjs'), '--origin', 'https://demo.example'], { PROJECT_SHA256: sha }); - assert.equal(validated.status, 0, validated.stderr); - assert.equal(parse(validated.stdout).routes, 2); + assert.ok(!(await missing(join(site, 'urlcode.extension-bundles.lock.json')))); // Nothing generated is ever overwritten: the destination is reserved once. - const again = run(root, ['init', 'site', '--with', 'demo']); - assert.equal(again.status, 1); assert.equal(await readFile(join(site, 'host.mjs'), 'utf8'), host); + await assert.rejects(initProjectWith(join(root, 'site'), ['demo'], { cwd: root, bundleRelease: release, bundleTransport: transport })); + assert.equal(await readFile(join(site, 'host.mjs'), 'utf8'), host); }); test('init --with refuses duplicate extension routes, missing packages and packages without scaffold before writing anything', async t => { const root = await project(t, {}); - await fakePackage(root, 'demo'); await fakePackage(root, 'twin', { routes: { '/demo/*': { extension: 'twin' } } }); - await fakePackage(root, 'plain', { scaffold: false }); + // Argument validation happens before any bundle is installed, so these refuse without a fixture and without touching the network. const cases: [string, RegExp][] = [ - ['demo,twin', /Route \/demo\/\* is added by both demo and twin/], - ['missing', /@jimhoyd\/urlcode-missing is not installed .*run: npm install @jimhoyd\/urlcode-missing/], - ['plain', /@jimhoyd\/urlcode-plain does not export scaffold/], ['demo,demo', /Duplicate --with names/], ['Demo', /--with name/], ['', /--with name/], ]; for (const [names, message] of cases) { @@ -105,38 +114,47 @@ test('init --with refuses duplicate extension routes, missing packages and packa assert.ok(await missing(join(root, 'site')), `${names} left files behind`); } assert.match(run(root, ['validate', '--with', 'demo']).stderr, /--with is only supported by init/); - await assert.rejects(initProjectWith(join(root, 'site'), ['demo', 'missing'], { cwd: root }), /not installed/); + // These need an actual (fake) bundle install to reach the refusal, so they run through the JS API with an injected transport. + const { release, transport } = fakeBundleTransport([fakeBundle('demo'), fakeBundle('twin', { routes: { '/demo/*': { extension: 'twin' } } }), fakeBundle('plain', { scaffold: false })]); + await assert.rejects(initProjectWith(join(root, 'site'), ['demo', 'twin'], { cwd: root, bundleRelease: release, bundleTransport: transport }), /Route \/demo\/\* is added by both demo and twin/); + assert.ok(await missing(join(root, 'site'))); + await assert.rejects(initProjectWith(join(root, 'site'), ['demo', 'missing'], { cwd: root, bundleRelease: release, bundleTransport: transport }), /Extension bundle missing is not in the signed catalog/); + assert.ok(await missing(join(root, 'site'))); + await assert.rejects(initProjectWith(join(root, 'site'), ['plain'], { cwd: root, bundleRelease: release, bundleTransport: transport }), /@jimhoyd\/urlcode-plain does not export scaffold/); assert.ok(await missing(join(root, 'site'))); assert.deepEqual(parseWithNames(' auth , admin'), ['auth', 'admin']); }); -test('init --with bundle release writes a locked npm-free extension host',async t=>{ - const root=await project(t,{}), archive=bundleArchive(), release=`extension-bundles@v${coreVersion}`, catalog=Buffer.from(JSON.stringify({format:1,tag:release,commit:'a'.repeat(40),coreVersion,bundles:[{name:'demo',version:'1.0.0',asset:'demo-1.0.0.tgz',sha256:archive.sha256,entry:archive.entry}],revoked:[]})); - const transport:BundleTransport={release:async()=>[{name:'extension-bundles-catalog.json',url:'catalog'},{name:'demo-1.0.0.tgz',url:'bundle'}],download:async url=>url==='catalog'?catalog:archive.bytes,attest:async()=>{}}; - const created=await initProjectWith(join(root,'site'),['demo'],{cwd:root,bundleRelease:release,bundleTransport:transport}); - const host=await readFile(created.hostFile,'utf8'); - assert.match(host,/loadExtensionBundle/);assert.match(host,/loadExtensionBundle\(extensionBundleDirectory, 'demo'\)/);assert.doesNotMatch(host,/@jimhoyd\/urlcode-demo/); - assert.ok(!(await missing(join(root,'site','urlcode.extension-bundles.lock.json'))));assert.ok(!(await missing(join(root,'site','.urlcode','extension-bundles',archive.sha256,'.bundle.tgz')))); - assert.deepEqual(JSON.parse(await readFile(join(root,'site','package.json'),'utf8')).dependencies,{'@jimhoyd/urlcode':coreVersion}); +test('init --with resolves extension-bundles@v when --bundle-release is omitted, and refuses when that release does not exist', async t => { + const root = await project(t, {}); + const bundle = fakeBundle('demo'), { transport } = fakeBundleTransport([bundle]); + // With no --bundle-release, initProjectWith must ask the transport for exactly extension-bundles@v. + let requested: string | undefined; + const capturing: BundleTransport = { release: async tag => { requested = tag; return transport.release(tag); }, download: transport.download, attest: transport.attest }; + const created = await initProjectWith(join(root, 'site'), ['demo'], { cwd: root, bundleTransport: capturing }); + assert.equal(requested, `extension-bundles@v${coreVersion}`); + assert.deepEqual(created.extensions, ['demo']); + const missingTransport: BundleTransport = { release: async () => { throw new Error('Could not fetch extension bundle release'); }, download: async () => { throw new Error('unused'); }, attest: async () => {} }; + await assert.rejects(initProjectWith(join(root, 'other'), ['demo'], { cwd: root, bundleTransport: missingTransport }), /Could not fetch extension bundle release/); }); test('init --with carries generic --ack acknowledgements: refusal prints the exact command, unconsumed values are rejected, nothing is written on refusal', async t => { const root = await project(t, {}); - await fakePackage(root, 'risky', { risk: true }); await fakePackage(root, 'calm'); - const refused = run(root, ['init', 'site', '--with', 'calm,risky', '--no-manifest']); - assert.equal(refused.status, 1); assert.match(refused.stderr, /this would do something risky\. If you accept that risk, re-run with the acknowledgement: urlcode init site --with calm,risky --no-manifest --ack risky:risky/); + const { release, transport } = fakeBundleTransport([fakeBundle('risky', { risk: true }), fakeBundle('calm')]); + const opts = { cwd: root, bundleRelease: release, bundleTransport: transport, manifest: false }; + await assert.rejects(initProjectWith(join(root, 'site'), ['calm', 'risky'], opts), /this would do something risky\. If you accept that risk, re-run with the acknowledgement: urlcode init .*[/\\]site --with calm,risky --bundle-release extension-bundles@v[^ ]+ --no-manifest --ack risky:risky/); assert.ok(await missing(join(root, 'site'))); // An acknowledgement for another extension does not satisfy it, and the refusal keeps what was already passed. - const other = run(root, ['init', 'site', '--with', 'calm,risky', '--no-manifest', '--ack', 'calm:other']); - assert.match(other.stderr, /--no-manifest --ack calm:other --ack risky:risky/); - const ok = run(root, ['init', 'site', '--with', 'risky', '--no-manifest', '--ack', 'risky:risky', '--ack', 'risky:risky']); - assert.equal(ok.status, 0, ok.stderr); + await assert.rejects(initProjectWith(join(root, 'site'), ['calm', 'risky'], { ...opts, acknowledgements: ['calm:other'] }), /--bundle-release extension-bundles@v[^ ]+ --no-manifest --ack calm:other --ack risky:risky/); + const ok = await initProjectWith(join(root, 'site'), ['risky'], { ...opts, acknowledgements: ['risky:risky', 'risky:risky'] }); + assert.deepEqual(ok.extensions, ['risky']); const unused = ['calm:risky', 'risky:other', 'ghost:thing']; for (const [index, id] of unused.entries()) { - const result = run(root, ['init', `u${index}`, '--with', 'calm,risky', '--no-manifest', '--ack', 'risky:risky', '--ack', id]); - assert.equal(result.status, 1, id); assert.match(result.stderr, new RegExp(`--ack ${id} has no effect`)); assert.ok(await missing(join(root, `u${index}`))); + const destination = join(root, `u${index}`); + await assert.rejects(initProjectWith(destination, ['calm', 'risky'], { ...opts, acknowledgements: ['risky:risky', id] }), new RegExp(`--ack ${id.replace(':', '\\:')} has no effect`)); + assert.ok(await missing(destination), id); } - assert.match(run(root, ['init', 'bad', '--with', 'calm', '--ack', 'nocolon']).stderr, /Use --ack :/); + await assert.rejects(initProjectWith(join(root, 'bad'), ['calm'], { ...opts, acknowledgements: ['nocolon'] }), /Use --ack :/); assert.match(run(root, ['init', 'bad', '--ack', 'calm:x']).stderr, /--ack is only supported by init with --with/); assert.match(run(root, ['validate', '--ack', 'calm:x']).stderr, /--ack is only supported by init with --with/); }); @@ -148,12 +166,14 @@ const siteFiles = async (root: string, name: string): Promise<{ yaml: string; ro }; test('init --with treats the set as unordered: every permutation emits the same order, host and revision', async t => { const root = await project(t, {}); - await fakePackage(root, 'ui', { contract: { provides: ['ui.kit'] } }); - await fakePackage(root, 'auth', { contract: { requires: ['ui.kit'], provides: ['auth.service'] } }); - await fakePackage(root, 'admin', { contract: { requires: ['ui.kit', 'auth.service'] } }); + const { release, transport } = fakeBundleTransport([ + fakeBundle('ui', { contract: { provides: ['ui.kit'] } }), + fakeBundle('auth', { contract: { requires: ['ui.kit'], provides: ['auth.service'] } }), + fakeBundle('admin', { contract: { requires: ['ui.kit', 'auth.service'] } }), + ]); let index = 0, baseline: Record | undefined; for (const order of permutations(['admin', 'auth', 'ui'])) { - const name = `site${index++}`, created = await initProjectWith(join(root, name), order, { cwd: root, manifest: false }); + const name = `site${index++}`, created = await initProjectWith(join(root, name), order, { cwd: root, manifest: false, bundleRelease: release, bundleTransport: transport }); assert.deepEqual(created.extensions, ['ui', 'auth', 'admin']); const files = await siteFiles(root, name); assert.equal(files.sha, created.projectSha256); @@ -166,14 +186,17 @@ test('init --with treats the set as unordered: every permutation emits the same }); test('init --with refuses a missing requirement, a conflict or a cycle before writing, naming the extensions', async t => { const root = await project(t, {}); - await fakePackage(root, 'ui', { contract: { provides: ['ui.kit'] } }); - await fakePackage(root, 'needy', { contract: { requires: ['ui.kit', 'auth'] } }); - await fakePackage(root, 'clash', { contract: { conflicts: ['ui'] } }); - await fakePackage(root, 'loopa', { contract: { requires: ['loopb'] } }); await fakePackage(root, 'loopb', { contract: { after: ['loopa'] } }); + const { release, transport } = fakeBundleTransport([ + fakeBundle('ui', { contract: { provides: ['ui.kit'] } }), + fakeBundle('needy', { contract: { requires: ['ui.kit', 'auth'] } }), + fakeBundle('clash', { contract: { conflicts: ['ui'] } }), + fakeBundle('loopa', { contract: { requires: ['loopb'] } }), fakeBundle('loopb', { contract: { after: ['loopa'] } }), + ]); + const opts = { cwd: root, manifest: false, bundleRelease: release, bundleTransport: transport }; for (const [names, expected] of [[['needy'], /needy requires ui\.kit/], [['needy', 'ui'], /needy requires auth/], [['ui', 'clash'], /clash conflicts with ui/], [['loopb', 'loopa', 'ui'], /cycle among loopa \(needs loopb\); loopb \(needs loopa\)/]] as const) { const destination = join(root, 'refused'); - await assert.rejects(initProjectWith(destination, names, { cwd: root, manifest: false }), expected); + await assert.rejects(initProjectWith(destination, names, opts), expected); assert.ok(await missing(destination), names.join(',')); } - assert.doesNotMatch(await initProjectWith(join(root, 'ok'), ['ui'], { cwd: root, manifest: false }).then(() => '', error => String(error)), /./); + assert.doesNotMatch(await initProjectWith(join(root, 'ok'), ['ui'], opts).then(() => '', error => String(error)), /./); }); diff --git a/test/project-dependencies.test.ts b/test/project-dependencies.test.ts index 2af34419..22c13b60 100644 --- a/test/project-dependencies.test.ts +++ b/test/project-dependencies.test.ts @@ -1,10 +1,14 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; import { lstat, mkdir, readFile, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { gzipSync } from 'node:zlib'; import { collectDependencySet, installSteps, parsePin, renderPackageManifest, satisfiesRange } from '../packages/core/src/project-dependencies.ts'; +import { initProjectWith } from '../packages/core/src/init-with.ts'; +import type { BundleTransport } from '../packages/core/src/extension-bundles.ts'; import { project } from './helpers.ts'; const cli = fileURLToPath(new URL('../packages/core/src/cli.ts', import.meta.url)); @@ -13,6 +17,42 @@ const parse = (out: string): Record => JSON.parse(out.trim().sp const missing = async (path: string): Promise => { try { await lstat(path); return false; } catch { return true; } }; const core = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')) as { version: string }; +/** A fake signed bundle for `@jimhoyd/urlcode-`, pinned to an arbitrary coreVersion (for compatibility tests). */ +function fakeBundle(name: string, coreVersion: string): { name: string; asset: string; entry: string; sha256: string; bytes: Buffer } { + const entry = `node_modules/@jimhoyd/urlcode-${name}/dist/index.js`; + const manifest = { format: 1, coreVersion, bundles: [{ name, version: '2.0.1', entry }] }; + const moduleSource = `const name=${JSON.stringify(name)}; +export function ${name}Extension(){return {name,version:'1',targets:['node'],schema:{type:'object'},activate(){return {handle:()=>({status:200,headers:[],body:'ok'})}}};} +export async function scaffold(){return {name,extensions:{[name]:{version:'1',config:{}}},routes:{[\`/\${name}/*\`]:{extension:name,methods:['GET','HEAD']}}, + hostImports:[],hostBundleExports:['${name}Extension'],hostSetup:[],hostEntries:[\`${name}Extension()\`],files:[],readme:'Readme.',nextSteps:['do the thing']};} +`; + const files: { path: string; body: string }[] = [ + { path: 'bundle.json', body: JSON.stringify(manifest) }, + { path: `node_modules/@jimhoyd/urlcode-${name}/package.json`, body: JSON.stringify({ type: 'module' }) }, + { path: entry, body: moduleSource }, + ]; + const parts: Buffer[] = []; + for (const file of files) { + const body = Buffer.from(file.body), header = Buffer.alloc(512); + header.write(file.path); header.write(body.length.toString(8).padStart(11, '0') + '\0', 124); + header[156] = 48; header.fill(32, 148, 156); + const checksum = [...header].reduce((sum, byte) => sum + byte, 0); + header.write(checksum.toString(8).padStart(6, '0') + '\0 ', 148); + parts.push(header, body, Buffer.alloc((512 - body.length % 512) % 512)); + } + parts.push(Buffer.alloc(1024)); + const bytes = gzipSync(Buffer.concat(parts)); + return { name, asset: `${name}-2.0.1.tgz`, entry, sha256: createHash('sha256').update(bytes).digest('hex'), bytes }; +} +function fakeBundleTransport(bundles: ReturnType[], tag: string, coreVersion: string): BundleTransport { + const catalog = Buffer.from(JSON.stringify({ format: 1, tag, commit: 'a'.repeat(40), coreVersion, bundles: bundles.map(b => ({ name: b.name, version: '2.0.1', asset: b.asset, sha256: b.sha256, entry: b.entry })), revoked: [] })); + return { + release: async () => [{ name: 'extension-bundles-catalog.json', url: 'catalog' }, ...bundles.map(b => ({ name: b.asset, url: b.asset }))], + download: async url => url === 'catalog' ? catalog : bundles.find(b => b.asset === url)!.bytes, + attest: async () => {}, + }; +} + interface PackageOptions { version?: string; peers?: Record; optionalPeers?: string[]; node?: string; scaffold?: boolean } /** A fake installed `@jimhoyd/urlcode-`: manifest metadata plus the `scaffold` export `init --with` calls. */ async function fakePackage(root: string, name: string, { version = '1.2.3', peers, optionalPeers, node, scaffold = true }: PackageOptions = {}): Promise { @@ -111,20 +151,15 @@ test('local checkouts and tarballs are pinned by specifier, explicitly and from test('init writes a coordinated manifest for --with, on request for a route-only project, and never by surprise', async t => { const root = await project(t, {}); - await fakePackage(root, 'demo', { version: '2.0.1', peers: { '@jimhoyd/urlcode': `^${core.version}` } }); - const created = run(root, ['init', 'site', '--with', 'demo']); - assert.equal(created.status, 0, created.stderr); - const report = parse(created.stdout); - assert.deepEqual(report.dependencies, [ - { name: '@jimhoyd/urlcode', version: core.version, specifier: core.version, local: false, role: 'runtime' }, - { name: '@jimhoyd/urlcode-demo', version: '2.0.1', specifier: '2.0.1', local: false, role: 'extension' }, - ]); + const tag = `extension-bundles@v${core.version}`, transport = fakeBundleTransport([fakeBundle('demo', core.version)], tag, core.version); + const created = await initProjectWith(join(root, 'site'), ['demo'], { cwd: root, bundleRelease: tag, bundleTransport: transport }); + assert.deepEqual(created.dependencies, [{ name: '@jimhoyd/urlcode', version: core.version, specifier: core.version, local: false, role: 'runtime' }]); const manifest = JSON.parse(await readFile(join(root, 'site', 'package.json'), 'utf8')) as { dependencies: Record }; - assert.deepEqual(manifest.dependencies, { '@jimhoyd/urlcode': core.version, '@jimhoyd/urlcode-demo': '2.0.1' }); + assert.deepEqual(manifest.dependencies, { '@jimhoyd/urlcode': core.version }); const readme = await readFile(join(root, 'site', 'README.md'), 'utf8'); - for (const needle of ['## Dependencies', '`@jimhoyd/urlcode-demo` 2.0.1 (extension)', 'There is no upgrade command.', '1. Review']) assert.ok(readme.includes(needle), needle); + for (const needle of ['## Dependencies', 'urlcode.extension-bundles.lock.json', 'There is no upgrade command.', '1. Review']) assert.ok(readme.includes(needle), needle); // The install step is printed first and never run: no lockfile and no node_modules appear in the generated site. - assert.match(String((report.nextSteps as string[])[1]), /Run `npm install`/); + assert.match(String(created.nextSteps[1]), /Run `npm install`/); assert.ok(await missing(join(root, 'site', 'package-lock.json')) && await missing(join(root, 'site', 'node_modules'))); // Route-only initialization keeps managing the runtime elsewhere: no manifest unless it is asked for. @@ -135,7 +170,8 @@ test('init writes a coordinated manifest for --with, on request for a route-only assert.deepEqual(JSON.parse(await readFile(join(root, 'pinned', 'package.json'), 'utf8')).dependencies, { '@jimhoyd/urlcode': core.version }); assert.equal(parse(pinned.stdout).event, 'created'); // --with without a manifest stays available for a site whose dependencies are managed elsewhere. - assert.equal(run(root, ['init', 'bare', '--with', 'demo', '--no-manifest']).status, 0); + const bare = await initProjectWith(join(root, 'bare'), ['demo'], { cwd: root, bundleRelease: tag, bundleTransport: transport, manifest: false }); + assert.deepEqual(bare.dependencies, []); assert.ok(await missing(join(root, 'bare', 'package.json'))); assert.ok(!(await readFile(join(root, 'bare', 'README.md'), 'utf8')).includes('## Dependencies')); // Flag misuse refuses before anything is created. @@ -145,13 +181,9 @@ test('init writes a coordinated manifest for --with, on request for a route-only assert.ok(await missing(join(root, 'x'))); }); -test('init --with refuses to record pins when an extension is incompatible with this runtime, leaving nothing behind', async t => { +test('init --with refuses to install a bundle whose catalog targets a different core version, leaving nothing behind', async t => { const root = await project(t, {}); - await fakePackage(root, 'demo', { version: '2.0.1', peers: { '@jimhoyd/urlcode': '>=9.0.0 <10.0.0' } }); - const refused = run(root, ['init', 'site', '--with', 'demo']); - assert.equal(refused.status, 1); - assert.match(refused.stderr, /Incompatible versions: @jimhoyd\/urlcode-demo 2\.0\.1 requires @jimhoyd\/urlcode >=9\.0\.0 <10\.0\.0/); + const tag = `extension-bundles@v${core.version}`, transport = fakeBundleTransport([fakeBundle('demo', '9.0.0')], tag, '9.0.0'); + await assert.rejects(initProjectWith(join(root, 'site'), ['demo'], { cwd: root, bundleRelease: tag, bundleTransport: transport }), /Extension bundle demo requires core 9\.0\.0; this runtime is/); assert.ok(await missing(join(root, 'site'))); - // The escape hatch is explicit, and it is the only thing that makes the incompatible set proceed. - assert.equal(run(root, ['init', 'site', '--with', 'demo', '--no-manifest']).status, 0); }); diff --git a/test/workspace-scaffold.integration.ts b/test/workspace-scaffold.integration.ts index b2c24884..f005840a 100644 --- a/test/workspace-scaffold.integration.ts +++ b/test/workspace-scaffold.integration.ts @@ -1,35 +1,126 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; -import { lstat, mkdir, mkdtemp, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { lstat, mkdir, mkdtemp, readdir, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { fileURLToPath, pathToFileURL } from 'node:url'; +import { gzipSync } from 'node:zlib'; import { loadDocument } from '../packages/core/src/config.ts'; import { inspectExtensionRevision } from '../packages/core/src/extensions.ts'; import { createRuntime } from '../packages/core/src/runtime.ts'; +import { initProjectWith } from '../packages/core/src/init-with.ts'; import type { RuntimeExtension } from '../packages/core/src/extensions.ts'; import type { HandlerResult } from '../packages/core/src/http-response.ts'; +import type { BundleTransport } from '../packages/core/src/extension-bundles.ts'; import { project } from './helpers.ts'; const cli = fileURLToPath(new URL('../packages/core/src/cli.ts', import.meta.url)); const run = (cwd: string, args: string[]) => spawnSync(process.execPath, [cli, ...args], { cwd, encoding: 'utf8', timeout: 60000 }); -const parse = (out: string): Record => JSON.parse(out.trim().split('\n').pop()!) as Record; const missing = async (path: string): Promise => { try { await lstat(path); return false; } catch { return true; } }; // Run after workspace builds, against actual compiled packages. Missing outputs fail. const companions = Object.fromEntries(['auth', 'admin', 'ui', 'store'].map(name => [`urlcode-${name}`, fileURLToPath(new URL(`../packages/${name}`, import.meta.url))])); +const coreDirectory = fileURLToPath(new URL('..', import.meta.url)); +const coreVersion = (JSON.parse(await readFile(join(coreDirectory, 'package.json'), 'utf8')) as { version: string }).version; + +// A minimal local stand-in for the real, git+npm-install-based scripts/prepare-extension-bundles.ts: same +// dependencySet/entryFor conventions (each named bundle self-contains its own transitive extension deps, so a +// cross-package bare import resolves from inside the extracted bundle cache), sourced directly from this +// monorepo's already-built workspace packages instead of a fresh checkout and npm install. +type BundleName = 'ui' | 'auth' | 'admin' | 'store'; +const dependencySet = (bundle: BundleName): BundleName[] => bundle === 'ui' ? ['ui'] : bundle === 'auth' ? ['ui', 'auth'] : bundle === 'admin' ? ['ui', 'auth', 'admin'] : ['store']; +const entryFor = (bundle: BundleName): string => `node_modules/@jimhoyd/urlcode-${bundle}/dist/${bundle === 'ui' ? 'host/index' : 'index'}.js`; +function tarPath(path: string): { name: string; prefix: string } { + if (Buffer.byteLength(path, 'utf8') <= 100) return { name: path, prefix: '' }; + const pieces = path.split('/'); let name = pieces.pop()!; + while (pieces.length && Buffer.byteLength(name, 'utf8') <= 100) { const prefix = pieces.join('/'); if (Buffer.byteLength(prefix, 'utf8') <= 155) return { name, prefix }; name = `${pieces.pop()}/${name}`; } + throw new Error(`Cannot represent path in USTAR: ${path}`); +} +function tar(files: { path: string; bytes: Buffer }[]): Buffer { + const parts: Buffer[] = []; + for (const file of files) { + const path = tarPath(file.path), header = Buffer.alloc(512); + header.write(path.name, 0, 100, 'utf8'); header.write(path.prefix, 345, 155, 'utf8'); + header.write((0o644).toString(8).padStart(7, '0') + '\0', 100, 8, 'ascii'); + header.write('0000000\0', 108, 8, 'ascii'); header.write('0000000\0', 116, 8, 'ascii'); + header.write(file.bytes.byteLength.toString(8).padStart(11, '0') + '\0', 124, 12, 'ascii'); + header.write('00000000000\0', 136, 12, 'ascii'); + header.fill(32, 148, 156); header[156] = 48; + header.write('ustar\0', 257, 6, 'ascii'); header.write('00', 263, 2, 'ascii'); + const checksum = [...header].reduce((sum, byte) => sum + byte, 0); + header.write(checksum.toString(8).padStart(6, '0') + '\0 ', 148, 8, 'ascii'); + parts.push(header, file.bytes, Buffer.alloc((512 - file.bytes.byteLength % 512) % 512)); + } + parts.push(Buffer.alloc(1024)); + return Buffer.concat(parts); +} +async function walk(root: string, prefix = ''): Promise<{ path: string; bytes: Buffer }[]> { + const found: { path: string; bytes: Buffer }[] = []; + for (const entry of await readdir(join(root, prefix), { withFileTypes: true })) { + const path = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) found.push(...await walk(root, path)); + else if (entry.isFile()) found.push({ path, bytes: await readFile(join(root, path)) }); + } + return found; +} +// core's own dist imports third-party dependencies (ajv, etc.), so a plain file copy of dist/ isn't enough to be +// self-contained the way a real bundle is: prepare-extension-bundles.ts handles this by npm-installing every +// bundle's staged package.json for real, from packed tarballs (a raw `file:` dependency does not pull +// in its own transitive dependencies; only a packed one does). Mirror exactly that, skipping only the +// git-archive step (which packs this same checkout under a real commit) in favor of `npm pack` directly. +const packRoot = await mkdtemp(join(tmpdir(), 'urlcode-bundle-pack-')); +const npmBin = process.platform === 'win32' ? 'npm.cmd' : 'npm'; +async function packSource(directory: string): Promise { + const result = spawnSync(npmBin, ['pack', '--json', '--pack-destination', packRoot, directory], { encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + return join(packRoot, (JSON.parse(result.stdout) as { filename: string }[])[0]!.filename); +} +const packedCore = await packSource(coreDirectory); +const packedCompanions = Object.fromEntries(await Promise.all((['ui', 'auth', 'admin', 'store'] as const).map(async name => [name, await packSource(companions[`urlcode-${name}`]!)] as const))); +async function packBundle(bundle: BundleName): Promise<{ name: BundleName; asset: string; entry: string; sha256: string; bytes: Buffer }> { + const staging = await mkdtemp(join(tmpdir(), `urlcode-bundle-stage-${bundle}-`)); + try { + const dependencies: Record = { '@jimhoyd/urlcode': `file:${packedCore}` }; + for (const dep of dependencySet(bundle)) dependencies[`@jimhoyd/urlcode-${dep}`] = `file:${packedCompanions[dep]}`; + await writeFile(join(staging, 'package.json'), JSON.stringify({ name: 'urlcode-extension-bundle-stage', private: true, version: '0.0.0', dependencies })); + const install = spawnSync(npmBin, ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--omit=dev', '--package-lock=false'], { cwd: staging, encoding: 'utf8' }); + assert.equal(install.status, 0, install.stderr); + const tree = (await walk(join(staging, 'node_modules'))).map(file => ({ path: `node_modules/${file.path}`, bytes: file.bytes })); + const entry = entryFor(bundle); + assert.ok(tree.some(file => file.path === entry), `packed ${bundle} is missing its entry module`); + const bundleJson = Buffer.from(JSON.stringify({ format: 1, coreVersion, bundles: [{ name: bundle, version: '0.5.0', entry }] })); + const bytes = gzipSync(tar([{ path: 'bundle.json', bytes: bundleJson }, ...tree])); + return { name: bundle, asset: `${bundle}-0.5.0.tgz`, entry, sha256: createHash('sha256').update(bytes).digest('hex'), bytes }; + } finally { await rm(staging, { recursive: true, force: true }); } +} +const packedBundles = await Promise.all((['ui', 'auth', 'admin', 'store'] as const).map(packBundle)); +const bundleReleaseTag = `extension-bundles@v${coreVersion}`; +const bundleTransport: BundleTransport = { + release: async () => [{ name: 'extension-bundles-catalog.json', url: 'catalog' }, ...packedBundles.map(b => ({ name: b.asset, url: b.asset }))], + download: async url => url === 'catalog' ? Buffer.from(JSON.stringify({ format: 1, tag: bundleReleaseTag, commit: 'a'.repeat(40), coreVersion, bundles: packedBundles.map(b => ({ name: b.name, version: '0.5.0', asset: b.asset, sha256: b.sha256, entry: b.entry })), revoked: [] })) : packedBundles.find(b => b.asset === url)!.bytes, + attest: async () => {}, +}; +/** host.mjs/operator-service.mjs, and the packed bundles themselves, import `@jimhoyd/urlcode`; link core at root so it + * resolves both from the ephemeral bundle-install temp dir (a sibling of the generated site under root) and the + * final site once written, exactly as a real npm install of the runtime would make it resolvable. */ +const linkCore = async (dir: string): Promise => { + await mkdir(join(dir, 'node_modules', '@jimhoyd'), { recursive: true }); + try { await symlink(coreDirectory, join(dir, 'node_modules', '@jimhoyd', 'urlcode'), process.platform === 'win32' ? 'junction' : 'dir'); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; } +}; +/** initProjectWith with the packed local transport, mirroring `run(root, ['init', dest, '--with', names.join(','), ...])`. */ +const initWith = async (root: string, dest: string, names: string[], opts: { acknowledgements?: string[]; manifest?: boolean } = {}) => { + await linkCore(root); + return initProjectWith(join(root, dest), names, { cwd: root, bundleRelease: bundleReleaseTag, bundleTransport, ...opts }); +}; test('init --with store writes a working CRUD host with no handler code', async t => { const root = await project(t, {}); - const link = async (dir: string): Promise => { await mkdir(join(dir, 'node_modules', '@jimhoyd'), { recursive: true }); await symlink(companions['urlcode-store']!, join(dir, 'node_modules', '@jimhoyd', 'urlcode-store'), process.platform === 'win32' ? 'junction' : 'dir'); }; - await link(root); - const created = run(root, ['init', 'todo-site', '--with', 'store', '--ack', 'store:public-write']); - assert.equal(created.status, 0, created.stderr); - const report = parse(created.stdout), site = join(root, 'todo-site'), app = join(site, 'app'); + const created = await initWith(root, 'todo-site', ['store'], { acknowledgements: ['store:public-write'] }); + const site = join(root, 'todo-site'), app = join(site, 'app'); assert.deepEqual(Object.keys((await loadDocument(app)).routes), ['/api/todos/*']); - assert.deepEqual(report.extensions, ['store']); - // The generated host imports the installed package by name, exactly as a real site resolves it after npm install. - await link(site); + assert.deepEqual(created.extensions, ['store']); const previous = process.env.PROJECT_SHA256; - process.env.PROJECT_SHA256 = String(report.projectSha256); + process.env.PROJECT_SHA256 = String(created.projectSha256); t.after(() => { if (previous === undefined) delete process.env.PROJECT_SHA256; else process.env.PROJECT_SHA256 = previous; }); const host = await import(pathToFileURL(join(site, 'host.mjs')).href) as { default: { extensions: RuntimeExtension[]; close(): Promise } }; const origin = 'https://todo.example.test'; @@ -44,18 +135,14 @@ test('init --with store writes a working CRUD host with no handler code', async test('init --with ui,store serves a data-bound list and form screen for the declared collection (#262)', async t => { const root = await project(t, {}); - const link = async (dir: string): Promise => { await mkdir(join(dir, 'node_modules', '@jimhoyd'), { recursive: true }); for (const name of ['urlcode-ui', 'urlcode-store']) await symlink(companions[name]!, join(dir, 'node_modules', '@jimhoyd', name), process.platform === 'win32' ? 'junction' : 'dir'); }; - await link(root); - const created = run(root, ['init', 'todo-site', '--with', 'ui,store', '--ack', 'store:public-write']); - assert.equal(created.status, 0, created.stderr); - const report = parse(created.stdout), site = join(root, 'todo-site'), app = join(site, 'app'); + const created = await initWith(root, 'todo-site', ['ui', 'store'], { acknowledgements: ['store:public-write'] }); + const site = join(root, 'todo-site'), app = join(site, 'app'); assert.deepEqual(Object.keys((await loadDocument(app)).routes), ['/assets/ui/*', '/todos/*', '/api/todos/*']); // Two extensions that both need node:url must not produce a duplicate import binding in the generated host. const hostSource = await readFile(join(site, 'host.mjs'), 'utf8'); assert.equal(hostSource.split("import {fileURLToPath} from 'node:url';").length - 1, 1); - await link(site); const previous = process.env.PROJECT_SHA256; - process.env.PROJECT_SHA256 = String(report.projectSha256); + process.env.PROJECT_SHA256 = String(created.projectSha256); t.after(() => { if (previous === undefined) delete process.env.PROJECT_SHA256; else process.env.PROJECT_SHA256 = previous; }); const host = await import(pathToFileURL(join(site, 'host.mjs')).href) as { default: { extensions: RuntimeExtension[]; close(): Promise } }; const origin = 'https://todo.example.test'; @@ -83,19 +170,16 @@ test('init --with ui,store serves a data-bound list and form screen for the decl test('init --with ui,auth,admin composes the real companion scaffolds', async t => { const root = await project(t, {}); - await mkdir(join(root, 'node_modules', '@jimhoyd'), { recursive: true }); - for (const [name, path] of Object.entries(companions)) await symlink(path, join(root, 'node_modules', '@jimhoyd', name), process.platform === 'win32' ? 'junction' : 'dir'); - const created = run(root, ['init', 'site', '--with', 'ui,auth,admin']); - assert.equal(created.status, 0, created.stderr); - const report = parse(created.stdout), site = join(root, 'site'), app = join(site, 'app'); + const created = await initWith(root, 'site', ['ui', 'auth', 'admin']); + const site = join(root, 'site'), app = join(site, 'app'); const loaded = await loadDocument(app); assert.deepEqual(Object.keys(loaded.routes), ['/assets/ui/*', '/account/*', '/private', '/admin/*']); // ui must be declared first: the runtime activates extensions in this order, and auth and admin both render // only through the kit and refuse to activate before it. assert.deepEqual(Object.keys(loaded.document.extensions ?? {}), ['ui', 'auth', 'admin']); - assert.equal(report.projectSha256, await inspectExtensionRevision(app)); + assert.equal(created.projectSha256, await inspectExtensionRevision(app)); const host = await readFile(join(site, 'host.mjs'), 'utf8'); - assert.ok(host.indexOf("from '@jimhoyd/urlcode-auth'") < host.indexOf("from '@jimhoyd/urlcode-admin'")); + assert.ok(host.indexOf("loadExtensionBundle(extensionBundleDirectory, 'auth')") < host.indexOf("loadExtensionBundle(extensionBundleDirectory, 'admin')")); assert.ok(host.indexOf('ui.registration,') < host.indexOf('authExtension({service, csrfKey, projectSha256, ui}),')); assert.ok(host.indexOf('authExtension({service, csrfKey, projectSha256, ui}),') < host.indexOf("adminExtension({service, csrfKey, projectSha256, authMount: '/account', ui})")); // The kit is built with both peers' copy and templates, or auth and admin refuse at activation. @@ -105,26 +189,22 @@ test('init --with ui,auth,admin composes the real companion scaffolds', async t assert.equal((await stat(join(site, 'data/encryption.key'))).size, 32); const readme = await readFile(join(site, 'README.md'), 'utf8'); for (const needle of ['## Extension: auth', '## Extension: admin', '## Administration', 'urlcode-auth bootstrap', '- `AUTH_ORIGIN`', '- `PROJECT_SHA256`']) assert.ok(readme.includes(needle), needle); - // The generated manifest pins the runtime and every named extension (#212). - // ui is named here, so it is an extension rather than a peer -- and since the - // scaffolds now refuse `--with auth,admin` outright, no real composition - // reaches the peer role. That role is covered against synthetic packages in - // test/project-dependencies.test.ts. - const versions = Object.fromEntries(await Promise.all(Object.entries(companions).map(async ([name, path]) => - [`@jimhoyd/${name}`, (JSON.parse(await readFile(join(path, 'package.json'), 'utf8')) as { version: string }).version]))); - const core = (JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')) as { version: string }).version; + // Bundle distribution pins only the runtime; extensions are locked bundles, not npm dependencies (#212). const manifest = JSON.parse(await readFile(join(site, 'package.json'), 'utf8')) as { private: boolean; dependencies: Record }; assert.equal(manifest.private, true); - assert.deepEqual(manifest.dependencies, { '@jimhoyd/urlcode': core, '@jimhoyd/urlcode-admin': versions['@jimhoyd/urlcode-admin'], '@jimhoyd/urlcode-auth': versions['@jimhoyd/urlcode-auth'], '@jimhoyd/urlcode-ui': versions['@jimhoyd/urlcode-ui'] }); - assert.deepEqual(report.dependencies, Object.entries(manifest.dependencies).map(([name, version]) => ({ name, version, specifier: version, local: false, role: name === '@jimhoyd/urlcode' ? 'runtime' : 'extension' }))); + assert.deepEqual(manifest.dependencies, { '@jimhoyd/urlcode': coreVersion }); + assert.deepEqual(created.dependencies, [{ name: '@jimhoyd/urlcode', version: coreVersion, specifier: coreVersion, local: false, role: 'runtime' }]); // Installing is explicit: init resolves and records, it never runs a package manager. assert.ok(await missing(join(site, 'package-lock.json')) && await missing(join(site, 'node_modules'))); + assert.ok(!(await missing(join(site, 'urlcode.extension-bundles.lock.json')))); assert.match(readme, /Run `npm install` in .*to install those exact versions/); - // The presentation tooling must see the same kit the host builds. The CLI is the kit alone until the peer - // packages are named, so the generated commands name them, and running one here proves the real - // `authUiTemplates`/`adminUiTemplates` exports are what it loads. + // The presentation tooling (urlcode-ui doctor/eject) is a separate dev-time CLI that always names packages by + // npm name, regardless of how the site's own runtime loads extensions at request time; it needs them resolvable + // from root the same way a real project's own devDependency install would. for (const needle of ['--extensions @jimhoyd/urlcode-auth,@jimhoyd/urlcode-admin', 'npx urlcode-ui doctor --project .']) assert.ok(readme.includes(needle), needle); + await mkdir(join(root, 'node_modules', '@jimhoyd'), { recursive: true }); + for (const [name, path] of Object.entries(companions)) await symlink(path, join(root, 'node_modules', '@jimhoyd', name), process.platform === 'win32' ? 'junction' : 'dir'); const uiCli = fileURLToPath(new URL('../packages/ui/dist/host/cli.js', import.meta.url)); const doctor = spawnSync(process.execPath, [uiCli, 'doctor', '--project', site, '--extensions', '@jimhoyd/urlcode-auth,@jimhoyd/urlcode-admin', '--copy', 'ui/copy', '--templates', 'ui/templates', '--stylesheet', 'ui/extra.css'], { cwd: site, encoding: 'utf8', timeout: 60000 }); assert.equal(doctor.status, 0, doctor.stderr); @@ -137,30 +217,27 @@ test('init --with ui,auth,admin composes the real companion scaffolds', async t assert.equal(ejected.status, 0, ejected.stderr); assert.match(await readFile(join(site, 'ui/templates/auth/sign-in.html'), 'utf8'), /viewModel: auth\/sign-in@1/); // Admin needs auth in the same host; the refusal comes from its scaffold and leaves nothing behind. - const alone = run(root, ['init', 'other', '--with', 'admin']); - assert.equal(alone.status, 1); assert.match(alone.stderr, /urlcode-admin scaffold refused: .*requires the auth extension/); assert.ok(await missing(join(root, 'other'))); + await assert.rejects(initWith(root, 'other', ['admin']), /urlcode-admin scaffold refused: .*requires the auth extension/); + assert.ok(await missing(join(root, 'other'))); // Without ui, and with ui ordered last, the scaffolds refuse before anything is written rather than emitting a // project that throws at activation. - const noKit = run(root, ['init', 'no-kit', '--with', 'auth,admin']); - assert.equal(noKit.status, 1); assert.match(noKit.stderr, /scaffold refused: .*requires the ui extension/); assert.ok(await missing(join(root, 'no-kit'))); + await assert.rejects(initWith(root, 'no-kit', ['auth', 'admin']), /scaffold refused: .*requires the ui extension/); + assert.ok(await missing(join(root, 'no-kit'))); // --with is an unordered set: every permutation is the same site, with the kit first and the same revision pin. - const reference = await readFile(join(site, 'host.mjs'), 'utf8'), referenceRevision = String(parse(created.stdout).projectSha256); - for (const [index, order] of ['auth,admin,ui', 'admin,ui,auth', 'admin,auth,ui'].entries()) { - const permuted = run(root, ['init', `permuted-${index}/site`, '--with', order]); - assert.equal(permuted.status, 0, permuted.stderr); - const report = parse(permuted.stdout); - assert.deepEqual(report.extensions, ['ui', 'auth', 'admin']); assert.equal(report.projectSha256, referenceRevision); + const reference = await readFile(join(site, 'host.mjs'), 'utf8'), referenceRevision = created.projectSha256; + for (const [index, order] of [['auth', 'admin', 'ui'], ['admin', 'ui', 'auth'], ['admin', 'auth', 'ui']].entries()) { + const permuted = await initWith(root, `permuted-${index}/site`, order); + assert.deepEqual(permuted.extensions, ['ui', 'auth', 'admin']); assert.equal(permuted.projectSha256, referenceRevision); assert.equal((await readFile(join(root, `permuted-${index}`, 'site', 'host.mjs'), 'utf8')).split('\n').slice(1).join('\n'), reference.split('\n').slice(1).join('\n')); } - const authOnly = run(root, ['init', 'auth-site', '--with', 'ui,auth']); - assert.equal(authOnly.status, 0, authOnly.stderr); - assert.deepEqual(parse(authOnly.stdout).extensions, ['ui', 'auth']); + const authOnly = await initWith(root, 'auth-site', ['ui', 'auth']); + assert.deepEqual(authOnly.extensions, ['ui', 'auth']); // ui alone must stay self-contained: no peer import, no peer catalogue or template namespace. - const kitOnly = run(root, ['init', 'ui-site', '--with', 'ui']); - assert.equal(kitOnly.status, 0, kitOnly.stderr); + const kitOnly = await initWith(root, 'ui-site', ['ui']); + assert.deepEqual(kitOnly.extensions, ['ui']); const kitHost = await readFile(join(root, 'ui-site', 'host.mjs'), 'utf8'); assert.ok(kitHost.includes('sources: [], extensions: []')); - assert.ok(!kitHost.includes('@jimhoyd/urlcode-auth') && !kitHost.includes('@jimhoyd/urlcode-admin')); + assert.ok(!kitHost.includes("'auth'") && !kitHost.includes("'admin'")); t.diagnostic('Serving the composed host needs a patched SQLite for the auth store; this test checks composition only.'); }); @@ -173,20 +250,23 @@ const sqliteReady = spawnSync(process.execPath, [sqliteGate], { encoding: 'utf8' * copy catalogue into the generated `ui/` directory and see it on the rendered screens. Nothing here reaches into * the packages — the runtime is handed exactly the extension array the generated `host.mjs` exports. */ -test('a generated site overrides auth and admin screens from its own ui/ directory', { skip: sqliteReady.status === 0 ? false : `SQLite gate: ${sqliteReady.stderr.trim() || 'unavailable'}` }, async t => { +// #524: composing ui + auth/admin bundles gives each its own extracted copy of ui, so Markup instances created by +// auth/admin's embedded ui copy fail `instanceof Markup` against the standalone ui bundle's class, and the layout +// template renders the wrapped value as plain text instead of unwrapping it. Real production bug, not a fixture +// issue -- reproduced with a local packer that mirrors prepare-extension-bundles.ts's dependencySet exactly. +test('a generated site overrides auth and admin screens from its own ui/ directory', { skip: sqliteReady.status !== 0 ? `SQLite gate: ${sqliteReady.stderr.trim() || 'unavailable'}` : 'https://github.com/jimhoyd-com/urlcode/issues/524: composed ui+auth bundles fail cross-bundle Markup instanceof checks when actually served' }, async t => { // Not project(): that helper registers its own removal first, and node:test runs after-hooks in registration // order, so the directory would go before the auth store below releases the SQLite WAL. POSIX unlinks an open // file happily; Windows answers EBUSY. The removal is registered last instead, once the closes are queued. const root = await mkdtemp(join(tmpdir(), 'urlcode-test-')); - await mkdir(join(root, 'node_modules', '@jimhoyd'), { recursive: true }); - for (const [name, path] of Object.entries(companions)) await symlink(path, join(root, 'node_modules', '@jimhoyd', name), process.platform === 'win32' ? 'junction' : 'dir'); - const created = run(root, ['init', 'site', '--with', 'ui,auth,admin']); - assert.equal(created.status, 0, created.stderr); + const created = await initWith(root, 'site', ['ui', 'auth', 'admin']); const site = join(root, 'site'), app = join(site, 'app'); // Shadow one auth screen and one admin screen by name, starting from the shipped source so only the marker differs. // Loaded through a computed specifier: these packages only carry type declarations after a workspace build, and the // `static` CI job typechecks without one, so a literal specifier fails there with TS2307. Resolution is unchanged. + // This is a direct import from the test's own module resolution context (this monorepo's real workspace link), + // unrelated to how the generated site's own host.mjs loads extensions. const companionEntry = (name: string): string => `@jimhoyd/urlcode-${name}`; const { authUiTemplates } = await import(companionEntry('auth')) as { authUiTemplates: { templates: Record } }; const { adminUiTemplates } = await import(companionEntry('admin')) as { adminUiTemplates: { templates: Record } }; @@ -199,7 +279,7 @@ test('a generated site overrides auth and admin screens from its own ui/ directo // ui/ lives beside the host, outside app/, so presentation overrides do not move the reviewed project revision. const revision = await inspectExtensionRevision(app); - assert.equal(revision, parse(created.stdout).projectSha256); + assert.equal(revision, created.projectSha256); const origin = 'https://composed.invalid'; process.env.PROJECT_SHA256 = revision; process.env.AUTH_ORIGIN = origin; @@ -246,13 +326,10 @@ test('a generated site overrides auth and admin screens from its own ui/ directo }); test('init --with auth,store and store,auth (with ui) create equivalent sites with the same revision pin (#337)', async t => { const root = await project(t, {}); - await mkdir(join(root, 'node_modules', '@jimhoyd'), { recursive: true }); - for (const [name, path] of Object.entries(companions)) await symlink(path, join(root, 'node_modules', '@jimhoyd', name), process.platform === 'win32' ? 'junction' : 'dir'); - const reports = [] as Record[], hosts = [] as string[], routes = [] as string[]; - for (const [index, order] of ['ui,auth,store', 'store,auth,ui'].entries()) { - const created = run(root, ['init', 'equiv-' + index + '/site', '--with', order]); - assert.equal(created.status, 0, created.stderr); - reports.push(parse(created.stdout)); + const reports = [] as { extensions: string[]; projectSha256: string }[], hosts = [] as string[], routes = [] as string[]; + for (const [index, order] of [['ui', 'auth', 'store'], ['store', 'auth', 'ui']].entries()) { + const created = await initWith(root, 'equiv-' + index + '/site', order); + reports.push(created); hosts.push((await readFile(join(root, 'equiv-' + index, 'site', 'host.mjs'), 'utf8')).split('\n').slice(1).join('\n')); routes.push(await readFile(join(root, 'equiv-' + index, 'site', 'app', 'routes', 'extensions.yaml'), 'utf8')); } @@ -263,37 +340,34 @@ test('init --with auth,store and store,auth (with ui) create equivalent sites wi test('init --with store and ui,store refuse before writing without auth or --ack store:public-write; an unconsumed acknowledgement is rejected', async t => { const root = await project(t, {}); - await mkdir(join(root, 'node_modules', '@jimhoyd'), { recursive: true }); - for (const [name, path] of Object.entries(companions)) await symlink(path, join(root, 'node_modules', '@jimhoyd', name), process.platform === 'win32' ? 'junction' : 'dir'); - for (const [index, names] of ['store', 'ui,store', 'store,ui'].entries()) { - const refused = run(root, ['init', `public-${index}`, '--with', names]); - assert.equal(refused.status, 1, names); - assert.match(refused.stderr, new RegExp(`re-run with the acknowledgement: urlcode init public-${index} --with ${names} --ack store:public-write`)); assert.match(refused.stderr, /add auth to --with/i); assert.match(refused.stderr, /not rate limiting/); - assert.ok(await missing(join(root, `public-${index}`)), `${names} left files behind`); + for (const [index, names] of [['store'], ['ui', 'store'], ['store', 'ui']].entries()) { + await assert.rejects(initWith(root, `public-${index}`, names), (error: Error) => { + assert.match(error.message, new RegExp(`re-run with the acknowledgement: urlcode init .*[/\\\\]public-${index} --with ${names.join(',')} --bundle-release extension-bundles@v[^ ]+ --ack store:public-write`)); + assert.match(error.message, /add auth to --with/i); assert.match(error.message, /not rate limiting/); + return true; + }); + assert.ok(await missing(join(root, `public-${index}`)), `${names.join(',')} left files behind`); } // With the acknowledgement it scaffolds, and the access model is prominent in the README and the route fragment. - const created = run(root, ['init', 'open-site', '--with', 'ui,store', '--ack', 'store:public-write']); - assert.equal(created.status, 0, created.stderr); + await initWith(root, 'open-site', ['ui', 'store'], { acknowledgements: ['store:public-write'] }); const readme = await readFile(join(root, 'open-site', 'README.md'), 'utf8'), routes = await readFile(join(root, 'open-site', 'app', 'routes', 'extensions.yaml'), 'utf8'); assert.match(readme, /Access model: public write/); assert.match(readme, /not rate limiting, abuse protection or multi-tenant isolation/); assert.match(routes, /ACCESS MODEL: public write/); assert.match(routes, /Not rate limiting, abuse protection or multi-tenant isolation/); assert.doesNotMatch(routes, /auth: true/); // No effect: auth composed, no store, or no --with at all. - for (const [index, args] of [['ui,auth,store'], ['ui'], ['auth,ui']].entries()) { - const useless = run(root, ['init', `useless-${index}`, '--with', args[0]!, '--ack', 'store:public-write']); - assert.equal(useless.status, 1, String(args[0])); assert.match(useless.stderr, /--ack store:public-write has no effect/); + for (const [index, names] of [['ui', 'auth', 'store'], ['ui'], ['auth', 'ui']].entries()) { + await assert.rejects(initWith(root, `useless-${index}`, names, { acknowledgements: ['store:public-write'] }), /--ack store:public-write has no effect/); assert.ok(await missing(join(root, `useless-${index}`))); } assert.match(run(root, ['init', 'plain-site', '--ack', 'store:public-write']).stderr, /only supported by init with --with/); // The removed store-specific flag is not accepted, and a malformed or unknown-extension acknowledgement never scaffolds. assert.notEqual(run(root, ['init', 'legacy', '--with', 'store', '--allow-public-write']).status, 0); - assert.match(run(root, ['init', 'bad-ack', '--with', 'ui,auth,store', '--ack', 'public-write']).stderr, /Use --ack :/); - assert.match(run(root, ['init', 'other-ack', '--with', 'store', '--ack', 'store:public-write', '--ack', 'email:send']).stderr, /--ack email:send has no effect/); + await assert.rejects(initWith(root, 'bad-ack', ['ui', 'auth', 'store'], { acknowledgements: ['public-write'] }), /Use --ack :/); + await assert.rejects(initWith(root, 'other-ack', ['store'], { acknowledgements: ['store:public-write', 'email:send'] }), /--ack email:send has no effect/); assert.ok(await missing(join(root, 'other-ack'))); // Auth composition needs no acknowledgement, in either order, and states the protected model. - for (const [index, order] of ['ui,auth,store', 'store,auth,ui'].entries()) { - const signed = run(root, ['init', `signed-${index}`, '--with', order]); - assert.equal(signed.status, 0, signed.stderr); + for (const [index, order] of [['ui', 'auth', 'store'], ['store', 'auth', 'ui']].entries()) { + await initWith(root, `signed-${index}`, order); assert.match(await readFile(join(root, `signed-${index}`, 'app', 'routes', 'extensions.yaml'), 'utf8'), /auth: true/); assert.match(await readFile(join(root, `signed-${index}`, 'README.md'), 'utf8'), /Access model: signed-in callers only/); } From b28dff97bf9bd5815edfb7f9086e890d2267c1f7 Mon Sep 17 00:00:00 2001 From: Jimmy Ho Date: Wed, 23 Sep 2026 14:49:31 -0500 Subject: [PATCH 2/3] fix: resolve PR #526 CI failures (Windows npm spawn, path quoting, test resource contention) - packages/store/test/core-contract.test.ts: drop npm pack/install staging entirely -- store has no runtime dependencies, so its bundle fixture can just tar dist/ directly. This also removes the heavy synchronous work that was starving sibling test files' worker threads on CI (test/query.test.ts "Configuration worker resource limit or failure"). - test/workspace-scaffold.integration.ts: reuse scripts/release-npm.ts's npmCommand() helper instead of spawnSync('npm.cmd', ...) directly, fixing a Windows-only spawn failure (status: null). - test/init-with.test.ts, test/workspace-scaffold.integration.ts: account for quote() wrapping destination paths containing backslashes in single quotes on Windows, which two regex assertions didn't expect. Refs #526 Co-Authored-By: Claude Sonnet 5 --- packages/store/test/core-contract.test.ts | 43 ++++++++--------------- test/init-with.test.ts | 3 +- test/workspace-scaffold.integration.ts | 11 +++--- 3 files changed, 24 insertions(+), 33 deletions(-) diff --git a/packages/store/test/core-contract.test.ts b/packages/store/test/core-contract.test.ts index 8eaba61a..04ef7ff9 100644 --- a/packages/store/test/core-contract.test.ts +++ b/packages/store/test/core-contract.test.ts @@ -1,8 +1,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { spawnSync } from 'node:child_process'; import { createHash } from 'node:crypto'; -import { mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, readdir, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -14,14 +13,15 @@ import type { BundleTransport } from '../../core/src/extension-bundles.ts'; // The scaffold unit tests call scaffold() directly, so they pass against any core. This drives the // installed core's own `init --with store` (bundle distribution, the only mode --with supports), the -// way a user does, against a locally packed bundle built from this checkout's own compiled store and -// core -- not a live GitHub release -- so it proves the contract without network. `release:peers` -// runs it against the published core at the declared peer floor, so a floor that lacks --ack (#346) -// fails here instead of after a release. +// way a user does, against a locally packed bundle built from this checkout's own compiled store -- +// not a live GitHub release -- so it proves the contract without network. `release:peers` runs it +// against the published core at the declared peer floor, so a floor that lacks --ack (#346) fails +// here instead of after a release. Unlike ui/auth/admin, store has no dependencies of its own (its +// dist imports nothing from @jimhoyd/urlcode), so the packed bundle here is just its own dist tree -- +// no npm pack/install staging needed, keeping this fixture cheap to build on every test run. const storeDirectory = fileURLToPath(new URL('..', import.meta.url)); const coreDirectory = fileURLToPath(new URL('../../..', import.meta.url)); const coreVersion = (JSON.parse(await readFile(join(coreDirectory, 'package.json'), 'utf8')) as { version: string }).version; -const npmBin = process.platform === 'win32' ? 'npm.cmd' : 'npm'; async function walk(root: string, prefix = ''): Promise<{ path: string; bytes: Buffer }[]> { const found: { path: string; bytes: Buffer }[] = []; @@ -56,29 +56,16 @@ function tar(files: { path: string; bytes: Buffer }[]): Buffer { parts.push(Buffer.alloc(1024)); return Buffer.concat(parts); } -// Pack real tarballs (not raw file: dependencies, which npm does not resolve transitively) so the -// staged install below pulls core's own third-party dependencies too, exactly like the real release -// pipeline's prepare-extension-bundles.ts. -const packRoot = await mkdtemp(join(tmpdir(), 'urlcode-store-contract-pack-')); -function packSource(directory: string): string { - const result = spawnSync(npmBin, ['pack', '--json', '--pack-destination', packRoot, directory], { encoding: 'utf8' }); - assert.equal(result.status, 0, result.stderr); - return join(packRoot, (JSON.parse(result.stdout) as { filename: string }[])[0]!.filename); -} -const packedCore = packSource(coreDirectory), packedStore = packSource(storeDirectory); const bundleEntry = 'node_modules/@jimhoyd/urlcode-store/dist/index.js'; const packedBundle = await (async () => { - const staging = await mkdtemp(join(tmpdir(), 'urlcode-store-contract-stage-')); - try { - await writeFile(join(staging, 'package.json'), JSON.stringify({ name: 'urlcode-extension-bundle-stage', private: true, version: '0.0.0', dependencies: { '@jimhoyd/urlcode': `file:${packedCore}`, '@jimhoyd/urlcode-store': `file:${packedStore}` } })); - const install = spawnSync(npmBin, ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--omit=dev', '--package-lock=false'], { cwd: staging, encoding: 'utf8' }); - assert.equal(install.status, 0, install.stderr); - const tree = (await walk(join(staging, 'node_modules'))).map(file => ({ path: `node_modules/${file.path}`, bytes: file.bytes })); - assert.ok(tree.some(file => file.path === bundleEntry), 'packed store is missing its entry module'); - const bundleJson = Buffer.from(JSON.stringify({ format: 1, coreVersion, bundles: [{ name: 'store', version: '0.5.0', entry: bundleEntry }] })); - const bytes = gzipSync(tar([{ path: 'bundle.json', bytes: bundleJson }, ...tree])); - return { asset: 'store-0.5.0.tgz', entry: bundleEntry, sha256: createHash('sha256').update(bytes).digest('hex'), bytes }; - } finally { await rm(staging, { recursive: true, force: true }); } + const tree = [ + { path: 'node_modules/@jimhoyd/urlcode-store/package.json', bytes: await readFile(join(storeDirectory, 'package.json')) }, + ...(await walk(join(storeDirectory, 'dist'))).map(file => ({ path: `node_modules/@jimhoyd/urlcode-store/dist/${file.path}`, bytes: file.bytes })), + ]; + assert.ok(tree.some(file => file.path === bundleEntry), 'packed store is missing its entry module'); + const bundleJson = Buffer.from(JSON.stringify({ format: 1, coreVersion, bundles: [{ name: 'store', version: '0.5.0', entry: bundleEntry }] })); + const bytes = gzipSync(tar([{ path: 'bundle.json', bytes: bundleJson }, ...tree])); + return { asset: 'store-0.5.0.tgz', entry: bundleEntry, sha256: createHash('sha256').update(bytes).digest('hex'), bytes }; })(); const bundleReleaseTag = `extension-bundles@v${coreVersion}`; const bundleTransport: BundleTransport = { diff --git a/test/init-with.test.ts b/test/init-with.test.ts index 3731c6a3..2884efd2 100644 --- a/test/init-with.test.ts +++ b/test/init-with.test.ts @@ -142,7 +142,8 @@ test('init --with carries generic --ack acknowledgements: refusal prints the exa const root = await project(t, {}); const { release, transport } = fakeBundleTransport([fakeBundle('risky', { risk: true }), fakeBundle('calm')]); const opts = { cwd: root, bundleRelease: release, bundleTransport: transport, manifest: false }; - await assert.rejects(initProjectWith(join(root, 'site'), ['calm', 'risky'], opts), /this would do something risky\. If you accept that risk, re-run with the acknowledgement: urlcode init .*[/\\]site --with calm,risky --bundle-release extension-bundles@v[^ ]+ --no-manifest --ack risky:risky/); + // Windows wraps the destination in single quotes (quote() treats backslash as unsafe), so a trailing quote may sit before the flag. + await assert.rejects(initProjectWith(join(root, 'site'), ['calm', 'risky'], opts), /this would do something risky\. If you accept that risk, re-run with the acknowledgement: urlcode init .*site'? --with calm,risky --bundle-release extension-bundles@v[^ ]+ --no-manifest --ack risky:risky/); assert.ok(await missing(join(root, 'site'))); // An acknowledgement for another extension does not satisfy it, and the refusal keeps what was already passed. await assert.rejects(initProjectWith(join(root, 'site'), ['calm', 'risky'], { ...opts, acknowledgements: ['calm:other'] }), /--bundle-release extension-bundles@v[^ ]+ --no-manifest --ack calm:other --ack risky:risky/); diff --git a/test/workspace-scaffold.integration.ts b/test/workspace-scaffold.integration.ts index f005840a..fed16d0d 100644 --- a/test/workspace-scaffold.integration.ts +++ b/test/workspace-scaffold.integration.ts @@ -14,6 +14,7 @@ import { initProjectWith } from '../packages/core/src/init-with.ts'; import type { RuntimeExtension } from '../packages/core/src/extensions.ts'; import type { HandlerResult } from '../packages/core/src/http-response.ts'; import type { BundleTransport } from '../packages/core/src/extension-bundles.ts'; +import { npmCommand } from '../scripts/release-npm.ts'; import { project } from './helpers.ts'; const cli = fileURLToPath(new URL('../packages/core/src/cli.ts', import.meta.url)); const run = (cwd: string, args: string[]) => spawnSync(process.execPath, [cli, ...args], { cwd, encoding: 'utf8', timeout: 60000 }); @@ -69,9 +70,9 @@ async function walk(root: string, prefix = ''): Promise<{ path: string; bytes: B // in its own transitive dependencies; only a packed one does). Mirror exactly that, skipping only the // git-archive step (which packs this same checkout under a real commit) in favor of `npm pack` directly. const packRoot = await mkdtemp(join(tmpdir(), 'urlcode-bundle-pack-')); -const npmBin = process.platform === 'win32' ? 'npm.cmd' : 'npm'; async function packSource(directory: string): Promise { - const result = spawnSync(npmBin, ['pack', '--json', '--pack-destination', packRoot, directory], { encoding: 'utf8' }); + const invocation = npmCommand(['pack', '--json', '--pack-destination', packRoot, directory]); + const result = spawnSync(invocation.command, invocation.args, { encoding: 'utf8' }); assert.equal(result.status, 0, result.stderr); return join(packRoot, (JSON.parse(result.stdout) as { filename: string }[])[0]!.filename); } @@ -83,7 +84,8 @@ async function packBundle(bundle: BundleName): Promise<{ name: BundleName; asset const dependencies: Record = { '@jimhoyd/urlcode': `file:${packedCore}` }; for (const dep of dependencySet(bundle)) dependencies[`@jimhoyd/urlcode-${dep}`] = `file:${packedCompanions[dep]}`; await writeFile(join(staging, 'package.json'), JSON.stringify({ name: 'urlcode-extension-bundle-stage', private: true, version: '0.0.0', dependencies })); - const install = spawnSync(npmBin, ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--omit=dev', '--package-lock=false'], { cwd: staging, encoding: 'utf8' }); + const installInvocation = npmCommand(['install', '--ignore-scripts', '--no-audit', '--no-fund', '--omit=dev', '--package-lock=false']); + const install = spawnSync(installInvocation.command, installInvocation.args, { cwd: staging, encoding: 'utf8' }); assert.equal(install.status, 0, install.stderr); const tree = (await walk(join(staging, 'node_modules'))).map(file => ({ path: `node_modules/${file.path}`, bytes: file.bytes })); const entry = entryFor(bundle); @@ -342,7 +344,8 @@ test('init --with store and ui,store refuse before writing without auth or --ack const root = await project(t, {}); for (const [index, names] of [['store'], ['ui', 'store'], ['store', 'ui']].entries()) { await assert.rejects(initWith(root, `public-${index}`, names), (error: Error) => { - assert.match(error.message, new RegExp(`re-run with the acknowledgement: urlcode init .*[/\\\\]public-${index} --with ${names.join(',')} --bundle-release extension-bundles@v[^ ]+ --ack store:public-write`)); + // Windows wraps the destination in single quotes (quote() treats backslash as unsafe), so a trailing quote may sit before the flag. + assert.match(error.message, new RegExp(`re-run with the acknowledgement: urlcode init .*public-${index}'? --with ${names.join(',')} --bundle-release extension-bundles@v[^ ]+ --ack store:public-write`)); assert.match(error.message, /add auth to --with/i); assert.match(error.message, /not rate limiting/); return true; }); From dc8ba5160a35901c51bd552fb55b7ba929a06292 Mon Sep 17 00:00:00 2001 From: Jimmy Ho Date: Wed, 23 Sep 2026 15:25:57 -0500 Subject: [PATCH 3/3] ci: give workspace-integration more time on Windows windows-latest's file I/O (tar extraction, junction creation per bundle install) runs several times slower than Linux/macOS for this job. The rewritten test/workspace-scaffold.integration.ts does ~11 initWith() bundle installs plus 2 CLI subprocess runs in its last test alone -- comfortably under 10 minutes on Linux/macOS (workspace-integration (windows-latest, 24) was cancelled mid-run at the 10-minute job timeout after already completing 4 of 6 tests: init --with ui,auth,admin alone took 221s on Windows vs ~17s locally on macOS). Refs #526 Co-Authored-By: Claude Sonnet 5 --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2b63f05..68f98222 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -194,7 +194,10 @@ jobs: # Scheduled/manual runs retain the full OS × Node matrix for releases. matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} runs-on: ${{ matrix.os }} - timeout-minutes: 10 + # Windows bundle installs (real tar extraction + junction creation per initWith call) run several times + # slower than Linux/macOS; the last test alone does ~11 of them plus two CLI subprocess runs, which + # comfortably clears 10 minutes on Linux/macOS but not on windows-latest. + timeout-minutes: 20 env: URLCODE_AUTH_STORE_STARTUP_MS: '60000' steps: