diff --git a/packages/cdktn/src/asset-staging.ts b/packages/cdktn/src/asset-staging.ts index 9e8c758a3..eb8e2249a 100644 --- a/packages/cdktn/src/asset-staging.ts +++ b/packages/cdktn/src/asset-staging.ts @@ -2,17 +2,29 @@ // SPDX-License-Identifier: MPL-2.0 import { Construct, IConstruct } from "constructs"; import * as crypto from "crypto"; -import { AssetHashType, AssetOptions, IAsset, IAssetPackaging } from "./assets"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; import { + AssetHashType, + AssetOptions, + IAsset, + IAssetBundler, + IAssetPackaging, +} from "./assets"; +import { + assetFilePackagingWithBundlerUnsupported, assetHashConflictingExcludeOptions, assetHashConflictingHashType, assetHashInvalid, assetHashTypeCustomRequiresHash, assetHashTypeUnknown, + assetStagingAlreadyStaged, + assetStagingBundlerOutputNotDirectory, } from "./errors"; import { CANONICAL_ASSET_HASHES } from "./features"; import { ExcludeIgnoreStrategy, IIgnoreStrategy } from "./ignore-strategy"; -import { hashPath } from "./private/fs"; +import { copySync, hashPath } from "./private/fs"; // A resolved hash is used verbatim as a path segment (see `TerraformAsset.path`), // so it may only contain characters that are always safe there. @@ -50,6 +62,61 @@ function hashCacheFor(root: IConstruct): Map { return cache; } +/** + * Eager-build scratch directories awaiting cleanup, swept on process exit or + * an interrupting signal. + * + * An `OUTPUT`-hash build runs in the constructor but is normally torn down in + * `stage()`. When the asset's stack is not synthesized this pass, `stage()` + * never runs, so the sweep reclaims what can be large trees (`node_modules`, + * build caches) that `os.tmpdir()` does not clear reliably. + */ +const strandedScratchDirs = new Set(); +let exitSweepInstalled = false; + +/** + * Synchronously remove every still-registered scratch directory. + */ +function sweepStrandedScratch(): void { + for (const stray of strandedScratchDirs) { + try { + fs.rmSync(stray, { recursive: true, force: true }); + } catch { + // Best-effort on the way out; nothing useful to do if it fails. + } + } + strandedScratchDirs.clear(); +} + +/** + * Track a scratch directory for removal if it survives to process exit. + * + * `exit` alone misses a Ctrl-C or a `kill` during synth — exactly when the + * largest trees (`node_modules`, build caches) are left behind — because Node + * does not run `exit` handlers on `SIGINT`/`SIGTERM`. The signal handlers + * sweep, then re-raise the signal so any other listeners (the CLI's own) still + * run, or the default disposition terminates the process if there are none. + */ +function registerStrandedScratch(dir: string): void { + strandedScratchDirs.add(dir); + if (!exitSweepInstalled) { + exitSweepInstalled = true; + // `exit` handlers must be synchronous; `rmSync` is. + process.once("exit", sweepStrandedScratch); + for (const signal of ["SIGINT", "SIGTERM"] as const) { + process.once(signal, () => { + sweepStrandedScratch(); + // Re-raise so the signal is not swallowed. `process.once` has already + // removed this listener, so the re-raise reaches whatever else is + // registered — the CLI's own SIGINT/SIGTERM handlers — or, if nothing + // is, the default disposition that terminates the process. Removing + // other listeners here would discard exactly those the app relies on. + process.kill(process.pid, signal); + }); + } + } +} + /** * Options for {@link AssetStaging}. */ @@ -91,32 +158,60 @@ export interface AssetStagingOptions extends AssetOptions { * @default - no extra hash */ readonly extraHash?: string; + + /** + * A bundler that builds the source into an artifact before staging. + * + * Under the default `SOURCE` hashing the build is deferred to `stage()` and + * stays skippable; `OUTPUT` hashing builds eagerly at construction time to + * hash the artifact, forgoing skippability. A bundler always produces a + * directory, so single-file packaging (`AssetType.FILE`) is rejected. + * + * @default - the source is staged verbatim, with no build step + */ + readonly bundler?: IAssetBundler; + + /** + * Identifier used in error messages, so they name the user-facing construct + * (e.g. the `TerraformAsset`) rather than this internal staging child. + * + * @default - the staging construct's own id + */ + readonly displayName?: string; } /** - * Resolves an asset's identity (`SOURCE`/`OUTPUT`/`CUSTOM` hashing, with - * `exclude`/`extraHash`) and stages it to disk. + * Resolves an asset's identity and stages its content to disk. * - * Hashing happens eagerly in the constructor; staging the content to - * `targetPath` only happens when `stage()` is called, which callers do from - * their own `onSynthesize` hook. This keeps the filesystem side effect in the - * one window where it is safe to run, and keeps this class skippable once a - * bundler is introduced. - * - * `SOURCE` and `OUTPUT` compute identically here: without a bundler, the - * "output" of an asset is its source verbatim. A future bundler changes what - * `OUTPUT` hashes, not this class. - * - * The source-tree walk behind `SOURCE`/`OUTPUT` is cached per synth (see - * {@link hashCachesByRoot}), so referencing the same asset from more than one - * resource or stack hashes it once. `ASSET_HASH_SALT_CONTEXT_KEY` folds an - * app-wide value into every computed hash, for bulk cache-busting across an - * entire tree rather than one asset's `extraHash`. + * The hash is available immediately after construction; the filesystem write + * is deferred to `stage()`, the one window where it is safe. Hashing is + * cached per synth (see {@link hashCachesByRoot}) so an asset referenced from + * several places is walked once. Without a bundler, `SOURCE` and `OUTPUT` both + * hash the source; with one they diverge, as `AssetStagingOptions.bundler` + * describes. */ export class AssetStaging extends Construct implements IAsset { private readonly sourcePath: string; + private readonly displayName: string; private readonly ignoreStrategy: IIgnoreStrategy; + private readonly hasExclusions: boolean; private readonly hashCache: Map; + private readonly bundler?: IAssetBundler; + + /** + * Output of an eager `OUTPUT`-hash build, carried to `stage()` for reuse. + * + * Undefined on every other path, where the build is deferred or absent. + */ + private eagerBuild?: { readonly scratch: string; readonly produced: string }; + + /** + * Set once `stage()` has run, so a second call cannot rebuild. An eager + * `OUTPUT` build is captured at construction and consumed by the first + * `stage()`; without this guard a second call would fall through to the + * deferred build path and stage bytes that no longer match the hash. + */ + private staged = false; public readonly packaging: IAssetPackaging; public readonly isDirectory: boolean; @@ -126,17 +221,30 @@ export class AssetStaging extends Construct implements IAsset { super(scope, id); this.sourcePath = props.sourcePath; + this.displayName = props.displayName ?? id; this.packaging = props.packaging; this.isDirectory = props.packaging.producesDirectory; this.hashCache = hashCacheFor(this.node.root); + this.bundler = props.bundler; if (props.exclude?.length && props.ignoreStrategy) { throw assetHashConflictingExcludeOptions(); } this.ignoreStrategy = props.ignoreStrategy ?? new ExcludeIgnoreStrategy(props.exclude ?? []); + // A custom strategy is assumed to exclude something; the built-in one only + // does with a non-empty `exclude`. Drives whether the bundler is handed a + // materialised filtered copy or the source directly. + this.hasExclusions = + props.ignoreStrategy !== undefined || !!props.exclude?.length; - this.assetHash = this.resolveAssetHash(id, props); + // A bundler always produces a directory, so packaging that cannot take a + // directory source would fail with an opaque EISDIR/EPERM at synth. + if (this.bundler && !this.packaging.acceptsDirectorySource) { + throw assetFilePackagingWithBundlerUnsupported(this.displayName); + } + + this.assetHash = this.resolveAssetHash(this.displayName, props); } private resolveAssetHash(id: string, props: AssetStagingOptions): string { @@ -165,41 +273,26 @@ export class AssetStaging extends Construct implements IAsset { const archive = this.packaging.omitsDirectoryEntries; const salt = this.node.tryGetContext(ASSET_HASH_SALT_CONTEXT_KEY); - // Only cacheable when the ignore strategy can summarize its behavior - // as a string (see `IIgnoreStrategy.cacheKey`); otherwise every call - // is treated as unique. - const cacheKey = - this.ignoreStrategy.cacheKey !== undefined - ? JSON.stringify({ - sourcePath: this.sourcePath, - canonical, - archive, - ignore: this.ignoreStrategy.cacheKey, - }) - : undefined; - - let baseHash = cacheKey ? this.hashCache.get(cacheKey) : undefined; - if (baseHash === undefined) { - baseHash = hashPath(this.sourcePath, { - canonical, - archive, - shouldExclude: (relativePath, isDirectory) => - this.ignoreStrategy.ignores({ relativePath, isDirectory }), - descendIntoExcludedDirectories: - this.ignoreStrategy.pruneExcludedDirectories === false, - }); - if (cacheKey) { - this.hashCache.set(cacheKey, baseHash); - } - } + // OUTPUT hashes the built artifact and so must build eagerly, forgoing + // skippability (#380); every other case hashes the source verbatim. + const baseHash = + this.bundler && assetHashType === AssetHashType.OUTPUT + ? this.hashOutput(canonical, archive) + : this.hashSource(canonical, archive); - if (!extraHash && !salt) { + // For SOURCE, folding in bundlerKey is the only way build identity + // reaches the hash, since the source tree cannot see the build. + const bundlerKey = this.bundler?.bundlerKey; + if (!extraHash && !salt && !bundlerKey) { return baseHash; } const folded = crypto.createHash("md5").update(baseHash); if (extraHash) { folded.update(extraHash); } + if (bundlerKey) { + folded.update(bundlerKey); + } if (salt) { folded.update(String(salt)); } @@ -212,15 +305,181 @@ export class AssetStaging extends Construct implements IAsset { } /** - * Write the staged content to `targetPath`. Called from the owning - * construct's `onSynthesize` hook, once the target path is known. - * @param targetPath - path the packaged result should be written to + * Hash the source tree, honoring the ignore strategy. + * + * Cached per synth so an asset referenced more than once is walked once. + */ + private hashSource(canonical: boolean, archive: boolean): string { + // Only cacheable when the ignore strategy can summarize its behavior + // as a string (see `IIgnoreStrategy.cacheKey`); otherwise every call + // is treated as unique. + const cacheKey = + this.ignoreStrategy.cacheKey !== undefined + ? JSON.stringify({ + sourcePath: this.sourcePath, + canonical, + archive, + ignore: this.ignoreStrategy.cacheKey, + }) + : undefined; + + const cached = cacheKey ? this.hashCache.get(cacheKey) : undefined; + if (cached !== undefined) { + return cached; + } + + const hash = hashPath(this.sourcePath, { + canonical, + archive, + shouldExclude: (relativePath, isDirectory) => + this.ignoreStrategy.ignores({ relativePath, isDirectory }), + descendIntoExcludedDirectories: + this.ignoreStrategy.pruneExcludedDirectories === false, + }); + if (cacheKey) { + this.hashCache.set(cacheKey, hash); + } + return hash; + } + + /** + * Build the bundler's output eagerly and hash the built artifact. + * + * The output is stashed for `stage()` to reuse, so the build is not + * repeated. It is hashed verbatim: `exclude` filters the source a bundler + * reads, not the artifact it produces. + */ + private hashOutput(canonical: boolean, archive: boolean): string { + const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "cdktn-bundle-")); + // Registered so the exit sweep reclaims it if stage() never runs for an + // unsynthesized stack; stage() removes and unregisters it otherwise. + registerStrandedScratch(scratch); + const produced = this.runBundle(scratch); + this.eagerBuild = { scratch, produced }; + return hashPath(produced, { canonical, archive }); + } + + /** + * Run the bundler against the filtered source and return its output. + * + * The bundler is handed an absolute `source` (its cwd is its own — docker + * `-w`, esbuild — so a relative path would resolve elsewhere) that has + * already had `exclude` applied. Materialising the filtered input here is + * what makes the bundler read exactly the tree the hash was taken over: the + * two would otherwise disagree, since `hashSource` honours `exclude` but a + * raw source hand-off does not. The returned path is validated to be a + * directory before packaging. + * + * @param scratch - caller-owned scratch directory to build within + */ + private runBundle(scratch: string): string { + const outputDir = path.join(scratch, "output"); + fs.mkdirSync(outputDir); + const produced = this.bundler!.bundle({ + source: this.filteredSource(scratch), + outputDir, + }); + if (!fs.existsSync(produced) || !fs.statSync(produced).isDirectory()) { + throw assetStagingBundlerOutputNotDirectory(this.displayName, produced); + } + return produced; + } + + /** + * Absolute path to the source the bundler should read. + * + * With no exclusions the resolved source is handed over directly. Otherwise + * the excluded source tree is materialised into the scratch directory, so + * the bundler sees the same file set the hash was taken over. + * + * @param scratch - caller-owned scratch directory to materialise into + */ + private filteredSource(scratch: string): string { + const absoluteSource = path.resolve(this.sourcePath); + if (!this.hasExclusions) { + return absoluteSource; + } + const input = path.join(scratch, "input"); + fs.mkdirSync(input); + copySync(absoluteSource, input, { + shouldExclude: (relativePath, isDirectory) => + this.ignoreStrategy.ignores({ relativePath, isDirectory }), + descendIntoExcludedDirectories: + this.ignoreStrategy.pruneExcludedDirectories === false, + }); + return input; + } + + /** + * Stage the asset's content to `targetPath`. + * + * Called from the owning construct's `onSynthesize` hook. Without a bundler + * the source is packaged verbatim; with one, the bundler's output is, built + * eagerly for `OUTPUT` hashing or deferred to here for `SOURCE`. */ public stage(targetPath: string): void { + // Stage exactly once. An eager OUTPUT build is captured at construction + // and consumed below; a second call would otherwise fall through to the + // deferred path and rebuild, staging bytes a non-deterministic bundler + // does not match its already-computed hash. + if (this.staged) { + throw assetStagingAlreadyStaged(this.displayName); + } + this.staged = true; + + if (!this.bundler) { + this.packaging.pack({ + source: this.sourcePath, + target: targetPath, + ignoreStrategy: this.ignoreStrategy, + }); + return; + } + + // `OUTPUT` hashing already built the artifact in the constructor; package + // that exact result and clean up its scratch, rather than building twice. + if (this.eagerBuild) { + const { scratch, produced } = this.eagerBuild; + this.eagerBuild = undefined; + try { + this.packBundlerOutput(produced, targetPath); + } finally { + this.cleanupScratch(scratch); + } + return; + } + + // `SOURCE` hashing defers the build to here. + const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "cdktn-bundle-")); + try { + const produced = this.runBundle(scratch); + this.packBundlerOutput(produced, targetPath); + } finally { + this.cleanupScratch(scratch); + } + } + + /** + * Remove a bundler scratch directory and drop it from the exit-sweep set. + */ + private cleanupScratch(scratch: string): void { + fs.rmSync(scratch, { recursive: true, force: true }); + strandedScratchDirs.delete(scratch); + } + + /** + * Package a bundler's output into `targetPath`, verbatim. + * + * The ignore strategy is deliberately not applied. `exclude` filters the + * source a bundler reads, not its product: an install-style bundler + * (`npm install --production`, `pip install -t`) writes exactly the + * dependency directory a user excludes from source, and re-applying the + * exclusion would ship an artifact missing those dependencies. + */ + private packBundlerOutput(produced: string, targetPath: string): void { this.packaging.pack({ - source: this.sourcePath, + source: produced, target: targetPath, - ignoreStrategy: this.ignoreStrategy, }); } } diff --git a/packages/cdktn/src/assets.ts b/packages/cdktn/src/assets.ts index 2fcfe80a1..ef0d50659 100644 --- a/packages/cdktn/src/assets.ts +++ b/packages/cdktn/src/assets.ts @@ -102,6 +102,16 @@ export interface IAssetPackaging { */ readonly producesDirectory: boolean; + /** + * Whether `pack` accepts a directory as its `source`. + * + * Independent of `producesDirectory` and `omitsDirectoryEntries`, both of + * which describe the output: a `tar.gz` packaging takes a directory source + * yet emits a single file. Bundler output is always a directory, so a + * packaging that is `false` here cannot stage it. + */ + readonly acceptsDirectorySource: boolean; + /** * Whether `pack` emits an artifact with no directory entries of its own — * only the ignore-strategy-aware source walk. `hashPath`'s `archive` frame @@ -116,9 +126,7 @@ export interface IAssetPackaging { readonly omitsDirectoryEntries: boolean; /** - * Perform the staging transformation, writing the packaged result to - * `options.target`. - * @param options - see {@link PackOptions} + * Write the packaged result to `options.target`. */ pack(options: PackOptions): void; } @@ -157,6 +165,7 @@ export interface PackOptions { class FilePackaging implements IAssetPackaging { public readonly extension = ""; public readonly producesDirectory = false; + public readonly acceptsDirectorySource = false; public readonly omitsDirectoryEntries = false; public pack(options: PackOptions): void { fs.copyFileSync(options.source, options.target); @@ -169,6 +178,7 @@ class FilePackaging implements IAssetPackaging { class DirectoryPackaging implements IAssetPackaging { public readonly extension = ""; public readonly producesDirectory = true; + public readonly acceptsDirectorySource = true; public readonly omitsDirectoryEntries = false; public pack(options: PackOptions): void { copySync(options.source, options.target, { @@ -188,6 +198,7 @@ class DirectoryPackaging implements IAssetPackaging { class ZipPackaging implements IAssetPackaging { public readonly extension = ".zip"; public readonly producesDirectory = false; + public readonly acceptsDirectorySource = true; public readonly omitsDirectoryEntries = true; public pack(options: PackOptions): void { archiveSync( @@ -225,6 +236,142 @@ export class AssetPackaging { private constructor() {} } +/** + * Options handed to an {@link IAssetBundler} when it runs. + * + * A struct rather than positional parameters: adding a field is additive, + * adding a method parameter is not, and `bundle` is called through JSII where + * that distinction is a breaking-change boundary. + */ +export interface BundleOptions { + /** + * Absolute path to the asset's source file or directory. The bundler reads + * from here and must not modify it. + * + * Exclusions (`exclude` / `ignoreStrategy`) are already applied: when any + * are configured this points at a filtered copy, not the original tree, so + * the bundler reads exactly the file set the asset hash was taken over. + */ + readonly source: string; + + /** + * A scratch directory the bundler may write into, owned and created by the + * caller. The bundler produces its output here (or in a subdirectory) and + * returns the directory that holds the finished artifact — see + * {@link IAssetBundler.bundle}. + */ + readonly outputDir: string; +} + +/** + * Transforms a source tree into a built artifact. Runs at synth, before the + * output is packaged and staged. + * + * This is the extension point for asset bundling: core ships no bundler. + * Docker, esbuild, pip, `go build`, and similar are an open-ended set that + * is not cloud-specific, so each lives in its own package and implements this + * one interface — the same way {@link IIgnoreStrategy} lets richer exclusion + * live outside core without core taking on a glob parser. A third party + * develops a bundler by implementing this interface and publishing it as a + * package; users pass an instance via the consuming construct's `bundler` + * option. + * + * `bundle` runs during the owning construct's `onSynthesize` hook and may + * touch the filesystem. Deferring it there keeps it skippable when the asset's + * stack is not being synthesized, which holds as long as the hash is taken + * over the source rather than the built output. + */ +export interface IAssetBundler { + /** + * A value identifying the build, folded into the asset hash. + * + * The source tree alone cannot see the build, so swapping a `node:18` base + * image for `node:20` would otherwise leave identity unchanged. A value + * capturing the build (e.g. `docker::`) closes that gap. + * + * Under `SOURCE` hashing this is the only channel by which the build reaches + * identity, so it must serialize every input that can move the output — + * base image, command, tool version, environment, arguments. Anything left + * out means a changed build silently reuses a stale artifact. {@link + * BundlerKey} builds one from an ordered set of parts so the format is not + * reinvented per bundler. + * + * Mirrors {@link IIgnoreStrategy.cacheKey}: omit it when the build cannot be + * summarized as a string, and fall back to `extraHash`. + * + * @default - the build does not contribute to the hash + */ + readonly bundlerKey?: string; + + /** + * Produce the artifact and return the directory holding it. + * + * Implementations write into `options.outputDir` and return it or a + * subdirectory, never writing back to `options.source`. The returned + * directory is then packaged as an unbundled source directory would be. + * Returning a file, or a path that does not exist, is rejected — the + * contract is a directory, and packaging always treats the result as one. + */ + bundle(options: BundleOptions): string; +} + +/** + * Builds an {@link IAssetBundler.bundlerKey} from an ordered set of parts. + * + * A `bundlerKey` has to serialize everything that can move a build's output; + * done ad hoc, every bundler invents its own delimiter and forgets an input + * differently. This gives the convention one implementation: parts are joined + * with a separator that is escaped where it appears in a value, so distinct + * inputs can never collide into the same key (`["a:b", "c"]` and + * `["a", "b:c"]` stay different). + * + * @example + * const key = BundlerKey.of("docker", image, command) + * .withEnv({ NODE_ENV: nodeEnv }) + * .toString(); + */ +export class BundlerKey { + /** + * Start a key from an ordered list of parts. + * + * Order is significant: it is part of what the key identifies. + */ + public static of(...parts: string[]): BundlerKey { + return new BundlerKey(parts); + } + + private static escape(part: string): string { + return part.replace(/\\/g, "\\\\").replace(/:/g, "\\:"); + } + + private constructor(private readonly parts: string[]) {} + + /** + * Append parts, returning a new key. + */ + public add(...parts: string[]): BundlerKey { + return new BundlerKey([...this.parts, ...parts]); + } + + /** + * Append `key=value` parts for a record, sorted by key so the result does + * not depend on property order. + */ + public withEnv(entries: Record): BundlerKey { + const kept = Object.keys(entries) + .sort() + .map((k) => `${k}=${entries[k]}`); + return new BundlerKey([...this.parts, ...kept]); + } + + /** + * Render the collected parts to the string passed as `bundlerKey`. + */ + public toString(): string { + return this.parts.map(BundlerKey.escape).join(":"); + } +} + /** * A staged artifact, ready to hand to an `IAssetPublisher`. * diff --git a/packages/cdktn/src/errors.ts b/packages/cdktn/src/errors.ts index c99fa3df1..e1e2b8330 100644 --- a/packages/cdktn/src/errors.ts +++ b/packages/cdktn/src/errors.ts @@ -95,12 +95,35 @@ Place a cdktf.json at the root of your project, or pass an absolute path. Learn `, ); +export const assetFilePackagingWithBundlerUnsupported = (id: string) => + new Error( + `TerraformAsset ${id} was configured with a 'bundler' and file packaging (AssetType.FILE). A bundler produces a directory of output, which cannot be staged as a single file. + +Use AssetType.ARCHIVE to zip the bundler's output, or AssetType.DIRECTORY to stage it as a tree. +Learn more about TerraformAsset: https://cdktn.io/docs/concepts/assets`, + ); + export const assetHashInvalid = (id: string, assetHash: string) => new Error( `TerraformAsset ${id} resolved an 'assetHash' of '${assetHash}', but it names the staged asset file and so may only contain letters, digits, '_', '.' and '-'. Learn more about TerraformAsset: https://cdktn.io/docs/concepts/assets`, ); +export const assetStagingBundlerOutputNotDirectory = ( + id: string, + produced: string, +) => + new Error( + `TerraformAsset ${id}'s bundler returned '${produced}', which is not a directory. A bundler must write its output into 'options.outputDir' and return that directory (or a subdirectory of it); the returned tree is then packaged. +Learn more about TerraformAsset: https://cdktn.io/docs/concepts/assets`, + ); + +export const assetStagingAlreadyStaged = (id: string) => + new Error( + `TerraformAsset ${id} was already staged. An AssetStaging stages exactly once — its bundler output is captured on the first call and cannot be rebuilt or restaged. +Learn more about TerraformAsset: https://cdktn.io/docs/concepts/assets`, + ); + export const dynamicBlockNotSupported = (_foreachExpression: string) => new Error( `We do not support directly resolving a TerraformDynamicBlock. Dynamic blocks are only supported on block attributes of resources, data sources, and providers. diff --git a/packages/cdktn/src/terraform-asset.ts b/packages/cdktn/src/terraform-asset.ts index d0e6a48eb..82e7f854c 100644 --- a/packages/cdktn/src/terraform-asset.ts +++ b/packages/cdktn/src/terraform-asset.ts @@ -7,6 +7,7 @@ import { AssetPackaging, AssetHashType, IAsset, + IAssetBundler, IAssetPackaging, } from "./assets"; import { AssetStaging } from "./asset-staging"; @@ -31,10 +32,9 @@ export interface TerraformAssetConfig { * How the `assetHash` is derived. * * `SOURCE` (the default) hashes the source path. `CUSTOM` uses the - * `assetHash` value verbatim and requires it to be set. `OUTPUT` also - * hashes the source path today — there is no bundling step yet, so the - * "output" of an asset is its source verbatim — but will hash the - * bundler's output once bundling is introduced. + * `assetHash` value verbatim and requires it to be set. `OUTPUT` hashes the + * source too, unless a `bundler` is set — then it hashes the bundler's + * built output, which forces an eager build (see `bundler`). * * If `assetHash` is set, this must be `undefined` or `AssetHashType.CUSTOM`. * @@ -58,6 +58,19 @@ export interface TerraformAssetConfig { * @default - no extra hash */ readonly extraHash?: string; + + /** + * A bundler that builds the source into an artifact before staging. + * + * Core ships no bundler; implement `IAssetBundler` or use one from a bundler + * package. Under the default `SOURCE` hashing the build is deferred to synth + * and stays skippable; `OUTPUT` hashing builds eagerly to hash the artifact. + * `AssetType.FILE` is rejected, since bundler output is always a directory. + * See `AssetStagingOptions.bundler`. + * + * @default - the source is staged verbatim, with no build step + */ + readonly bundler?: IAssetBundler; } export enum AssetType { @@ -128,24 +141,25 @@ export class TerraformAsset extends Construct implements IAsset { const inferredType = stat.isFile() ? AssetType.FILE : AssetType.DIRECTORY; this.type = config.type ?? inferredType; + // Validate the type against the source before staging, so an invalid + // combination is rejected here rather than after AssetStaging has already + // run an eager bundler build. + if (stat.isFile() !== (this.type === AssetType.FILE)) { + throw assetExpectsDirectory(id, config.path); + } + this.staging = new AssetStaging(this, "Staging", { sourcePath: this.sourcePath, + displayName: id, packaging: this.packaging, assetHash: config.assetHash, assetHashType: config.assetHashType, exclude: config.exclude, extraHash: config.extraHash, + bundler: config.bundler, }); this.assetHash = this.staging.assetHash; - if (stat.isFile() && this.type !== AssetType.FILE) { - throw assetExpectsDirectory(id, config.path); - } - - if (!stat.isFile() && this.type === AssetType.FILE) { - throw assetExpectsDirectory(id, config.path); - } - addCustomSynthesis(this, { onSynthesize: this._onSynthesize.bind(this), }); diff --git a/packages/cdktn/test/asset-staging.test.ts b/packages/cdktn/test/asset-staging.test.ts index 6c33932b3..6cf3b0301 100644 --- a/packages/cdktn/test/asset-staging.test.ts +++ b/packages/cdktn/test/asset-staging.test.ts @@ -284,6 +284,554 @@ describe("AssetStaging", () => { expect(zip.isDirectory).toBe(false); }); + describe("with a bundler", () => { + test("SOURCE hashing takes the hash over the source, not the bundler output", () => { + const withoutBundler = new AssetStaging(stack(), "plain", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + }); + const withBundler = new AssetStaging(stack(), "bundled", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + // Under the default SOURCE hashing, a bundler that produces entirely + // different bytes must not change the hash — identity is a function + // of the inputs, and the build is deferred (never runs here). + bundler: { + bundle: (opts) => { + fs.writeFileSync(path.join(opts.outputDir, "built.txt"), "output"); + return opts.outputDir; + }, + }, + }); + + expect(withBundler.assetHash).toEqual(withoutBundler.assetHash); + }); + + describe("OUTPUT hashing", () => { + test("takes the hash over the built output, differing from the source", () => { + const sourceHashed = new AssetStaging(stack(), "source", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + bundler: { + bundle: (opts) => { + fs.writeFileSync( + path.join(opts.outputDir, "built.txt"), + "output", + ); + return opts.outputDir; + }, + }, + }); + const outputHashed = new AssetStaging(stack(), "output", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + assetHashType: AssetHashType.OUTPUT, + bundler: { + bundle: (opts) => { + fs.writeFileSync( + path.join(opts.outputDir, "built.txt"), + "output", + ); + return opts.outputDir; + }, + }, + }); + + // The output hash reflects the built artifact, which the source hash + // (over srcDir, which has no built.txt) cannot equal. + expect(outputHashed.assetHash).not.toEqual(sourceHashed.assetHash); + }); + + test("tracks changes in the built output even when the source is unchanged", () => { + const buildA = new AssetStaging(stack(), "a", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + assetHashType: AssetHashType.OUTPUT, + bundler: { + bundle: (opts) => { + fs.writeFileSync(path.join(opts.outputDir, "built.txt"), "one"); + return opts.outputDir; + }, + }, + }); + const buildB = new AssetStaging(stack(), "b", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + assetHashType: AssetHashType.OUTPUT, + bundler: { + bundle: (opts) => { + fs.writeFileSync(path.join(opts.outputDir, "built.txt"), "two"); + return opts.outputDir; + }, + }, + }); + + // Same source, different output bytes -> different identity. This is + // exactly what SOURCE hashing cannot catch. + expect(buildA.assetHash).not.toEqual(buildB.assetHash); + }); + + test("builds once: the constructor builds, and stage() reuses that output", () => { + let buildCount = 0; + const staging = new AssetStaging(stack(), "staging", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + assetHashType: AssetHashType.OUTPUT, + bundler: { + bundle: (opts) => { + buildCount++; + fs.writeFileSync( + path.join(opts.outputDir, "built.txt"), + "output", + ); + return opts.outputDir; + }, + }, + }); + + // Built eagerly during construction, before stage(). + expect(buildCount).toBe(1); + + const targetPath = path.join(createTempDir(), "out"); + fs.mkdirSync(targetPath, { recursive: true }); + staging.stage(targetPath); + + // stage() packaged the eager build rather than building again. + expect(buildCount).toBe(1); + expect(fs.existsSync(path.join(targetPath, "built.txt"))).toBe(true); + }); + + test("cleans up the eagerly-built scratch directory after stage()", () => { + let observedOutputDir: string | undefined; + const staging = new AssetStaging(stack(), "staging", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + assetHashType: AssetHashType.OUTPUT, + bundler: { + bundle: (opts) => { + observedOutputDir = opts.outputDir; + fs.writeFileSync( + path.join(opts.outputDir, "built.txt"), + "output", + ); + return opts.outputDir; + }, + }, + }); + const targetPath = path.join(createTempDir(), "out"); + fs.mkdirSync(targetPath, { recursive: true }); + + staging.stage(targetPath); + + expect(observedOutputDir).toBeDefined(); + expect(fs.existsSync(observedOutputDir!)).toBe(false); + }); + + test("sweeps the scratch directory on process exit when stage() never runs", () => { + // The eager build happens in the constructor but is normally cleaned + // up in stage(). When stage() never runs (an unsynthesized stack), + // the process-exit hook is the safety net. Exercised in a child + // process against the compiled lib so a real `exit` fires; the child + // prints the scratch path it created, and the parent asserts it was + // swept. + const marker = path.join(createTempDir(), "scratch-path.txt"); + const script = ` + const fs = require("fs"); + const { AssetStaging, App, TerraformStack } = require(${JSON.stringify( + path.resolve(__dirname, "../lib"), + )}); + const stack = new TerraformStack(new App(), "s"); + new AssetStaging(stack, "staging", { + sourcePath: ${JSON.stringify(srcDir)}, + packaging: require(${JSON.stringify( + path.resolve(__dirname, "../lib"), + )}).AssetPackaging.DIRECTORY, + assetHashType: "output", + bundler: { + bundle: (opts) => { + fs.writeFileSync(${JSON.stringify(marker)}, opts.outputDir); + fs.writeFileSync(opts.outputDir + "/built.txt", "output"); + return opts.outputDir; + }, + }, + }); + // Intentionally never call stage(). + `; + require("child_process").execFileSync(process.execPath, ["-e", script]); + + const scratch = fs.readFileSync(marker, "utf-8"); + expect(scratch).toContain("cdktn-bundle-"); + expect(fs.existsSync(scratch)).toBe(false); + }); + }); + + test("stage() runs the bundler and packages its output, not the source", () => { + const staging = new AssetStaging(stack(), "staging", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + bundler: { + bundle: (opts) => { + expect(opts.source).toBe(srcDir); + fs.writeFileSync(path.join(opts.outputDir, "built.txt"), "output"); + return opts.outputDir; + }, + }, + }); + const targetPath = path.join(createTempDir(), "out"); + fs.mkdirSync(targetPath, { recursive: true }); + + staging.stage(targetPath); + + // The bundler's output is staged... + expect(fs.existsSync(path.join(targetPath, "built.txt"))).toBe(true); + // ...and the original source is not. + expect(fs.existsSync(path.join(targetPath, "a.txt"))).toBe(false); + }); + + test("exclude filters the source, not the bundler output", () => { + // `exclude` is source filtering: an install-style bundler produces the + // very directory a user excludes from source (e.g. node_modules), and + // re-applying the exclusion to the output would strip built content. + const staging = new AssetStaging(stack(), "staging", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + exclude: ["node_modules"], + bundler: { + bundle: (opts) => { + const built = path.join(opts.outputDir, "node_modules"); + fs.mkdirSync(built); + fs.writeFileSync(path.join(built, "dep.js"), "dependency"); + return opts.outputDir; + }, + }, + }); + const targetPath = path.join(createTempDir(), "out"); + fs.mkdirSync(targetPath, { recursive: true }); + + staging.stage(targetPath); + + // The bundler's node_modules survives — it is built output, not source. + expect( + fs.existsSync(path.join(targetPath, "node_modules", "dep.js")), + ).toBe(true); + }); + + test("bundlerKey changes the hash", () => { + const withoutKey = new AssetStaging(stack(), "without", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + bundler: { bundle: (opts) => opts.outputDir }, + }); + const withKey = new AssetStaging(stack(), "with", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + bundler: { + bundlerKey: "docker:node:20:npm run build", + bundle: (opts) => opts.outputDir, + }, + }); + const withDifferentKey = new AssetStaging(stack(), "different", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + bundler: { + bundlerKey: "docker:node:18:npm run build", + bundle: (opts) => opts.outputDir, + }, + }); + + expect(withKey.assetHash).not.toEqual(withoutKey.assetHash); + expect(withKey.assetHash).not.toEqual(withDifferentKey.assetHash); + }); + + test("FILE packaging with a bundler throws", () => { + fs.writeFileSync(path.join(srcDir, "single.txt"), "content"); + expect( + () => + new AssetStaging(stack(), "staging", { + sourcePath: path.join(srcDir, "single.txt"), + packaging: AssetPackaging.FILE, + bundler: { bundle: (opts) => opts.outputDir }, + }), + ).toThrow(/file packaging|AssetType\.FILE/i); + }); + + test("ARCHIVE packaging with a bundler is allowed", () => { + const staging = new AssetStaging(stack(), "staging", { + sourcePath: srcDir, + packaging: AssetPackaging.ZIP, + bundler: { + bundle: (opts) => { + fs.writeFileSync(path.join(opts.outputDir, "built.txt"), "output"); + return opts.outputDir; + }, + }, + }); + const targetPath = path.join(createTempDir(), "archive.zip"); + fs.mkdirSync(path.dirname(targetPath), { recursive: true }); + + expect(() => staging.stage(targetPath)).not.toThrow(); + expect(fs.existsSync(targetPath)).toBe(true); + }); + + test("a directory-source single-file packaging (e.g. tar.gz) with a bundler is allowed", () => { + // Rejection keys on `acceptsDirectorySource`, not on producing a + // directory or omitting directory entries. A tar.gz-style packaging + // takes a directory source, emits one file, and keeps directory + // entries (omitsDirectoryEntries: false) — it must not be rejected. + const tarLike: IAssetPackaging = { + extension: ".tar.gz", + producesDirectory: false, + acceptsDirectorySource: true, + omitsDirectoryEntries: false, + pack: (opts) => { + // Stand-in for a real tar writer; only needs to read a directory + // source and emit a single file. + fs.writeFileSync(opts.target, "tarball"); + }, + }; + + expect( + () => + new AssetStaging(stack(), "staging", { + sourcePath: srcDir, + packaging: tarLike, + bundler: { bundle: (opts) => opts.outputDir }, + }), + ).not.toThrow(); + }); + + test("a bundler may return a subdirectory of outputDir", () => { + const staging = new AssetStaging(stack(), "staging", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + bundler: { + bundle: (opts) => { + const dist = path.join(opts.outputDir, "dist"); + fs.mkdirSync(dist); + fs.writeFileSync(path.join(dist, "bundle.js"), "built"); + return dist; + }, + }, + }); + const targetPath = path.join(createTempDir(), "out"); + fs.mkdirSync(targetPath, { recursive: true }); + + staging.stage(targetPath); + + // Only the returned subdirectory's contents are staged. + expect(fs.existsSync(path.join(targetPath, "bundle.js"))).toBe(true); + }); + + test("the scratch directory is cleaned up after a successful bundle", () => { + let observedOutputDir: string | undefined; + const staging = new AssetStaging(stack(), "staging", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + bundler: { + bundle: (opts) => { + observedOutputDir = opts.outputDir; + fs.writeFileSync(path.join(opts.outputDir, "built.txt"), "output"); + return opts.outputDir; + }, + }, + }); + const targetPath = path.join(createTempDir(), "out"); + fs.mkdirSync(targetPath, { recursive: true }); + + staging.stage(targetPath); + + expect(observedOutputDir).toBeDefined(); + expect(fs.existsSync(observedOutputDir!)).toBe(false); + }); + + test("a throwing bundler propagates and still cleans up the scratch directory", () => { + let observedOutputDir: string | undefined; + const staging = new AssetStaging(stack(), "staging", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + bundler: { + bundle: (opts) => { + observedOutputDir = opts.outputDir; + throw new Error("build failed"); + }, + }, + }); + const targetPath = path.join(createTempDir(), "out"); + fs.mkdirSync(targetPath, { recursive: true }); + + expect(() => staging.stage(targetPath)).toThrow(/build failed/); + expect(observedOutputDir).toBeDefined(); + expect(fs.existsSync(observedOutputDir!)).toBe(false); + }); + + test("the bundler is handed an absolute source path", () => { + // A relative sourcePath (the common case: TerraformAsset stores one + // relative to process.cwd()) must still reach the bundler as absolute, + // since a bundler runs a tool with its own cwd. + const relativeSource = path.relative(process.cwd(), srcDir); + let observedSource: string | undefined; + const staging = new AssetStaging(stack(), "staging", { + sourcePath: relativeSource, + packaging: AssetPackaging.DIRECTORY, + bundler: { + bundle: (opts) => { + observedSource = opts.source; + fs.writeFileSync(path.join(opts.outputDir, "built.txt"), "output"); + return opts.outputDir; + }, + }, + }); + const targetPath = path.join(createTempDir(), "out"); + fs.mkdirSync(targetPath, { recursive: true }); + + staging.stage(targetPath); + + expect(observedSource).toBeDefined(); + expect(path.isAbsolute(observedSource!)).toBe(true); + }); + + test("exclude filters what the bundler reads, and the source is a copy", () => { + // With exclusions configured, the bundler must see a filtered copy — + // not the original tree — so it reads exactly the file set the hash was + // taken over. b.md is excluded, so it must be absent from what the + // bundler is handed. + let sawExcluded = true; + let sawIncluded = false; + let handedSource: string | undefined; + const staging = new AssetStaging(stack(), "staging", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + exclude: ["*.md"], + bundler: { + bundle: (opts) => { + handedSource = opts.source; + sawExcluded = fs.existsSync(path.join(opts.source, "b.md")); + sawIncluded = fs.existsSync(path.join(opts.source, "a.txt")); + fs.writeFileSync(path.join(opts.outputDir, "built.txt"), "output"); + return opts.outputDir; + }, + }, + }); + const targetPath = path.join(createTempDir(), "out"); + fs.mkdirSync(targetPath, { recursive: true }); + + staging.stage(targetPath); + + expect(sawExcluded).toBe(false); + expect(sawIncluded).toBe(true); + // The bundler read a materialised copy, never the original source dir. + expect(handedSource).not.toEqual(srcDir); + expect(handedSource).not.toEqual(path.resolve(srcDir)); + }); + + test("without exclusions the bundler reads the source directly", () => { + let handedSource: string | undefined; + const staging = new AssetStaging(stack(), "staging", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + bundler: { + bundle: (opts) => { + handedSource = opts.source; + fs.writeFileSync(path.join(opts.outputDir, "built.txt"), "output"); + return opts.outputDir; + }, + }, + }); + const targetPath = path.join(createTempDir(), "out"); + fs.mkdirSync(targetPath, { recursive: true }); + + staging.stage(targetPath); + + expect(handedSource).toEqual(path.resolve(srcDir)); + }); + + test("a bundler returning a non-directory throws", () => { + const staging = new AssetStaging(stack(), "staging", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + bundler: { + bundle: (opts) => { + const file = path.join(opts.outputDir, "artifact.js"); + fs.writeFileSync(file, "built"); + return file; + }, + }, + }); + const targetPath = path.join(createTempDir(), "out"); + fs.mkdirSync(targetPath, { recursive: true }); + + expect(() => staging.stage(targetPath)).toThrow(/not a directory/i); + }); + + test("a bundler returning a nonexistent path throws", () => { + const staging = new AssetStaging(stack(), "staging", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + bundler: { + bundle: (opts) => path.join(opts.outputDir, "does-not-exist"), + }, + }); + const targetPath = path.join(createTempDir(), "out"); + fs.mkdirSync(targetPath, { recursive: true }); + + expect(() => staging.stage(targetPath)).toThrow(/not a directory/i); + }); + + test("a second stage() throws instead of rebuilding (OUTPUT)", () => { + let buildCount = 0; + const staging = new AssetStaging(stack(), "staging", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + assetHashType: AssetHashType.OUTPUT, + bundler: { + bundle: (opts) => { + buildCount++; + fs.writeFileSync(path.join(opts.outputDir, "built.txt"), "output"); + return opts.outputDir; + }, + }, + }); + const first = path.join(createTempDir(), "out"); + fs.mkdirSync(first, { recursive: true }); + staging.stage(first); + + const second = path.join(createTempDir(), "out"); + fs.mkdirSync(second, { recursive: true }); + + // The eager build was consumed by the first stage(); a second would + // otherwise rebuild and stage bytes not matching the computed hash. + expect(() => staging.stage(second)).toThrow(/already staged/i); + expect(buildCount).toBe(1); + }); + + test("a second stage() throws even without a bundler", () => { + const staging = new AssetStaging(stack(), "staging", { + sourcePath: srcDir, + packaging: AssetPackaging.DIRECTORY, + }); + const first = path.join(createTempDir(), "out"); + fs.mkdirSync(first, { recursive: true }); + staging.stage(first); + + const second = path.join(createTempDir(), "out"); + fs.mkdirSync(second, { recursive: true }); + expect(() => staging.stage(second)).toThrow(/already staged/i); + }); + + test("errors name the caller's displayName, not the staging child id", () => { + fs.writeFileSync(path.join(srcDir, "single.txt"), "content"); + expect( + () => + new AssetStaging(stack(), "Staging", { + sourcePath: path.join(srcDir, "single.txt"), + packaging: AssetPackaging.FILE, + displayName: "MyAsset", + bundler: { bundle: (opts) => opts.outputDir }, + }), + ).toThrow(/TerraformAsset MyAsset/); + }); + }); + test("a custom packaging's own omitsDirectoryEntries decides the hash frame, not identity with AssetPackaging.ZIP", () => { const customZip: IAssetPackaging = { ...AssetPackaging.ZIP, diff --git a/packages/cdktn/test/assets.test.ts b/packages/cdktn/test/assets.test.ts index e952cea9e..cbdc42b33 100644 --- a/packages/cdktn/test/assets.test.ts +++ b/packages/cdktn/test/assets.test.ts @@ -1,6 +1,11 @@ // Copyright (c) HashiCorp, Inc // SPDX-License-Identifier: MPL-2.0 -import { TerraformHclModule, TerraformStack, Testing } from "../src"; +import { + BundlerKey, + TerraformHclModule, + TerraformStack, + Testing, +} from "../src"; import * as path from "path"; import { TerraformModuleAsset } from "../src/terraform-module-asset"; @@ -142,3 +147,43 @@ describe("createAssetsFromLocalModules", () => { expect(moduleOptionsFalse.source).toEqual(localSource); }); }); + +describe("BundlerKey", () => { + test("joins ordered parts, escaping the separator inside a part", () => { + // The colon inside "node:20" is escaped so it cannot be mistaken for a + // part boundary. + expect(BundlerKey.of("docker", "node:20", "npm run build").toString()).toBe( + "docker:node\\:20:npm run build", + ); + }); + + test("order is significant", () => { + expect(BundlerKey.of("a", "b").toString()).not.toBe( + BundlerKey.of("b", "a").toString(), + ); + }); + + test("escapes the separator so distinct inputs cannot collide", () => { + expect(BundlerKey.of("a:b", "c").toString()).not.toBe( + BundlerKey.of("a", "b:c").toString(), + ); + }); + + test("add appends parts", () => { + expect(BundlerKey.of("docker").add("node:20", "build").toString()).toBe( + BundlerKey.of("docker", "node:20", "build").toString(), + ); + }); + + test("withEnv sorts record entries so property order does not matter", () => { + const a = BundlerKey.of("base").withEnv({ B: "2", A: "1" }).toString(); + const b = BundlerKey.of("base").withEnv({ A: "1", B: "2" }).toString(); + expect(a).toBe(b); + }); + + test("different env values produce different keys", () => { + const dev = BundlerKey.of("build").withEnv({ NODE_ENV: "development" }); + const prod = BundlerKey.of("build").withEnv({ NODE_ENV: "production" }); + expect(dev.toString()).not.toBe(prod.toString()); + }); +}); diff --git a/packages/cdktn/test/canonical-asset-hash.test.ts b/packages/cdktn/test/canonical-asset-hash.test.ts index e58a862a3..03bde2a16 100644 --- a/packages/cdktn/test/canonical-asset-hash.test.ts +++ b/packages/cdktn/test/canonical-asset-hash.test.ts @@ -616,6 +616,27 @@ describe("TerraformAsset artifact layout derives from the packaging", () => { expect(asset.fileName).toBe("archive.zip"); expect(asset.path.endsWith("/archive.zip")).toBe(true); }); + + test("an invalid type is rejected before the bundler runs", () => { + let bundled = false; + expect( + () => + new TerraformAsset(stack(), "asset", { + path: srcDir, + type: AssetType.FILE, + assetHashType: AssetHashType.OUTPUT, + bundler: { + bundle: (opts) => { + bundled = true; + return opts.outputDir; + }, + }, + }), + ).toThrow(/directory/i); + + // The type/source mismatch is caught before staging, so no eager build ran. + expect(bundled).toBe(false); + }); }); describe("TerraformAsset stages inside the stack's own directory (#380)", () => {