feat: resolve placeholders from consumer dependency pins

docs/release.md

@@ -51,9 +51,11 @@ Two things worth knowing:
 ## Versioning
 
 What couples the CLI to a payload is the manifest's `schemaVersion` (currently
-`2`), not the package version. A CLI major supports exactly one, exported at the
-`./manifest` subpath. **Bumping it is a CLI major**, and the supporting CLI has
-to ship before any payload adopts it.
+`3`), not the package version. A CLI accepts a set of versions — `{2, 3}` today —
+exported at the `./manifest` subpath as `SUPPORTED_SCHEMA_VERSION` (the current
+one) plus `SUPPORTED_SCHEMA_VERSIONS` (the accepted set). Adding a version is a
+minor when older ones keep parsing, **a major once one is dropped**; either way
+the supporting CLI ships before any payload adopts it.
 
 The config file's `version` pin governs the payload package only. `upgrade` moves
 that pin and leaves the CLI version alone. The CLI and the payload release on

src/engine/compose.ts

@@ -41,7 +41,7 @@ function finalizeComposed(
 }
 
 // Fragment sources are read from the payload here so `renderFile` itself stays pure.
-async function renderManagedTemplate(file: ManagedFile, payload: PayloadHandle, config?: StreamctlConfig): Promise<string> {
+async function renderManagedTemplate(file: ManagedFile, payload: PayloadHandle, config: StreamctlConfig | undefined, deps: Record<string, string>): Promise<string> {
   const raw = await payload.read(file.source);
   if (file.renderDef === undefined) {
     return raw;
@@ -52,16 +52,18 @@ async function renderManagedTemplate(file: ManagedFile, payload: PayloadHandle,
       fragmentSources[fragment.source] = await payload.read(fragment.source);
     }
   }
-  return renderFile(raw, file.renderDef, config, fragmentSources, file.path);
+  return renderFile(raw, file.renderDef, config, fragmentSources, file.path, deps);
 }
 
+/** `deps` is the consumer's dependency map (see `dependencyMap`), built once per run by the caller and fed to placeholder `fromDependency` resolution. */
 export async function compose(
   file: ManagedFile,
   payload: PayloadHandle,
   currentContent: string | null = null,
   config?: StreamctlConfig,
+  deps: Record<string, string> = {},
 ): Promise<ComposeResult> {
-  const template = await renderManagedTemplate(file, payload, config);
+  const template = await renderManagedTemplate(file, payload, config, deps);
 
   switch (file.strategy) {
     case "full":

src/engine/drift.ts

@@ -4,6 +4,7 @@ import { join } from "node:path";
 import { compose } from "./compose";
 import { contentEquals } from "./jsonc";
 import { isFileEnabled } from "./render";
+import { dependencyMap, readPackageJson } from "./versions";
 import { readFileOrNull } from "./write";
 
 export type DriftKind = "content" | "missing" | "extra";
@@ -48,6 +49,9 @@ export async function detectDrift(
 ): Promise<DriftReport> {
   const drift: DriftEntry[] = [];
   const structuralFaults: StructuralFault[] = [];
+  // Read once per run: every file composes against the same dependency snapshot, which is
+  // what keeps `check` byte-identical to `sync`.
+  const deps = dependencyMap(await readPackageJson(cwd));
 
   for (const file of managedFiles) {
     // `files: off` opt-out, or a v2 `enabledBy` gate that is false, means not managed.
@@ -56,7 +60,7 @@ export async function detectDrift(
     }
 
     const current = await readFileOrNull(join(cwd, file.path));
-    const result = await compose(file, payload, current, config);
+    const result = await compose(file, payload, current, config, deps);
 
     if (result.status === "structural-error") {
       structuralFaults.push({ path: result.path, reason: result.reason });

src/engine/manifest.ts

@@ -3,7 +3,7 @@ import type { ManagedFile as EngineManagedFile } from "../config/types";
 import type { ConfigKeyType, ManagedFile as ManifestManagedFile, PayloadManifest, PresetManifest, RenderDef } from "../manifest/schema";
 import type { PayloadHandle } from "../payload/resolve";
 import { StreamctlError } from "../errors";
-import { payloadManifestSchema, presetManifestSchema, SUPPORTED_SCHEMA_VERSION, zodToIssues } from "../manifest/schema";
+import { FROM_DEPENDENCY_MIN_SCHEMA_VERSION, payloadManifestSchema, presetManifestSchema, SUPPORTED_SCHEMA_VERSION_LIST, SUPPORTED_SCHEMA_VERSIONS, zodToIssues } from "../manifest/schema";
 
 const PAYLOAD_MANIFEST = "presets/manifest.json";
 
@@ -16,9 +16,9 @@ function summarize(error: ZodError): string {
 }
 
 /**
- * A `schemaVersion` pre-check runs before the full zod parse; an integer other
- * than the supported one throws `SCHEMA_UNSUPPORTED`, so a v3 payload gets the
- * version message instead of a wall of zod issues.
+ * A `schemaVersion` pre-check runs before the full zod parse; an integer outside
+ * the supported set throws `SCHEMA_UNSUPPORTED`, so a payload from the future gets
+ * the version message instead of a wall of zod issues.
  */
 export async function loadPayloadManifest(payload: PayloadHandle): Promise<PayloadManifest> {
   if (!(await payload.list()).includes("manifest.json")) {
@@ -37,11 +37,11 @@ export async function loadPayloadManifest(payload: PayloadHandle): Promise<Paylo
     );
   }
 
-  if (isRecord(parsed) && typeof parsed.schemaVersion === "number" && Number.isInteger(parsed.schemaVersion) && parsed.schemaVersion !== SUPPORTED_SCHEMA_VERSION) {
+  if (isRecord(parsed) && typeof parsed.schemaVersion === "number" && Number.isInteger(parsed.schemaVersion) && !SUPPORTED_SCHEMA_VERSIONS.has(parsed.schemaVersion)) {
     throw new StreamctlError(
       "SCHEMA_UNSUPPORTED",
-      `${PAYLOAD_MANIFEST} declares schemaVersion ${parsed.schemaVersion}, but this streamctl supports schemaVersion ${SUPPORTED_SCHEMA_VERSION}. Upgrade the CLI or the payload.`,
-      { file: PAYLOAD_MANIFEST, found: parsed.schemaVersion, supported: SUPPORTED_SCHEMA_VERSION },
+      `${PAYLOAD_MANIFEST} declares schemaVersion ${parsed.schemaVersion}, but this streamctl supports schemaVersion ${SUPPORTED_SCHEMA_VERSION_LIST}. Upgrade the CLI or the payload.`,
+      { file: PAYLOAD_MANIFEST, found: parsed.schemaVersion, supported: [...SUPPORTED_SCHEMA_VERSIONS] },
     );
   }
 
@@ -101,6 +101,23 @@ export async function loadPresetManifest(payload: PayloadHandle, name: string, m
       throw new StreamctlError("CONFIG_INVALID", `${file} extends "${parent}", which is not listed in ${PAYLOAD_MANIFEST} presets[].`, { file, parent });
     }
   }
+
+  // Feature gate. zod strictness only stops `fromDependency` on an OLD CLI; without this
+  // check a payload author who forgets the schemaVersion bump ships a payload this CLI
+  // accepts and every older one rejects with a raw zod wall.
+  if (manifest.schemaVersion < FROM_DEPENDENCY_MIN_SCHEMA_VERSION) {
+    for (const [render, def] of Object.entries(preset.renders ?? {})) {
+      for (const [key, placeholder] of Object.entries(def.placeholders ?? {})) {
+        if (placeholder.fromDependency !== undefined) {
+          throw new StreamctlError(
+            "CONFIG_INVALID",
+            `${file} renders.${render} placeholder "${key}" uses fromDependency, which requires schemaVersion ${FROM_DEPENDENCY_MIN_SCHEMA_VERSION}, but ${PAYLOAD_MANIFEST} declares schemaVersion ${manifest.schemaVersion}.`,
+            { file, preset: name, render, placeholder: key, found: manifest.schemaVersion, required: FROM_DEPENDENCY_MIN_SCHEMA_VERSION },
+          );
+        }
+      }
+    }
+  }
   return preset;
 }
 

src/engine/render.ts

@@ -2,6 +2,7 @@ import type { ManagedFile, StreamctlConfig } from "../config/types";
 import type { Placeholder, RenderDef } from "../manifest/schema";
 import { StreamctlError } from "../errors";
 import { isIndexable } from "./jsonc";
+import { parseRangeMin } from "./versions";
 
 // Generic renderer (v2): manifest-driven placeholders, fragments, enabledBy gates.
 // Pure and deterministic, so output is byte-stable and a re-sync is idempotent.
@@ -47,8 +48,32 @@ function configStringList(config: StreamctlConfig | undefined, path: string): st
 // A floor, independent of the payload's optional `pattern`.
 const SHELL_META_RE = /[\s;|&$`<>(){}\\'"*?[\]]/;
 
-/** Resolve a placeholder to its substitution string: config value, falling back to `default`; `string[]` gets deduped, sorted, and joined. */
-function resolvePlaceholder(def: Placeholder, config: StreamctlConfig | undefined, key: string, filePath: string): string {
+/** A full `x.y.z` triple with an optional prerelease tail. */
+const FULL_VERSION_RE = /^\d+\.\d+\.\d+(?:-[0-9a-z.-]+)?$/i;
+
+/**
+ * The consumer's pin for `name`, stripped to its range floor (`^6.19.3` → `6.19.3`), or
+ * `null` when there is nothing reproducible to use. `parseRangeMin` also yields partial
+ * cores (`^6` → `"6"`, `~1.2` → `"1.2"`), which would let the rendered version drift
+ * between builds, so only full triples pass — everything else falls back to `default`.
+ */
+function dependencyFloor(deps: Record<string, string>, name: string): string | null {
+  const spec = deps[name];
+  if (spec === undefined) {
+    return null;
+  }
+  const floor = parseRangeMin(spec);
+  return floor !== null && FULL_VERSION_RE.test(floor) ? floor : null;
+}
+
+/**
+ * Resolve a placeholder to its substitution string: config value, then the `fromDependency`
+ * floor, falling back to `default`; `string[]` gets deduped, sorted, and joined.
+ *
+ * `deps` is the consumer's merged dependency map, passed in as data — this stays pure and
+ * never touches the filesystem.
+ */
+function resolvePlaceholder(def: Placeholder, config: StreamctlConfig | undefined, key: string, filePath: string, deps: Record<string, string>): string {
   const raw = readConfigPath(config, def.configPath);
   if (Array.isArray(raw)) {
     const list = [...new Set(raw.filter((item): item is string => typeof item === "string"))].sort();
@@ -73,26 +98,71 @@ function resolvePlaceholder(def: Placeholder, config: StreamctlConfig | undefine
   if (typeof raw === "boolean" || typeof raw === "number") {
     return String(raw);
   }
+  if (def.fromDependency !== undefined) {
+    const floor = dependencyFloor(deps, def.fromDependency);
+    if (floor !== null) {
+      return floor;
+    }
+  }
   return def.default;
 }
 
 /** Extract the inner token name of every streamctl `${TOKEN}`; the `(?!\{)` guard excludes GitHub `${{ ... }}` expressions. */
 const LEFTOVER_TOKEN_RE = /\$\{(?!\{)([^}]*)\}/g;
 
+/**
+ * Substitution repeats until the output stops changing, so a placeholder value that carries
+ * another placeholder's token resolves whatever order the manifest declares them in. The cap
+ * turns a cyclic manifest (A's value names B, B's names A) into a loud error instead of a
+ * hang; ten levels of nesting is far past anything a payload legitimately needs.
+ */
+const MAX_SUBSTITUTION_PASSES = 10;
+
+/**
+ * Every `${TOKEN}` still in `text`, plus the tokens reachable through the values those name.
+ * A cycle leaves only one of its members in the output at any given pass (the others were
+ * just substituted), so following the values is what lets the error name the whole loop.
+ */
+function unresolvedTokens(text: string, values: ReadonlyMap<string, string>, passthrough: ReadonlySet<string>): string[] {
+  const names: string[] = [];
+  const seen = new Set<string>();
+  const collect = (source: string): void => {
+    for (const match of source.matchAll(LEFTOVER_TOKEN_RE)) {
+      const name = match[1] ?? "";
+      if (!passthrough.has(name) && !seen.has(name)) {
+        seen.add(name);
+        names.push(name);
+      }
+    }
+  };
+  collect(text);
+  for (let index = 0; index < names.length; index++) {
+    const value = values.get(names[index] ?? "");
+    if (value !== undefined) {
+      collect(value);
+    }
+  }
+  return names.sort().map(name => `\${${name}}`);
+}
+
 /**
  * Assemble fragments in declared order (`toggle` includes when true, `forEach`
  * repeats per `string[]` item as `${ITEM}`), then substitute `${KEY}` placeholders
  * over the assembled result. GitHub `${{ ... }}` expressions pass through untouched.
  *
  * A `${TOKEN}` left unresolved throws `CONFIG_INVALID`, unless declared in
  * `renderDef.passthrough` (e.g. a Dockerfile build `ARG`).
+ *
+ * `deps` feeds placeholder `fromDependency`; omitting it just means no placeholder resolves
+ * that way.
  */
 export function renderFile(
   sourceContent: string,
   renderDef: RenderDef,
   config: StreamctlConfig | undefined,
   fragmentSources: Record<string, string>,
   filePath = "(template)",
+  deps: Record<string, string> = {},
 ): string {
   const parts = [sourceContent.replace(/\n+$/, "")];
   for (const fragment of renderDef.fragments ?? []) {
@@ -122,19 +192,49 @@ export function renderFile(
 
   let output = `${parts.join("\n")}\n`;
 
+  // Values are resolved (and pattern-checked) once, ahead of substitution: they come from
+  // the config and the dependency map, never from the output being assembled.
+  const values = new Map<string, string>();
   for (const [key, def] of Object.entries(renderDef.placeholders ?? {})) {
-    const value = resolvePlaceholder(def, config, key, filePath);
+    const value = resolvePlaceholder(def, config, key, filePath, deps);
     if (def.pattern !== undefined && !new RegExp(def.pattern).test(value)) {
       throw new StreamctlError(
         "CONFIG_INVALID",
         `render placeholder "${key}" for "${filePath}" has value ${JSON.stringify(value)}, which violates its pattern /${def.pattern}/.`,
         { file: filePath, placeholder: key, value },
       );
     }
-    output = output.replaceAll(`\${${key}}`, () => value);
+    values.set(key, value);
   }
 
   const passthrough = new Set(renderDef.passthrough ?? []);
+
+  // Callback form: a plain replacement string would honor `$&`/`$1` patterns in the value.
+  // Config values are data and must land verbatim.
+  const substitutePass = (text: string): string => {
+    let next = text;
+    for (const [key, value] of values) {
+      next = next.replaceAll(`\${${key}}`, () => value);
+    }
+    return next;
+  };
+
+  for (let pass = 1; ; pass++) {
+    const next = substitutePass(output);
+    if (next === output) {
+      break; // fixpoint
+    }
+    output = next;
+    if (pass >= MAX_SUBSTITUTION_PASSES) {
+      const tokens = unresolvedTokens(output, values, passthrough);
+      throw new StreamctlError(
+        "CONFIG_INVALID",
+        `render for "${filePath}" did not stabilize after ${MAX_SUBSTITUTION_PASSES} substitution passes; still unresolved: ${tokens.join(", ")}. A placeholder value references itself, directly or through another placeholder.`,
+        { file: filePath, tokens, passes: MAX_SUBSTITUTION_PASSES },
+      );
+    }
+  }
+
   for (const match of output.matchAll(LEFTOVER_TOKEN_RE)) {
     const token = match[0];
     const name = match[1] ?? "";

src/engine/status.ts

@@ -10,7 +10,7 @@ import { contentEquals } from "./jsonc";
 import { resolvePresetChain } from "./manifest";
 import { lockfileExists } from "./pm";
 import { isFileActive, isFileEnabled } from "./render";
-import { checkUpdateAvailable, detectVersionSkew } from "./versions";
+import { checkUpdateAvailable, dependencyMap, detectVersionSkew, readPackageJson } from "./versions";
 import { readFileOrNull } from "./write";
 
 /**
@@ -56,7 +56,7 @@ export interface RunStatusOptions {
 }
 
 /** Classify one managed file's state from a read-only compose (mirrors the sync plan-pass, but writes nothing). */
-async function fileState(file: ManagedFile, payload: PayloadHandle, config: StreamctlConfig, cwd: string): Promise<FileState> {
+async function fileState(file: ManagedFile, payload: PayloadHandle, config: StreamctlConfig, cwd: string, deps: Record<string, string>): Promise<FileState> {
   if (config.files?.[file.path] === "off") {
     return "off";
   }
@@ -65,7 +65,7 @@ async function fileState(file: ManagedFile, payload: PayloadHandle, config: Stre
   }
 
   const current = await readFileOrNull(join(cwd, file.path));
-  const result = await compose(file, payload, current, config);
+  const result = await compose(file, payload, current, config, deps);
 
   if (result.status === "structural-error" || result.status === "marker-error" || result.status === "merge-error") {
     return "fault";
@@ -103,9 +103,12 @@ export async function runStatus(
   // Stage 2: a malformed config here is a hard exit-1.
   validateStreamctlConfigWithKeys(config, configKeys);
 
+  // Read once per run, so `status` composes the same bytes `sync`/`check` do.
+  const deps = dependencyMap(await readPackageJson(cwd));
+
   const files: StatusEntry[] = [];
   for (const file of managedFiles) {
-    files.push({ path: file.path, strategy: file.strategy, state: await fileState(file, payload, config, cwd) });
+    files.push({ path: file.path, strategy: file.strategy, state: await fileState(file, payload, config, cwd, deps) });
   }
 
   const hasEslintConfig = managedFiles.some(file => file.path.endsWith("eslint.config.ts") && isFileActive(file, config));

src/engine/sync.ts

@@ -10,7 +10,7 @@ import { dirtyTrackedPaths } from "./git";
 import { contentEquals } from "./jsonc";
 import { lockfileExists } from "./pm";
 import { isFileActive, isFileEnabled } from "./render";
-import { applyReconcile, planReconcile, readPackageJson } from "./versions";
+import { applyReconcile, dependencyMap, planReconcile, readPackageJson } from "./versions";
 import { atomicWrite, readFileOrNull } from "./write";
 
 /**
@@ -202,11 +202,18 @@ export async function runSync(opts: RunSyncOptions): Promise<SyncResult> {
     activeFiles.push(file);
   }
 
+  // Read once per run, ahead of the plan pass: renders resolve placeholder
+  // `fromDependency` against it, and the reconcile below reuses the same parse. A
+  // malformed package.json therefore throws CONFIG_INVALID before any managed write —
+  // now also when `versionSync` is off, since rendering needs it either way.
+  const parsedPkg = await readPackageJson(cwd);
+  const deps = dependencyMap(parsedPkg);
+
   // Plan pass: compose and validate everything, write nothing.
   for (const file of activeFiles) {
     const abs = join(cwd, file.path);
     const current = await readFileOrNull(abs);
-    const result = await compose(file, payload, current, config);
+    const result = await compose(file, payload, current, config, deps);
 
     // Never write output that does not parse. `--force` cannot fix corrupt output.
     if (result.status === "structural-error") {
@@ -248,12 +255,6 @@ export async function runSync(opts: RunSyncOptions): Promise<SyncResult> {
     pending.push({ change, abs, content: result.targetContent, file });
   }
 
-  // Still the plan pass. A malformed package.json throws CONFIG_INVALID here, before
-  // any managed write, rather than inside the post-write reconcile, which would leave
-  // managed files already on disk. Threaded into `planReconcile` so the file is
-  // read once per run.
-  const parsedPkg = config.versionSync === false ? null : await readPackageJson(cwd);
-
   // jiti is reconciled only for an ACTIVE eslint.config.ts: a file turned off or gated
   // out by `enabledBy` is not managed here, so its version key must not move either.
   const hasEslintConfig = opts.managedFiles.some(file => file.path.endsWith("eslint.config.ts") && isFileActive(file, config));