Support custom providers across all harnesses

.github/workflows/release-package.yml

@@ -19,6 +19,7 @@ jobs:
         include:
           - name: core
             dockerfile: deploy/core/Dockerfile
+            build-args: GIT_SHA=${{ github.sha }}
           - name: web-ui
             dockerfile: deploy/web-ui/Dockerfile
           - name: admin

cli/src/backends/aws.ts

@@ -47,6 +47,8 @@ import {
   resolveBuildRepoRoot,
   runInherit,
   sleep,
+  sourceBuildArgs,
+  sourceBuildInfo,
   streamLabeled,
 } from "../util.ts";
 import { doctorCommon } from "./doctor.ts";
@@ -471,22 +473,6 @@ function workloadImageProvenance(
   return { kind: "configured", source };
 }
 
-const sourceBuildInfoByRoot = new Map<string, { gitCommit?: string; dirty?: boolean }>();
-
-function sourceBuildInfo(root: string): { gitCommit?: string; dirty?: boolean } {
-  const cached = sourceBuildInfoByRoot.get(root);
-  if (cached) return cached;
-  const info: { gitCommit?: string; dirty?: boolean } = {};
-  try {
-    info.gitCommit = capture("git", ["-C", root, "rev-parse", "HEAD"]).trim();
-    info.dirty = capture("git", ["-C", root, "status", "--porcelain"]).trim().length > 0;
-  } catch {
-    void 0;
-  }
-  sourceBuildInfoByRoot.set(root, info);
-  return info;
-}
-
 function sourceImageDigest(source: string): string {
   const pinned = source.match(/@(?<digest>sha256:[0-9a-f]{64})$/)?.groups?.digest;
   if (pinned) return pinned;
@@ -556,10 +542,12 @@ function publishWorkloadImage(
       "-t",
       tagged,
     ];
-    const info = sourceBuildInfo(root);
-    if (info.gitCommit) args.push("--build-arg", `GIT_SHA=${info.gitCommit}${info.dirty ? "-dirty" : ""}`);
-    for (const [name, value] of Object.entries(workloadBuildArgs(config, workload)))
-      args.push("--build-arg", `${name}=${value}`);
+    const configuredBuildArgs = workloadBuildArgs(config, workload);
+    if (workload === "core" && Object.hasOwn(configuredBuildArgs, "GIT_SHA")) {
+      throw new CliError("aws.services.core.buildArgs.GIT_SHA is reserved for source-build provenance");
+    }
+    for (const [name, value] of Object.entries(configuredBuildArgs)) args.push("--build-arg", `${name}=${value}`);
+    args.push(...sourceBuildArgs(root, workload));
     args.push(root);
     runInherit("docker", args);
   } else {

cli/src/backends/docker.ts

@@ -14,6 +14,7 @@ import {
   resolveBuildRepoRoot,
   runInherit,
   sleep,
+  sourceBuildArgs,
   streamLabeled,
   tailString,
   which,
@@ -144,7 +145,7 @@ function resolveImage(ctx: DockerCtx, service: ServiceName): string {
     const dockerfile = join(root, "deploy", service, "Dockerfile");
     if (!existsSync(dockerfile)) throw new CliError(`no Dockerfile at ${dockerfile}`);
     const tag = `qm-${service}:local`;
-    const buildArgs: string[] = [];
+    const buildArgs = sourceBuildArgs(root, service);
     step(`building ${service} from ${dockerfile}`);
     dockerInherit(["build", "-f", dockerfile, "-t", tag, ...buildArgs, root]);
     return tag;

cli/src/backends/fly.ts

@@ -12,6 +12,7 @@ import {
   promptHidden,
   readEnvFile,
   settleAll,
+  sourceBuildArgs,
   streamLabeled,
   which,
 } from "../util.ts";
@@ -791,7 +792,13 @@ async function deployService(
   } else if (imageSource?.kind === "manifest") {
     args.push("--image", ctx.config.imageOverrides[service] ?? manifestRef(service));
   } else {
-    args.push("--remote-only", "--dockerfile", join(ctx.sourceRoot!, "deploy", service, "Dockerfile"), ctx.sourceRoot!);
+    args.push(
+      "--remote-only",
+      "--dockerfile",
+      join(ctx.sourceRoot!, "deploy", service, "Dockerfile"),
+      ...sourceBuildArgs(ctx.sourceRoot!, service),
+      ctx.sourceRoot!,
+    );
   }
   step(`fly ${args.join(" ")}`);
 
@@ -832,6 +839,7 @@ async function buildServiceImage(
     "--remote-only",
     "--dockerfile",
     join(ctx.sourceRoot!, "deploy", service, "Dockerfile"),
+    ...sourceBuildArgs(ctx.sourceRoot!, service),
     ctx.sourceRoot!,
   ];
   step(`fly ${args.join(" ")}`);

cli/src/util.ts

@@ -49,6 +49,29 @@ export function capture(
   }
 }
 
+const sourceBuildInfoByRoot = new Map<string, { gitCommit?: string; dirty?: boolean }>();
+
+export function sourceBuildInfo(root: string): { gitCommit?: string; dirty?: boolean } {
+  const cached = sourceBuildInfoByRoot.get(root);
+  if (cached) return cached;
+  const info: { gitCommit?: string; dirty?: boolean } = {};
+  try {
+    info.gitCommit = capture("git", ["-C", root, "rev-parse", "HEAD"]).trim();
+    info.dirty = capture("git", ["-C", root, "status", "--porcelain"]).trim().length > 0;
+  } catch {
+    void 0;
+  }
+  sourceBuildInfoByRoot.set(root, info);
+  return info;
+}
+
+export function sourceBuildArgs(root: string, service: string): string[] {
+  if (service !== "core") return [];
+  const info = sourceBuildInfo(root);
+  if (!info.gitCommit) throw new CliError("building core from source requires a Git checkout with a commit");
+  return ["--build-arg", `GIT_SHA=${info.gitCommit}${info.dirty ? "-dirty" : ""}`];
+}
+
 export function captureBoth(cmd: string, args: string[], opts: { cwd?: string; env?: NodeJS.ProcessEnv } = {}): string {
   const r = spawnSync(cmd, args, {
     encoding: "utf8",

cli/test/aws.test.ts

@@ -1830,11 +1830,27 @@ test("AWS source builds honor a per-service dockerfile override and stamp GIT_SH
       `core build uses the override: ${coreBuild}`,
     );
     assert.ok(coreBuild?.includes(`--build-arg GIT_SHA=${head}`), `core build stamps GIT_SHA: ${coreBuild}`);
+    assert.equal(coreBuild?.match(/--build-arg GIT_SHA=/g)?.length, 1);
     const webUiBuild = builds.find((line) => line.includes("qm-web-ui"));
     assert.ok(
       webUiBuild?.includes(`-f ${join(sourceDir, "deploy", "web-ui", "Dockerfile")}`),
       `web-ui build keeps the default: ${webUiBuild}`,
     );
+    assert.doesNotMatch(webUiBuild ?? "", /--build-arg GIT_SHA=/);
+    const reservedConfig: QmConfig = {
+      ...layeredConfig,
+      aws: {
+        ...layeredConfig.aws!,
+        services: {
+          ...layeredConfig.aws!.services,
+          core: { ...layeredConfig.aws!.services.core!, buildArgs: { GIT_SHA: "spoofed" } },
+        },
+      },
+    };
+    await assert.rejects(
+      () => awsUp(reservedConfig, dir, { yes: true, buildFrom: true, buildFromPath: sourceDir }),
+      /aws\.services\.core\.buildArgs\.GIT_SHA is reserved/,
+    );
     const dirtySource = join(dir, "dirty-source");
     for (const service of ["core", "web-ui", "portal"]) {
       mkdirSync(join(dirtySource, "deploy", service), { recursive: true });

cli/test/docker-secrets.test.ts

@@ -2,10 +2,13 @@ import { test } from "node:test";
 import assert from "node:assert/strict";
 import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
-import { join } from "node:path";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
 import { CONFIG_FILENAME, loadConfigAt } from "../src/config.ts";
 import { dockerUp } from "../src/backends/docker.ts";
 
+const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
+
 const SECRETS = {
   ANTHROPIC_API_KEY: "anthropic-supersecret",
   CAPABILITY_SECRET: "capability-supersecret",
@@ -137,9 +140,19 @@ test("docker up delivers secrets via a 0600 env-file, never on the docker argv",
     console.log = (...parts: unknown[]): void => void lines.push(parts.join(" "));
     console.warn = console.log;
     const { config } = loadConfigAt(join(dir, CONFIG_FILENAME));
-    await dockerUp(config, dir, {});
+    await dockerUp(config, dir, { buildFrom: true, buildFromPath: repoRoot });
 
     const argv = readFileSync(fake.argvLog, "utf8");
+    const build = argv
+      .split("\n")
+      .filter(Boolean)
+      .map((line) => JSON.parse(line) as string[])
+      .find((args) => args[0] === "build" && args.includes("qm-core:local"));
+    assert.ok(build, "the core image is built from source");
+    const stampIndexes = build.flatMap((value, index) => (value === "--build-arg" ? [index] : []));
+    const stamps = stampIndexes.map((index) => build[index + 1]!).filter((value) => value.startsWith("GIT_SHA="));
+    assert.equal(stamps.length, 1);
+    assert.match(stamps[0]!, /^GIT_SHA=[0-9a-f]{40}(?:-dirty)?$/);
     for (const value of Object.values(SECRETS)) {
       assert.ok(!argv.includes(value), `secret value must not reach the docker argv: ${value}`);
     }

cli/test/e2e/harness.ts

@@ -103,6 +103,13 @@ export function standInCheckout(services: readonly string[]): string {
     mkdirSync(dir, { recursive: true });
     writeFileSync(join(dir, "Dockerfile"), STANDIN_DOCKERFILE);
   }
+  execFileSync("git", ["init", "-q"], { cwd: root });
+  execFileSync("git", ["add", "-A"], { cwd: root });
+  execFileSync(
+    "git",
+    ["-c", "user.email=test@qm.invalid", "-c", "user.name=QM Test", "commit", "-q", "-m", "stand-in"],
+    { cwd: root },
+  );
   return root;
 }
 

cli/test/fly-timing.test.ts

@@ -375,16 +375,17 @@ test("fly up build-only pushes a tagged image without checking runtime deploy se
     true,
     "new apps receive only the ownership marker",
   );
-  assert.equal(
-    commands.some(
-      (args) =>
-        args[0] === "deploy" &&
-        args.includes("--build-only") &&
-        args.includes("--push") &&
-        args.includes("--image-label"),
-    ),
-    true,
+  const build = commands.find(
+    (args) =>
+      args[0] === "deploy" &&
+      args.includes("--build-only") &&
+      args.includes("--push") &&
+      args.includes("--image-label"),
   );
+  assert.ok(build);
+  const stamps = build.filter((value) => value.startsWith("GIT_SHA="));
+  assert.equal(stamps.length, 1);
+  assert.match(stamps[0]!, /^GIT_SHA=[0-9a-f]{40}(?:-dirty)?$/);
 });
 
 test("fly up build-only dry-run plans without pushing an image", () => {

cli/test/fly-up.test.ts

@@ -130,6 +130,12 @@ else console.log("ok");
     assert.match(calls, /secrets unset --stage -a acme-core FLY_DEPLOY_API_TOKEN/);
     assert.ok(calls.indexOf("secrets unset") < calls.indexOf("deploy"));
     assert.ok(calls.indexOf("storage create") < calls.indexOf("deploy"));
+    const deploy = calls
+      .split("\n")
+      .find((line) => line.startsWith("deploy ") && line.includes("deploy/core/Dockerfile"));
+    assert.ok(deploy);
+    assert.equal(deploy.match(/--build-arg GIT_SHA=/g)?.length, 1);
+    assert.match(deploy, /--build-arg GIT_SHA=[0-9a-f]{40}(?:-dirty)?(?:\s|$)/);
     assert.equal(
       JSON.parse(readFileSync(configPath, "utf8")).imageOverrides.core,
       `registry.fly.io/acme-core@sha256:${"b".repeat(64)}`,

cli/test/util.test.ts

@@ -1,9 +1,40 @@
 import { test } from "node:test";
 import assert from "node:assert/strict";
-import { chmodSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { canonicalJson, flyBin, isInvalidSecret, readEnvFile, writeEnvValue } from "../src/util.ts";
+import { canonicalJson, flyBin, isInvalidSecret, readEnvFile, sourceBuildArgs, writeEnvValue } from "../src/util.ts";
+
+test("source builds stamp only core from Git provenance and fail closed without a commit", (t) => {
+  const root = mkdtempSync(join(tmpdir(), "qm-source-build-"));
+  const dirtyRoot = mkdtempSync(join(tmpdir(), "qm-source-build-dirty-"));
+  const missing = mkdtempSync(join(tmpdir(), "qm-source-build-missing-"));
+  t.after(() => {
+    rmSync(root, { recursive: true, force: true });
+    rmSync(dirtyRoot, { recursive: true, force: true });
+    rmSync(missing, { recursive: true, force: true });
+  });
+  const initialize = (directory: string): string => {
+    mkdirSync(join(directory, "deploy", "core"), { recursive: true });
+    writeFileSync(join(directory, "deploy", "core", "Dockerfile"), "FROM scratch\n");
+    execFileSync("git", ["init", "-q"], { cwd: directory });
+    execFileSync("git", ["add", "-A"], { cwd: directory });
+    execFileSync(
+      "git",
+      ["-c", "user.email=test@qm.invalid", "-c", "user.name=QM Test", "commit", "-q", "-m", "initial"],
+      { cwd: directory },
+    );
+    return execFileSync("git", ["rev-parse", "HEAD"], { cwd: directory, encoding: "utf8" }).trim();
+  };
+  const head = initialize(root);
+  assert.deepEqual(sourceBuildArgs(root, "core"), ["--build-arg", `GIT_SHA=${head}`]);
+  assert.deepEqual(sourceBuildArgs(root, "web-ui"), []);
+  const dirtyHead = initialize(dirtyRoot);
+  writeFileSync(join(dirtyRoot, "dirty.txt"), "dirty\n");
+  assert.deepEqual(sourceBuildArgs(dirtyRoot, "core"), ["--build-arg", `GIT_SHA=${dirtyHead}-dirty`]);
+  assert.throws(() => sourceBuildArgs(missing, "core"), /requires a Git checkout with a commit/);
+});
 
 test("managed credential encryption keys require strong material", () => {
   assert.equal(isInvalidSecret("CONNECTOR_SECRET_KEY", "short"), true);