Test every preset against ComfyUI 0.44, and stop Health hiding working presets

CLAUDE.md

@@ -37,6 +37,8 @@ npm run build          tsc + vite build + scripts/copy-uxp-assets.mjs -> dist/
 npm run package        build + zip + .ccx into packages/ (incl. openlayer-latest.*)
 npm run setup-pack     GENERATED from presetRegistry; never hand-edit its output
 npm run audit-css      measures CSS redeclarations
+npm run object-info    record tests/fixtures/comfyui-object-info.json from a live ComfyUI
+                       (`-- --check` diffs a live ComfyUI against it; read-only)
 cd bridge && npm ci    the MCP bridge is a separate package with its own lockfile
 ```
 

docs/beta-gate.md

@@ -15,7 +15,7 @@ the same change.
 
 | # | Gate | How it is checked | Status |
 |---|---|---|---|
-| G1 | **Runs on a current ComfyUI.** Every runnable preset's nodes and inputs match a recorded `/object_info` from the newest ComfyUI release, and a test fails when they drift. | Roadmap U2: a snapshot in the repo, a test of every preset's `requiredNodes` against it, and a script that diffs a live ComfyUI against the snapshot. Its output against a current ComfyUI is pasted in the PR. | **Open.** Every preset has been checked only against ComfyUI 0.37.0, the version on the test machine. ComfyUI 0.40 was out by 2026-10-13. |
+| G1 | **Runs on a current ComfyUI.** Every runnable preset's nodes and inputs match a recorded `/object_info` from the newest ComfyUI release, and a test fails when they drift. | Roadmap U2: a snapshot in the repo, a test of every preset's `requiredNodes` against it, and a script that diffs a live ComfyUI against the snapshot. Its output against a current ComfyUI is pasted in the PR. | **Done in v0.41.** `tests/fixtures/comfyui-object-info.json` is recorded from ComfyUI 0.44.0 (80 node classes) by `npm run object-info`, and `tests/comfy/objectInfoSnapshot.test.ts` checks every node, input, nested input and fixed choice of all 39 presets against it. All match. Running Workflow Health against 0.44 found a real fault: it read only required inputs and marked 13 working presets "Missing ComfyUI node"; fixed, it now reports 28 ready, 10 experimental and 1 model in the wrong folder on the test machine. `npm run object-info -- --check` against a fresh Comfy Desktop 0.38.1 lists only the custom node packs it has not installed. |
 | G2 | **Finds Comfy Desktop.** The port scan includes Comfy Desktop's default port. | A test on `COMFY_PORT_CANDIDATES`, plus Settings › Find ComfyUI Active Port against a ComfyUI started on that port. | **Done in v0.41.** 8000 is scanned second, right after 8188, with a test. Not yet run against a real Comfy Desktop: on the test machine a ComfyUI on 8188 always answers first. A clean-install tester (G6) using Comfy Desktop checks it for real. |
 | G3 | **Says so when there is no GPU.** When ComfyUI reports only a CPU device, the panel says which tools will be too slow to use, rather than failing or hanging. | A unit test with a CPU-only `/system_stats` fixture, plus one run against ComfyUI started with `--cpu`. | **Done in v0.41.** Check ComfyUI says "ComfyUI is online, but on CPU only." with the reason underneath, and Detect Hardware reports "Processor only, no graphics card", VRAM none, tier No GPU. A CPU device no longer counts its system RAM as VRAM. Checked in Photoshop 2025 on 2026-11-04 against a real Comfy Desktop instance started with `--cpu` (ComfyUI 0.38.1, port 8189), and against a GPU ComfyUI 0.44.0 on 8188, which still reports normally. |
 | G4 | **Passes the full smoke checklist on the release build.** All blocks of [the UXP smoke checklist](uxp-smoke-checklist.md) on the build that will be tagged, with a row in its Results table. | The Results table. | **Open.** The only recorded full run is from v0.37.1. v0.38 to v0.40 were checked per feature, in their own PRs. |

package.json

@@ -13,6 +13,7 @@
     "package": "npm run build && node scripts/package.mjs",
     "setup-pack": "node scripts/build-setup-pack.mjs",
     "audit-css": "node scripts/audit-css.mjs",
+    "object-info": "node scripts/object-info-snapshot.mjs",
     "test": "vitest run --config vitest.config.ts",
     "test:e2e": "vitest run --config vitest.e2e.config.ts"
   },

scripts/object-info-snapshot.mjs

@@ -0,0 +1,171 @@
+import { readFile, readdir, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+/**
+ * Records, or checks against, the parts of ComfyUI's /object_info that OpenLayer
+ * depends on. Beta gate G1 (roadmap U2).
+ *
+ *   node scripts/object-info-snapshot.mjs [serverUrl]           record tests/fixtures/comfyui-object-info.json
+ *   node scripts/object-info-snapshot.mjs --check [serverUrl]   diff a live ComfyUI against the recording
+ *
+ * Only the node classes OpenLayer uses are kept: every class_type in the bundled
+ * API workflows, every classType the preset registry requires, and every
+ * class_type the TypeScript builds itself (Live Painting, the LoRA splice). For
+ * each input the snapshot keeps its type and, for a fixed list of choices such
+ * as a sampler or scheduler, the choices -- but never a list of model files,
+ * which differs on every machine. Nested inputs (ComfyUI's dynamic combo and
+ * autogrow inputs, written "parent.child" in a workflow) keep the names they
+ * allow. tests/comfy/objectInfoSnapshot.test.ts checks the presets against it.
+ *
+ * Read-only against ComfyUI: one GET of /object_info and one of /system_stats.
+ */
+
+const root = fileURLToPath(new URL("..", import.meta.url));
+const snapshotPath = resolve(root, "tests/fixtures/comfyui-object-info.json");
+const args = process.argv.slice(2);
+const checkMode = args.includes("--check");
+const serverUrl = (args.find((arg) => !arg.startsWith("--")) ?? "http://127.0.0.1:8188").replace(/\/+$/, "");
+
+const FILE_LIKE = /\.(safetensors|pt|pth|gguf|ckpt|bin|onnx|png|jpe?g|webp|json|ya?ml)$/i;
+
+// "Class.input" -> the literal values the bundled workflows set on it, so a
+// choice removed upstream only fails the check when a preset uses it.
+const usedChoices = new Map();
+
+async function usedClasses() {
+  const classes = new Set();
+  const apiDir = resolve(root, "src/workflows/api");
+
+  for (const name of await readdir(apiDir)) {
+    if (!name.endsWith(".json")) continue;
+    const workflow = JSON.parse(await readFile(join(apiDir, name), "utf8"));
+    for (const node of Object.values(workflow)) {
+      if (!node || typeof node !== "object" || typeof node.class_type !== "string") continue;
+      classes.add(node.class_type);
+      for (const [input, value] of Object.entries(node.inputs ?? {})) {
+        if (typeof value !== "string" || value === "") continue;
+        const key = `${node.class_type}.${input}`;
+        if (!usedChoices.has(key)) usedChoices.set(key, new Set());
+        usedChoices.get(key).add(value);
+      }
+    }
+  }
+
+  const comfyDir = resolve(root, "src/comfy");
+  for (const name of await readdir(comfyDir)) {
+    if (!name.endsWith(".ts")) continue;
+    const source = await readFile(join(comfyDir, name), "utf8");
+    for (const match of source.matchAll(/class(?:_t|T)ype: "([^"]+)"/g)) classes.add(match[1]);
+  }
+
+  return [...classes].sort();
+}
+
+function choicesOf(spec) {
+  if (Array.isArray(spec) && Array.isArray(spec[0])) return spec[0];
+  if (Array.isArray(spec) && spec[0] === "COMBO" && Array.isArray(spec[1]?.options)) return spec[1].options;
+  return null;
+}
+
+function summarizeInput(spec) {
+  const type = Array.isArray(spec) ? spec[0] : spec;
+
+  if (type === "COMFY_DYNAMICCOMBO_V3") {
+    const nested = {};
+    for (const option of spec[1]?.options ?? []) {
+      nested[option.key] = Object.keys({ ...(option.inputs?.required ?? {}), ...(option.inputs?.optional ?? {}) }).sort();
+    }
+    return { type, nested };
+  }
+
+  if (type === "COMFY_AUTOGROW_V3") {
+    // ComfyUI 0.44 keeps the slot names ("image_1", ...) under "template".
+    return { type, names: spec[1]?.template?.names ?? spec[1]?.names ?? [] };
+  }
+
+  const choices = choicesOf(spec);
+  if (choices) {
+    const fixed = choices.length > 0 && choices.every((choice) => typeof choice === "string" && !FILE_LIKE.test(choice) && !choice.includes("/") && !choice.includes("\\"));
+    return fixed ? { type: "COMBO", choices } : { type: "COMBO" };
+  }
+
+  return { type: typeof type === "string" ? type : "UNKNOWN" };
+}
+
+function summarizeNode(info) {
+  const inputs = {};
+  for (const [name, spec] of Object.entries({ ...(info.input?.required ?? {}), ...(info.input?.optional ?? {}) })) {
+    inputs[name] = summarizeInput(spec);
+  }
+  return { inputs };
+}
+
+async function fetchJson(path) {
+  const response = await fetch(`${serverUrl}${path}`);
+  if (!response.ok) throw new Error(`${serverUrl}${path} answered HTTP ${response.status}`);
+  return response.json();
+}
+
+const classes = await usedClasses();
+const [objectInfo, systemStats] = await Promise.all([fetchJson("/object_info"), fetchJson("/system_stats")]);
+const live = {
+  comfyuiVersion: systemStats.system?.comfyui_version ?? "unknown",
+  recordedAt: new Date().toISOString().slice(0, 10),
+  missingClasses: classes.filter((name) => !objectInfo[name]),
+  nodes: Object.fromEntries(classes.filter((name) => objectInfo[name]).map((name) => [name, summarizeNode(objectInfo[name])]))
+};
+
+// process.exitCode rather than process.exit(): exiting while fetch's sockets
+// are still closing trips a libuv assertion on Windows.
+if (!checkMode) {
+  await writeFile(snapshotPath, `${JSON.stringify(live, null, 2)}\n`);
+  console.log(`Recorded ${Object.keys(live.nodes).length} node classes from ComfyUI ${live.comfyuiVersion} at ${serverUrl}.`);
+  if (live.missingClasses.length > 0) console.log(`Not installed there: ${live.missingClasses.join(", ")}.`);
+} else {
+  process.exitCode = checkAgainstSnapshot(JSON.parse(await readFile(snapshotPath, "utf8"))) ? 0 : 1;
+}
+
+function checkAgainstSnapshot(recorded) {
+  const problems = [];
+  const notes = [];
+  let unusedChoicesGone = 0;
+
+  for (const missing of recorded.missingClasses ?? []) {
+    notes.push(`${missing}: was not installed when the snapshot was recorded`);
+  }
+
+  for (const [name, node] of Object.entries(recorded.nodes)) {
+    const now = live.nodes[name];
+    if (!now) {
+      problems.push(`${name}: class is not there (a core node renamed, or its node pack is not installed)`);
+      continue;
+    }
+    for (const [input, spec] of Object.entries(node.inputs)) {
+      const current = now.inputs[input];
+      if (!current) {
+        problems.push(`${name}.${input}: input is gone`);
+        continue;
+      }
+      for (const choice of spec.choices ?? []) {
+        if (!current.choices || current.choices.includes(choice)) continue;
+        if (usedChoices.get(`${name}.${input}`)?.has(choice)) problems.push(`${name}.${input}: choice "${choice}" is gone, and a preset uses it`);
+        else unusedChoicesGone += 1;
+      }
+      for (const [key, names] of Object.entries(spec.nested ?? {})) {
+        const nowNames = current.nested?.[key];
+        if (!nowNames) problems.push(`${name}.${input}: option "${key}" is gone`);
+        else for (const child of names) if (!nowNames.includes(child)) problems.push(`${name}.${input}.${child}: nested input is gone`);
+      }
+    }
+    for (const input of Object.keys(now.inputs)) {
+      if (!node.inputs[input]) notes.push(`${name}.${input}: new input`);
+    }
+  }
+
+  console.log(`Snapshot: ComfyUI ${recorded.comfyuiVersion} (${recorded.recordedAt}). Live: ComfyUI ${live.comfyuiVersion} at ${serverUrl}.`);
+  console.log(problems.length === 0 ? "Nothing OpenLayer uses is missing." : `Missing, and OpenLayer uses it:\n  ${problems.join("\n  ")}`);
+  if (unusedChoicesGone > 0) console.log(`${unusedChoicesGone} choice(s) no preset uses are also missing there (harmless).`);
+  if (notes.length > 0) console.log(`Notes:\n  ${notes.join("\n  ")}`);
+  return problems.length === 0;
+}

src/comfy/comfyClient.ts

@@ -256,6 +256,15 @@ export class ComfyClient {
     return inventory;
   }
 
+  /**
+   * For Workflow Health and Setup: every input each preset node *declares*,
+   * required and optional, because the question is whether the input OpenLayer
+   * wires exists -- the same reading validatePresetSetup uses before Generate.
+   * Reading required inputs only marked 13 working presets "Missing ComfyUI
+   * node" on ComfyUI 0.44, among them every crop-and-stitch Inpaint preset
+   * (InpaintCropImproved's mask), both Remove object presets (LaMaInpaint's
+   * lama_model) and Select by Text (SAM3_Detect's conditioning). Beta gate G1.
+   */
   async getWorkflowNodeAvailability(
     presets: readonly WorkflowPresetDefinition[]
   ): Promise<WorkflowNodeAvailability> {
@@ -267,23 +276,29 @@ export class ComfyClient {
       }
     }
 
-    return this.getNodeAvailability([...nodeClasses]);
+    return this.getNodeAvailability([...nodeClasses], "declared");
   }
 
   /**
    * The same question for an arbitrary list of class types, so a workflow
    * OpenLayer did not author can be checked against this server.
    *
-   * Records only the inputs ComfyUI marks *required*. A caller that treats the
-   * absence of an optional input as a fault repeats the mistake documented on
-   * the Klein edit preset, where it reported a working graph as broken.
+   * By default records only the inputs ComfyUI marks *required*: the custom
+   * workflow check asks whether a graph supplies every one of those, and an
+   * optional input left out is not a fault there. "declared" records required
+   * and optional together, for a caller asking whether an input exists at all
+   * (see getWorkflowNodeAvailability). Mixing the two up is how Workflow
+   * Health came to report working presets as broken.
    *
    * A class that does not exist is left out rather than recorded as empty.
    * That matters more than it looks: `GET /object_info/<ClassName>` answers 200
    * with an empty body for an unknown class, so presence of a *response* proves
    * nothing and only the presence of the schema does.
    */
-  async getNodeAvailability(classTypes: readonly string[]): Promise<WorkflowNodeAvailability> {
+  async getNodeAvailability(
+    classTypes: readonly string[],
+    inputs: "required" | "declared" = "required"
+  ): Promise<WorkflowNodeAvailability> {
     const availability: WorkflowNodeAvailability = {};
 
     for (const classType of classTypes) {
@@ -292,7 +307,11 @@ export class ComfyClient {
         const schema = objectInfo[classType];
 
         if (schema) {
-          availability[classType] = Object.keys(schema.input?.required ?? {});
+          availability[classType] = Object.keys(
+            inputs === "declared"
+              ? { ...(schema.input?.required ?? {}), ...(schema.input?.optional ?? {}) }
+              : schema.input?.required ?? {}
+          );
         }
       } catch {
         // Missing node classes are reported by the caller, not here.

src/comfy/workflowCompatibility.ts

@@ -47,6 +47,11 @@ export type WorkflowCompatibilityIssue = {
   link?: WorkflowCompatibilityLink;
 };
 
+/**
+ * Node class -> input names. Which inputs depends on who asked: the preset
+ * checks (Workflow Health, Setup) get every declared input, the custom
+ * workflow check gets the required ones. See ComfyClient.getNodeAvailability.
+ */
 export type WorkflowNodeAvailability = Record<string, readonly string[]>;
 
 export type WorkflowPhotoshopInputAvailability = Partial<Record<WorkflowPhotoshopInputKind, boolean>>;

tests/comfy/nodeAvailability.test.ts

@@ -0,0 +1,55 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { ComfyClient } from "../../src/comfy/comfyClient";
+import { getWorkflowPreset } from "../../src/comfy/presetRegistry";
+import { createWorkflowHealthItem } from "../../src/comfy/workflowHealth";
+
+/**
+ * Workflow Health asked ComfyUI which inputs each preset node has, but kept
+ * only the required ones. On ComfyUI 0.44, 13 working presets were marked
+ * "Missing ComfyUI node" because the input they wire is optional there:
+ * InpaintCropImproved's mask, LaMaInpaint's lama_model, SAM3_Detect's
+ * conditioning, ReferenceLatent's latent and others. Beta gate G1.
+ */
+describe("node availability", () => {
+  afterEach(() => {
+    vi.unstubAllGlobals();
+  });
+
+  // InpaintCropImproved as ComfyUI 0.44 reports it: `mask` is optional.
+  function stubInpaintCrop() {
+    vi.stubGlobal("fetch", async (url: string) => {
+      const classType = decodeURIComponent(String(url).split("/object_info/")[1] ?? "");
+      const body =
+        classType === "InpaintCropImproved"
+          ? { InpaintCropImproved: { input: { required: { image: ["IMAGE"] }, optional: { mask: ["MASK"] } } } }
+          : {};
+      return new Response(JSON.stringify(body), { status: 200 });
+    });
+  }
+
+  it("gives the preset checks every declared input, optional ones included", async () => {
+    stubInpaintCrop();
+    const nodes = await new ComfyClient("http://127.0.0.1:8188").getWorkflowNodeAvailability([
+      getWorkflowPreset("inpaint-flux2-klein")
+    ]);
+
+    expect(nodes.InpaintCropImproved).toEqual(["image", "mask"]);
+  });
+
+  it("keeps the custom workflow check on required inputs only", async () => {
+    // A pasted graph that leaves out an optional input is not broken.
+    stubInpaintCrop();
+    const nodes = await new ComfyClient("http://127.0.0.1:8188").getNodeAvailability(["InpaintCropImproved"]);
+
+    expect(nodes.InpaintCropImproved).toEqual(["image"]);
+  });
+
+  it("no longer reports a preset as missing a node when its input is optional", () => {
+    const preset = getWorkflowPreset("inpaint-flux2-klein");
+    const crop = preset.requiredNodes.find((node) => node.classType === "InpaintCropImproved");
+    const availableNodes = Object.fromEntries(preset.requiredNodes.map((node) => [node.classType, node.requiredInputs]));
+
+    expect(crop?.requiredInputs).toContain("mask");
+    expect(createWorkflowHealthItem(preset, { availableNodes }).state).not.toBe("missing-node");
+  });
+});