diff --git a/.gitattributes b/.gitattributes index 722d05e446c..b0304354a9a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -6,3 +6,4 @@ packages/registry-types/src/comfyRegistryTypes.ts linguist-generated=true packages/ingest-types/src/types.gen.ts linguist-generated=true packages/ingest-types/src/zod.gen.ts linguist-generated=true src/workbench/extensions/manager/types/generatedManagerTypes.ts linguist-generated=true +browser_tests/fixtures/data/customNodeManifest.cloud.json linguist-generated=true diff --git a/.github/actions/changes-filter/action.yaml b/.github/actions/changes-filter/action.yaml index 3a37ca4a325..b1d9341e68d 100644 --- a/.github/actions/changes-filter/action.yaml +++ b/.github/actions/changes-filter/action.yaml @@ -51,7 +51,7 @@ runs: - name: Filter typed changes if: ${{ github.event_name == 'pull_request' }} id: filter - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 with: filters: | app_website: @@ -75,7 +75,7 @@ runs: - name: Filter relevant changes if: ${{ github.event_name == 'pull_request' }} id: relevant - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 with: predicate-quantifier: 'every' filters: | diff --git a/.github/actions/setup-comfyui-server/action.yaml b/.github/actions/setup-comfyui-server/action.yaml index f05f75c9e02..f9bb9b0bd37 100644 --- a/.github/actions/setup-comfyui-server/action.yaml +++ b/.github/actions/setup-comfyui-server/action.yaml @@ -9,6 +9,10 @@ inputs: description: 'Whether to launch the server after setup' required: false default: 'false' + comfyui_ref: + description: 'ComfyUI git ref to check out (tag/branch/SHA). Empty = default branch (master).' + required: false + default: '' runs: using: 'composite' steps: @@ -19,6 +23,7 @@ runs: uses: actions/checkout@v7 with: repository: 'comfyanonymous/ComfyUI' + ref: ${{ inputs.comfyui_ref }} path: 'ComfyUI' - name: Install ComfyUI_devtools from frontend repo diff --git a/.github/actions/setup-playwright/action.yaml b/.github/actions/setup-playwright/action.yaml index 63e0c036210..0897c8c820b 100644 --- a/.github/actions/setup-playwright/action.yaml +++ b/.github/actions/setup-playwright/action.yaml @@ -1,5 +1,10 @@ name: Setup Playwright description: Cache and install Playwright browsers with dependencies +inputs: + install_deps: + description: Install Playwright operating-system dependencies + required: false + default: 'true' runs: using: composite steps: @@ -17,12 +22,33 @@ runs: path: '~/.cache/ms-playwright' key: ${{ runner.os }}-playwright-browsers-${{ steps.detect-version.outputs.playwright-version }} - - name: Install Playwright Browsers - if: steps.cache-playwright-browsers.outputs.cache-hit != 'true' + - name: Bound Playwright dependency downloads + if: inputs.install_deps == 'true' shell: bash - run: pnpm exec playwright install chromium --with-deps + run: | + set -euo pipefail + printf '%s\n' \ + 'Acquire::Retries "3";' \ + 'Acquire::http::Timeout "60";' \ + 'Acquire::https::Timeout "60";' \ + | sudo tee /etc/apt/apt.conf.d/80-playwright-ci >/dev/null - - name: Install Playwright Browsers (operating system dependencies) - if: steps.cache-playwright-browsers.outputs.cache-hit == 'true' + - name: Install Playwright browser and dependencies shell: bash - run: pnpm exec playwright install-deps + env: + BROWSER_CACHE_HIT: ${{ steps.cache-playwright-browsers.outputs.cache-hit }} + INSTALL_DEPS: ${{ inputs.install_deps }} + run: | + set -euo pipefail + if [ "$INSTALL_DEPS" != true ]; then + playwright_args=(install chromium) + elif [ "$BROWSER_CACHE_HIT" = true ]; then + playwright_args=(install-deps chromium) + else + playwright_args=(install chromium --with-deps) + fi + for attempt in 1 2; do + pnpm exec playwright "${playwright_args[@]}" && exit 0 + [ "$attempt" -lt 2 ] || exit 1 + sleep 2 + done diff --git a/.github/workflows/ci-tests-custom-nodes.yaml b/.github/workflows/ci-tests-custom-nodes.yaml new file mode 100644 index 00000000000..85cc5f31d1d --- /dev/null +++ b/.github/workflows/ci-tests-custom-nodes.yaml @@ -0,0 +1,751 @@ +# Runs the custom-node regression suite against a backend that has the manifest +# packs actually installed, so the load/run tiers execute for real. Within a +# run the result fails closed: if a pack fails to install or any tier is +# skipped, the job goes red - a suite that let a broken pack through as a +# "skip" would be pointless. +# +# NOT a PR gate yet: nightly schedule and manual dispatch only. PRs neither +# trigger this workflow nor wait on any of its checks, and none of its checks +# may be marked required in branch protection. Before ever adding a +# pull_request trigger, resolve the untrusted-source surface: the install loop +# fetches and pip-installs whatever the manifest's deployRefs point at +# (setup.py executes at install time), so a fork PR editing the manifest could +# aim it at attacker-controlled source. To test a branch, dispatch with the +# `branch` input. +# +# The matrix keeps the six-pack Core depth suite intact and adds the pinned +# Cloud snapshot as five breadth shards. The Cloud snapshot produces 87 +# manifest rows and one unjoined YAML pack; the unjoined pack and four reviewed +# quarantine entries are reported as excluded coverage. Every entry owns an +# isolated LOCAL ComfyUI backend and installs only its manifest slice. +# +# The gate is deterministic at the SOURCE level: ComfyUI core is pinned at a +# verified commit and every pack is installed at the exact deployRef the pinned +# Cloud snapshot declares - a commit for git-sourced packs, or a registry +# version's CDN artifact - so no source surface moves underneath a run. Python +# dependencies still resolve fresh, and the runner image updates independently. +name: 'CI: Tests Custom Nodes' + +on: + schedule: + - cron: '15 9 * * *' + workflow_dispatch: + inputs: + detection_proof_row: + description: 'Clean run or deliberate independent-tier break' + required: false + type: choice + options: ['0', 'all', '1', '2', '3', '9', '15'] + default: '0' + branch: + description: 'Frontend branch/ref to test (empty = the ref this workflow runs from)' + required: false + type: string + default: '' + grep: + description: "Playwright -g filter, e.g. 'VideoHelperSuite' or 'Pack startup' (empty = full suite; the exact-count gate is skipped when set)" + required: false + type: string + default: '' + comfyui_ref: + description: 'ComfyUI core ref to run against (empty = the verified pin)' + required: false + type: string + default: '' + +concurrency: + # Dispatches get a per-run group: two manual runs (e.g. comparing branches) + # must not cancel each other, while overlapping nightlies supersede per-ref. + group: ${{ github.event_name == 'workflow_dispatch' && format('{0}-dispatch-{1}', github.workflow, github.run_id) || format('{0}-{1}', github.workflow, github.ref) }} + cancel-in-progress: true + +env: + COMFYUI_PIN: b08e6cf35fac50d3ca8470dffb3f9a1fbb7187d2 + +jobs: + # Each job is intentionally one worker: the auto-run tier needs exclusive + # access to its backend's queue. Parallelism comes from the shard matrix + # instead, and every job owns a separate backend. + custom_nodes_e2e: + strategy: + fail-fast: false + matrix: + # THE shard-count knob. Each value is the `index/total` the manifest + # loader wants, so the count lives in exactly one place and the list + # cannot drift out of step with a separate total. Slices are balanced + # by expectedNodeCount (see shardOf), not by pack count: node counts + # run 1..285 with a median of 6, and a matrix costs its slowest shard. + shard: ['core', '1/5', '2/5', '3/5', '4/5', '5/5'] + proof_row: >- + ${{ fromJSON(github.event_name == 'workflow_dispatch' && inputs.detection_proof_row == 'all' && '["1","2","3","9","15"]' || format('["{0}"]', inputs.detection_proof_row || '0')) }} + exclude: + - shard: 'core' + proof_row: '1' + - shard: '1/5' + proof_row: '1' + - shard: '2/5' + proof_row: '1' + - shard: '3/5' + proof_row: '1' + - shard: '5/5' + proof_row: '1' + - shard: 'core' + proof_row: '2' + - shard: '1/5' + proof_row: '2' + - shard: '2/5' + proof_row: '2' + - shard: '3/5' + proof_row: '2' + - shard: '5/5' + proof_row: '2' + - shard: 'core' + proof_row: '3' + - shard: '1/5' + proof_row: '3' + - shard: '2/5' + proof_row: '3' + - shard: '3/5' + proof_row: '3' + - shard: '5/5' + proof_row: '3' + - shard: 'core' + proof_row: '9' + - shard: '1/5' + proof_row: '9' + - shard: '3/5' + proof_row: '9' + - shard: '4/5' + proof_row: '9' + - shard: '5/5' + proof_row: '9' + - shard: '1/5' + proof_row: '15' + - shard: '2/5' + proof_row: '15' + - shard: '3/5' + proof_row: '15' + - shard: '4/5' + proof_row: '15' + - shard: '5/5' + proof_row: '15' + runs-on: ubuntu-latest + # Recent green runs complete in 10-19 minutes per shard. 35 leaves room for a + # slow dependency resolve without letting a wedged backend run indefinitely. + timeout-minutes: 35 + permissions: + contents: read + # Job-level so every step that installs packs, runs the suite, or gates its + # results reads the same slice. The specs and the install loop both derive + # their pack list from these three (see scripts/customNodeShard.ts). + env: + CUSTOM_NODES_MANIFEST: ${{ matrix.shard == 'core' && 'core' || 'cloud' }} + CUSTOM_NODES_SHARD: ${{ matrix.shard == 'core' && '1/1' || matrix.shard }} + steps: + # Artifact and cache names cannot contain `/`, which the shard id does. + - name: Resolve shard identity + shell: bash + run: | + set -euo pipefail + if ! [[ "$CUSTOM_NODES_SHARD" =~ ^[0-9]+/[0-9]+$ ]]; then + echo "::error::matrix shard '$CUSTOM_NODES_SHARD' is not index/total"; exit 1 + fi + echo "SHARD_SLUG=${CUSTOM_NODES_MANIFEST}-${CUSTOM_NODES_SHARD/\//-of-}" >> "$GITHUB_ENV" + + - name: Checkout repository + uses: actions/checkout@v7 + with: + # workflow_dispatch may aim the suite at any frontend branch; every + # other event tests its own ref exactly as before. + ref: ${{ inputs.branch || github.ref }} + + - name: Apply Detection Proof source break + id: source-mutation + if: >- + github.event_name == 'workflow_dispatch' && + matrix.proof_row != '0' && + matrix.proof_row != '9' + shell: bash + env: + ROW: ${{ matrix.proof_row }} + run: | + set -euo pipefail + d=browser_tests/tests/customNodes/detection-proof + p=$(printf '%s/row-%02d-'"*.patch" "$d" "$ROW") + # shellcheck disable=SC2086 + p=$(ls $p 2>/dev/null | head -1 || true) + [ -n "$p" ] || { echo "::error::no patch file for S$ROW"; exit 1; } + git apply --check "$p" + git apply "$p" + if git diff --quiet -- src/; then + echo "::error::S$ROW patch did not change src/" + exit 1 + fi + echo "path=$p" >> "$GITHUB_OUTPUT" + echo "digest=$(shasum -a 256 "$p" | awk '{print $1}')" >> "$GITHUB_OUTPUT" + + - name: Setup frontend + uses: ./.github/actions/setup-frontend + with: + include_build_step: true + node_cache: 'false' + + - name: Verify Detection Proof witness shard + if: >- + github.event_name == 'workflow_dispatch' && + matrix.proof_row != '0' + shell: bash + env: + PROOF_ROW: ${{ matrix.proof_row }} + run: | + set -euo pipefail + case "$PROOF_ROW" in + 1|2|3) witness=comfyui-impact-pack ;; + 9) witness=comfyui-videohelpersuite ;; + 15) witness=ComfyUI-Impact-Pack ;; + esac + pnpm --silent custom-node-shard | cut -f1 | grep -qxF "$witness" || { + echo "::error::S$PROOF_ROW proof ran on a shard that does not own $witness" + exit 1 + } + + - name: Report cloud-manifest provenance + if: env.CUSTOM_NODES_MANIFEST == 'cloud' + shell: bash + run: | + set -euo pipefail + jq -r '.source | "Cloud pack snapshot: \(.repository)/blob/\(.ref)/\(.path) (imported \(.importedAt))"' \ + browser_tests/fixtures/data/customNodeManifest.cloud.json | tee /tmp/cloud-manifest-source.txt + { + echo '### Cloud pack snapshot' + echo + sed 's/^/- /' /tmp/cloud-manifest-source.txt + } >> "$GITHUB_STEP_SUMMARY" + + - name: Setup Playwright + uses: ./.github/actions/setup-playwright + with: + install_deps: 'false' + + # Checks out ComfyUI, installs Python/torch/requirements and ComfyUI_devtools. + # launch_server:false so we can add the manifest packs before booting. + # comfyui_ref pins core so the gate is reproducible: COMFYUI_PIN is the + # exact master commit the suite (incl. the KJNodes ledger entries and + # per-pack node counts) was last verified green against. Bump it + # deliberately, in its own PR, recalibrating ledgers/counts if the run + # says so. + - name: Setup ComfyUI server + uses: ./.github/actions/setup-comfyui-server + with: + launch_server: 'false' + # workflow_dispatch may override the pin (e.g. probing a core + # candidate before a deliberate bump); every gating event stays + # pinned. + comfyui_ref: ${{ inputs.comfyui_ref || env.COMFYUI_PIN }} + + # Pack sources are pinned, so their content is fully determined by the + # manifest: cache the fetched trees (sans .git) and restore by manifest + # hash, with a prefix fallback so a single-pack deployRef bump still + # reuses every other pack. The per-pack .ref marker is the integrity + # check - a stale or partial entry re-fetches. + - name: Restore manifest pack cache + id: pack-cache + uses: actions/cache/restore@v5 + with: + path: ~/.cache/cn-packs + key: cn-packs-v3-${{ env.SHARD_SLUG }}-${{ hashFiles('browser_tests/fixtures/data/customNodeManifest.*.json') }} + restore-keys: cn-packs-v3-${{ env.SHARD_SLUG }}- + + # Install exactly the packs this shard owns (DRY: a new pack row installs + # itself here, no workflow change). The slice comes from the same loader + # the specs register tests from, never from a stripe re-implemented in + # jq - the two disagreeing would install packs nothing tests and register + # tests for packs the backend lacks. A fetch or dependency failure fails + # the job - if a pack can't be installed, its coverage can't run, and + # that is a gate failure, not something to paper over. A `| while` pipe + # hides failures in a subshell, so read into an array and loop with + # `set -e`. + # Quarantined packs are excluded from the population, which is lost + # coverage that looks like a green run. + # Runs before install so the excluded-pack block is at the TOP of the + # summary, and `always()` so a red suite still reports what it never + # measured - the case where that block matters most. + - name: Report excluded packs + if: always() && env.CUSTOM_NODES_MANIFEST == 'cloud' && matrix.shard == '1/5' + run: pnpm custom-node-quarantine + + - name: Install manifest custom nodes + id: install + shell: bash + run: | + set -euo pipefail + # Pin the CPU torch stack that setup-comfyui-server installed so no + # pack's requirements.txt can pull a GPU/incompatible torch onto this + # --cpu runner. A pack that genuinely needs a different torch fails + # the constrained install loudly rather than silently swapping it. + pip freeze | grep -iE '^(torch|torchvision|torchaudio)==' \ + > /tmp/torch-constraints.txt || true + pnpm --silent custom-node-shard > /tmp/shard-packs.tsv + mapfile -t rows < /tmp/shard-packs.tsv + if [ "${#rows[@]}" -eq 0 ]; then + echo "::error::shard $CUSTOM_NODES_SHARD owns no packs"; exit 1 + fi + fetch_commit() { + local target=$1 commit=$2 + for attempt in 1 2 3; do + if git -C "$target" fetch --depth 1 origin "$commit"; then + return 0 + fi + if [ "$attempt" -lt 3 ]; then sleep $((attempt * 2)); fi + done + return 1 + } + echo "shard $CUSTOM_NODES_SHARD owns ${#rows[@]} pack(s)" + uninstallable=() + for row in "${rows[@]}"; do + # Install under the manifest `pack` key, not anything derived from + # the deployRef: node attribution keys on the install dirname via + # python_module, and the two only coincide by luck. Same charset + # the manifest loader enforces - belt for anything that bypasses it. + pack=${row%%$'\t'*} + ref=${row#*$'\t'} + if ! [[ "$pack" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + echo "::error::unsafe pack name: '$pack'"; exit 1 + fi + dir="ComfyUI/custom_nodes/$pack" + cache_dir="$HOME/.cache/cn-packs/$pack" + # The ref marker is a SIBLING of the tree, never inside it - a + # pack legitimately shipping a top-level .ref must install + # identically on cached and fresh runs. + if [ -f "$cache_dir.ref" ] && [ "$(cat "$cache_dir.ref")" = "$ref" ]; then + echo "::group::restore $pack from cache ($ref)" + cp -R "$cache_dir" "$dir" + else + echo "::group::install $pack ($ref)" + case "$ref" in + http://*|https://*) + url=${ref%@*} + sha=${ref##*@} + # git reads a leading `-` as an option, and options such as + # --upload-pack= run a command on the runner. + if ! [[ "$url" =~ ^https://github\.com/[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]]; then + echo "::error::$pack: deployRef repo must be an https://github.com// URL (got '$url')"; exit 1 + fi + # The gate tests exactly what Cloud deploys: a full SHA is + # mandatory here, before anything installs. + if ! [[ "$sha" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::$pack: deployRef must pin a full commit SHA (got '$sha')"; exit 1 + fi + # `clone --depth 1` can only land on a branch tip, and a + # deployRef commit frequently is not one. + git init -q -b main "$dir" + git -C "$dir" remote add origin "$url" + fetch_commit "$dir" "$sha" || { + echo "::error::$pack: deployRef '$ref' is unfetchable - the pinned commit does not exist upstream. Falling back to HEAD would test something the manifest does not declare, so fix the ref in cloud's supported_nodes.yaml"; exit 1 + } + git -C "$dir" checkout -q FETCH_HEAD + ;; + *) + id=${ref%@*} + version=${ref##*@} + if ! [[ "$id" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || ! [[ "$version" =~ ^[A-Za-z0-9][A-Za-z0-9.+_-]*$ ]]; then + echo "::error::$pack: deployRef must be @ (got '$ref')"; exit 1 + fi + # A registry version record carries no commit - it points at + # the CDN zip, and that artifact is what Cloud deploys, so + # installing from it is the faithful path. + download=$(curl --retry 3 --retry-delay 2 --retry-connrefused -sfL "https://api.comfy.org/nodes/$id/versions/$version" | jq -er '.downloadUrl') || { + echo "::error::$pack: deployRef '$ref' is unfetchable - the registry has no downloadUrl for that version"; exit 1 + } + if ! [[ "$download" =~ ^https://cdn\.comfy\.org/ ]]; then + echo "::error::$pack: registry downloadUrl is not on cdn.comfy.org (got '$download')"; exit 1 + fi + curl --retry 6 --retry-delay 5 --retry-all-errors \ + --retry-max-time 60 --remove-on-error -sfL \ + "$download" -o /tmp/pack.zip || { + echo "::error::$pack: deployRef '$ref' is unfetchable - the registry artifact $download did not download"; exit 1 + } + # Registry zips are the pack root, with no wrapper directory + # to strip (checked across all 47 cloud registry artifacts). + unzip -q /tmp/pack.zip -d "$dir" + rm -f /tmp/pack.zip + ;; + esac + # Repopulate the cache entry: tree without .git (nothing after + # this point reads pack git state), ref written LAST so a + # truncated copy fails the integrity check above. + rm -rf "$cache_dir" "$cache_dir.ref" + mkdir -p "$(dirname "$cache_dir")" + cp -R "$dir" "$cache_dir" + rm -rf "$cache_dir/.git" + printf '%s' "$ref" > "$cache_dir.ref" + fi + if [ -f "$dir/requirements.txt" ]; then + # Collected, not aborted. An unsatisfiable requirement upstream + # (radiance pins Imath>=3.1.0; PyPI has 0.0.2) is one pack's + # problem, and failing here took the other 21 packs in the shard + # down with it - a real pack bug erasing unrelated coverage. + if ! pip install -r "$dir/requirements.txt" \ + -c /tmp/torch-constraints.txt; then + uninstallable+=("$pack") + rm -rf "$dir" + fi + fi + echo "::endgroup::" + done + # Still a gate failure - a pack that cannot install has no coverage - + # but reported once, naming every offender, after the rest installed. + if [ ${#uninstallable[@]} -gt 0 ]; then + echo "::error::${#uninstallable[@]} pack(s) failed to install: ${uninstallable[*]}" + exit 1 + fi + # Prune entries for packs this shard no longer owns: the prefix + # fallback restores old entries and the save re-tars the whole + # directory, so without this the entry grows monotonically and + # eats the repo's shared cache budget. + if [ -d "$HOME/.cache/cn-packs" ]; then + current=$(cut -f1 /tmp/shard-packs.tsv) + for entry in "$HOME/.cache/cn-packs"/*; do + name=$(basename "$entry" .ref) + grep -qxF "$name" <<<"$current" || rm -rf "$entry" + done + fi + + # Save only when this run rebuilt entries. An exact-key restore means + # the stored tree is already current - actions/cache/save re-tars and + # uploads unconditionally and only then hits the duplicate-key + # rejection, so skipping here is what actually avoids the cost. Gating + # on the INSTALL step (not the job) still saves when the suite fails, + # while a failed or cancelled install can never freeze a partial tree + # under the immutable key. + - name: Save manifest pack cache + if: always() && steps.install.outcome == 'success' && steps.pack-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v5 + with: + path: ~/.cache/cn-packs + key: cn-packs-v3-${{ env.SHARD_SLUG }}-${{ hashFiles('browser_tests/fixtures/data/customNodeManifest.*.json') }} + + - name: Apply Detection Proof execution break + id: execution-mutation + if: >- + github.event_name == 'workflow_dispatch' && + matrix.proof_row == '9' + shell: bash + run: | + set -euo pipefail + f=ComfyUI/custom_nodes/comfyui-videohelpersuite/videohelpersuite/nodes.py + perl -0pi -e 's/(class LoadAudioUpload:.*?def load_audio\(self, start_time=0, duration=0, \*\*kwargs\):)/$1\n raise ValueError("DETECTION PROOF (row 9): pack node runtime failure")/s' "$f" + [ "$(grep -c 'DETECTION PROOF (row 9)' "$f")" = 1 ] + grep -q '"VHS_LoadAudioUpload": LoadAudioUpload' "$f" + echo "path=$f" >> "$GITHUB_OUTPUT" + echo "digest=$(shasum -a 256 "$f" | awk '{print $1}')" >> "$GITHUB_OUTPUT" + + - name: Bind Detection Proof identity + if: >- + github.event_name == 'workflow_dispatch' && + matrix.proof_row != '0' + shell: bash + env: + MUTATION_DIGEST: ${{ steps.source-mutation.outputs.digest || steps.execution-mutation.outputs.digest }} + MUTATION_PATH: ${{ steps.source-mutation.outputs.path || steps.execution-mutation.outputs.path }} + PROOF_ROW: ${{ matrix.proof_row }} + run: | + set -euo pipefail + PROOF_SHA=$(git rev-parse HEAD) + [ -n "$MUTATION_PATH" ] && [ -n "$MUTATION_DIGEST" ] || { echo '::error::mutation identity is incomplete'; exit 1; } + case "$PROOF_ROW" in + 1) title='S1: every registered node mounts on the canvas renderer'; pat='ImpactInt: instance is missing declared input \\"value\\" \\(litegraph\\)' ;; + 2) title='S2: every registered node mounts on the DOM renderer'; pat='ImpactInt: Vue mounts 0 of 1 widgets' ;; + 3) title='S3: registered-node save/reload outcomes match exact contracts'; pat='ImpactInt: widgets_values \[1\] -> \[0\] on set-values reload' ;; + 9) title='S9: calibrated model-free node corpus executes'; pat='DETECTION PROOF \\(row 9\\): pack node runtime failure' ;; + 15) title='Curated workflow execution: completes without error'; pat='ComfyUI-Impact-Pack/impact_primitives_run\\.json 2: output hash changed' ;; + esac + { + echo "proof_sha=$PROOF_SHA" + echo "tier=S$PROOF_ROW" + echo "test_identity=$title" + echo "mutation_path=$MUTATION_PATH" + echo "mutation_sha256=$MUTATION_DIGEST" + echo "expected_failure_regex=$pat" + echo 'workers=1' + echo 'retries=0' + } | tee tier-isolation-proof-identity.txt + + - name: Stage deterministic inputs + shell: bash + run: | + set -euo pipefail + mkdir -p ComfyUI/input + cp browser_tests/assets/test_upload_image.png ComfyUI/input/000_custom_node_probe.png + python - <<'PY' + import wave + + with wave.open('ComfyUI/input/000_custom_node_probe.wav', 'wb') as audio: + audio.setnchannels(1) + audio.setsampwidth(2) + audio.setframerate(8000) + audio.writeframes(b'\0\0' * 8000) + PY + cp ComfyUI/input/000_custom_node_probe.wav ComfyUI/input/001_custom_node_probe.wav + pack_rows=$(pnpm --silent custom-node-shard) || { + echo '::error::could not resolve the custom-node shard manifest' + exit 1 + } + if ! cut -f1 <<<"$pack_rows" | grep -qix comfyui-videohelpersuite; then + echo 'This shard has no run-tier pack; staged only deterministic media probes.' + exit 0 + fi + mkdir -p ComfyUI/input/images + cp ComfyUI/input/000_custom_node_probe.wav ComfyUI/input/plain_audio.wav + cp browser_tests/assets/plain_video.mp4 ComfyUI/input/plain_video.mp4 \ + || { echo "::error::could not stage plain_video.mp4 - the VHS run tier has no input to execute on"; exit 1; } + cp browser_tests/assets/image32x32.webp ComfyUI/input/test_upload_image.webp + cp browser_tests/assets/image32x32.webp ComfyUI/input/images/test_upload_image.webp + + # --cache-none so retried run-tier tests re-execute every node (a cached + # node emits no `executing` event and would false-fail PARTIAL). + - name: Start ComfyUI server + shell: bash + working-directory: ComfyUI + run: | + python main.py --cpu --multi-user --cache-none --front-end-root ../dist & + wait-for-it --service 127.0.0.1:8188 -t 600 + + - name: Run custom-node suite + id: suite + continue-on-error: >- + ${{ github.event_name == 'workflow_dispatch' && matrix.proof_row != '0' }} + env: + CN_ENABLE_S15: ${{ env.CUSTOM_NODES_MANIFEST == 'core' && '1' || '0' }} + CN_OUTPUT_HASHES_CORE: ${{ env.COMFYUI_PIN }} + PLAYWRIGHT_JSON_OUTPUT_NAME: custom-nodes-results.json + PROOF_ROW: ${{ matrix.proof_row }} + # Through env, not inline interpolation: dispatch input must never + # be shell-parsed. + GREP_FILTER: ${{ inputs.grep }} + run: | + set -o pipefail + # workers=1: the auto-run tier needs exclusive backend-queue access; + # parallel workers interrupt each other's executions. + # --project=custom-nodes: the `@custom-nodes`-tagged specs, which the + # main e2e job's `chromium` project excludes (see playwright.config). + # Json feeds the skip gate below; html writes playwright-report/ + # (the upload step's target) and folds in the retain-on-failure + # traces. List keeps live diagnostics visible. + grep_args=() + case "$PROOF_ROW" in + 0) [ -n "${GREP_FILTER:-}" ] && grep_args=(--grep "$GREP_FILTER") ;; + 1|2|3|9) + grep_args=(--grep 'all nodes by tier @custom-nodes.*S(1|2|3|9):') + ;; + 15) + grep_args=(--grep 'custom node: ComfyUI-Impact-Pack @custom-nodes.*Curated workflow execution: completes without error') + ;; + esac + pnpm exec playwright test browser_tests/tests/customNodes/ \ + --project=custom-nodes --reporter=list,json,html --workers=1 --retries=0 \ + "${grep_args[@]}" 2>&1 | tee custom-nodes.log + + # A skip here means a pack or devtools did not load: on this backend every + # tier is meant to run, so a skip is a gate failure, not an honest pass. + - name: Forbid failed, skipped, or flaky tests + if: >- + always() && + !(github.event_name == 'workflow_dispatch' && matrix.proof_row != '0') + shell: bash + env: + SUITE_OUTCOME: ${{ steps.suite.outcome }} + GREP_FILTER: ${{ inputs.grep }} + run: | + set -euo pipefail + # The JSON result is mandatory even when the suite step failed: the + # gate must independently prove the failed, skipped, and flaky totals. + if [ ! -f custom-nodes-results.json ]; then + echo "::error::suite wrote no results json (${SUITE_OUTCOME:-unknown}) - the result gate could not run" + exit 1 + fi + collected=$(jq -er '[.stats.expected, .stats.unexpected, .stats.flaky, .stats.skipped] | add' custom-nodes-results.json) + echo "collected tests: $collected" + # Derived from this shard's slice, not a literal: balanced slices + # differ in size and in how many rows carry the run tier, so one + # hardcoded number would be wrong on most shards. Same loader the + # specs register from, so a slice that silently shrank is caught + # here rather than passing as a smaller green run. + expected_tests=$(pnpm --silent custom-node-shard --expected-tests) + # A dispatch grep runs a deliberate subset; the exact-count gate + # applies only to full runs (every gating event runs unfiltered). + if [ -n "${GREP_FILTER:-}" ]; then + echo "grep filter active ('$GREP_FILTER') - exact-count gate skipped for this manual subset run" + elif [ "$collected" != "$expected_tests" ]; then + echo "::error::collected $collected tests; expected exactly $expected_tests for shard $CUSTOM_NODES_SHARD" + exit 1 + fi + unexpected=$(jq -er '.stats.unexpected | numbers' custom-nodes-results.json) + echo "failed tests: $unexpected" + if [ "$unexpected" != "0" ]; then + echo "::error::$unexpected test(s) failed - failed results are not acceptable in the gating job" + jq -r '.. | objects + | select(has("title") and has("tests")) + | select(any(.tests[]?; .status == "unexpected")) + | .title' custom-nodes-results.json | sort -u | head -40 + exit 1 + fi + skipped=$(jq -er '.stats.skipped | numbers' custom-nodes-results.json) + echo "skipped tests: $skipped" + if [ "$skipped" != "0" ]; then + echo "::error::$skipped test(s) skipped - a manifest pack or devtools failed to load; skips are not acceptable in the gating job" + jq -r '.. | objects + | select(has("title") and has("tests")) + | select(any(.tests[]?; .status == "skipped")) + | .title' custom-nodes-results.json | sort -u | head -40 + exit 1 + fi + flaky=$(jq -er '.stats.flaky | numbers' custom-nodes-results.json) + echo "flaky tests: $flaky" + if [ "$flaky" != "0" ]; then + echo "::error::$flaky test(s) passed only after retry - flaky results are not acceptable in the gating job" + jq -r '.. | objects + | select(has("title") and has("tests")) + | select(any(.tests[]?; .status == "flaky")) + | .title' custom-nodes-results.json | sort -u | head -40 + exit 1 + fi + + - name: Assert deliberate failure independence + if: >- + always() && + github.event_name == 'workflow_dispatch' && + matrix.proof_row != '0' + shell: bash + env: + PROOF_ROW: ${{ matrix.proof_row }} + SUITE_OUTCOME: ${{ steps.suite.outcome }} + run: | + set -euo pipefail + [ -f custom-nodes-results.json ] || { echo "::error::proof wrote no JSON"; exit 1; } + collected=$(jq -er '[.stats.expected, .stats.unexpected, .stats.flaky, .stats.skipped] | add' custom-nodes-results.json) + expected=$(jq -er '.stats.expected' custom-nodes-results.json) + unexpected=$(jq -er '.stats.unexpected' custom-nodes-results.json) + skipped=$(jq -er '.stats.skipped' custom-nodes-results.json) + flaky=$(jq -er '.stats.flaky' custom-nodes-results.json) + expected_collected=$(pnpm --silent custom-node-shard --expected-tier-tests) + if [ "$PROOF_ROW" = 15 ]; then + expected_collected=1 + fi + expected_passed=$((expected_collected - 1)) + expected_failed=1 + [ "$collected" = "$expected_collected" ] || { echo "::error::S$PROOF_ROW collected $collected tests, expected $expected_collected"; exit 1; } + [ "$expected" = "$expected_passed" ] || { echo "::error::S$PROOF_ROW expected-pass count is $expected, expected $expected_passed"; exit 1; } + [ "$unexpected" = "$expected_failed" ] || { echo "::error::S$PROOF_ROW failure count is $unexpected, expected $expected_failed"; exit 1; } + [ "$skipped" = 0 ] && [ "$flaky" = 0 ] || { echo "::error::S$PROOF_ROW has skipped=$skipped flaky=$flaky"; exit 1; } + [ "$SUITE_OUTCOME" = failure ] || { echo "::error::S$PROOF_ROW did not break its witness"; exit 1; } + + { + jq -r '.. | objects | select(.status? == "unexpected") | .results[]? | .error?.message? // empty' custom-nodes-results.json + jq -r '.. | objects | select(.status? == "unexpected") | .results[]?.attachments[]? | select(.contentType? == "application/json" and has("body")) | (.body | @base64d | fromjson) | if type == "array" then .[] else . end' custom-nodes-results.json + } > tier-isolation-proof-evidence.txt + if [ "$PROOF_ROW" = 15 ]; then + grep -Eq -- 'ComfyUI-Impact-Pack/impact_primitives_run\.json 2: output hash changed' tier-isolation-proof-evidence.txt || { + echo '::error::S15 failure was not attributable to the serialized-value break' + exit 1 + } + echo 'S15: serialized value drift changed a sink output hash while execution completed' + exit 0 + fi + pack_failures=$(jq -er --arg name "s${PROOF_ROW}-failures.json" ' + [.. | objects + | select(.status? == "unexpected") + | .results[]?.attachments[]? + | select(.name == $name and has("body")) + | (.body | @base64d | fromjson)] + | if length == 1 and (.[0] | type) == "array" + then .[0] | length + else -1 + end + ' custom-nodes-results.json) + [ "$pack_failures" = 1 ] || { echo "::error::S$PROOF_ROW produced $pack_failures pack failures, expected exactly one witness"; exit 1; } + case "$PROOF_ROW" in + 1) witness=comfyui-impact-pack; pat='ImpactInt: instance is missing declared input \\"value\\" \(litegraph\)' ;; + 2) witness=comfyui-impact-pack; pat='ImpactInt: Vue mounts 0 of 1 widgets' ;; + 3) witness=comfyui-impact-pack; pat='ImpactInt: widgets_values \[1\] -> \[0\] on set-values reload' ;; + 9) witness=comfyui-videohelpersuite; pat='DETECTION PROOF \(row 9\): pack node runtime failure' ;; + esac + grep -Fq -- "[$witness]" tier-isolation-proof-evidence.txt || { echo "::error::S$PROOF_ROW failure did not belong to $witness"; exit 1; } + grep -Eq -- "$pat" tier-isolation-proof-evidence.txt || { echo "::error::S$PROOF_ROW failure was not attributable to its Detection Proof break"; exit 1; } + + case "$PROOF_ROW" in + 1) target_tier=S1; target_title="S1: every registered node mounts on the canvas renderer"; next_tier=S2 ;; + 2) target_tier=S2; target_title="S2: every registered node mounts on the DOM renderer"; next_tier=S3 ;; + 3) target_tier=S3; target_title="S3: registered-node save/reload outcomes match exact contracts"; next_tier= ;; + 9) target_tier=S9; target_title="S9: calibrated model-free node corpus executes"; next_tier= ;; + esac + jq -e --arg title "$target_title" '.. | objects | select(.title? == $title) | .tests[] | select(.status == "unexpected")' custom-nodes-results.json >/dev/null + jq -e --arg title "$target_title" ' + [.. | objects + | select(.title? and .tests?) + | select(.title | test("^S(1|2|3|9): every ")) + | . as $suite + | .tests[] + | select($suite.title != $title and .status != "expected")] + | length == 0 + ' custom-nodes-results.json >/dev/null || { + echo "::error::a tier other than $target_tier did not pass independently" + exit 1 + } + mapfile -t tier_sessions < <(grep -oE '\[tier-session\] pid=[0-9]+ tier=S(1|2|3|9) pageId=[^ ]+' custom-nodes.log) + [ "${#tier_sessions[@]}" -eq "$expected_collected" ] || { echo "::error::$target_tier proof recorded ${#tier_sessions[@]} tier sessions, expected $expected_collected"; exit 1; } + page_ids=$(printf '%s\n' "${tier_sessions[@]}" | sed -E 's/.* pageId=([^ ]+).*/\1/' | sort -u | wc -l | tr -d ' ') + [ "$page_ids" = "$expected_collected" ] || { echo "::error::$target_tier proof reused an application page between tiers"; exit 1; } + pids=$(printf '%s\n' "${tier_sessions[@]}" | sed -E 's/.* pid=([0-9]+).*/\1/' | sort -u | wc -l | tr -d ' ') + if [ -n "$next_tier" ]; then + [ "$pids" = 2 ] || { echo "::error::$target_tier proof used $pids Playwright workers, expected the failed worker plus one replacement"; exit 1; } + target_pid=$(printf '%s\n' "${tier_sessions[@]}" | sed -nE "s/.*pid=([0-9]+) tier=$target_tier .*/\\1/p") + next_pid=$(printf '%s\n' "${tier_sessions[@]}" | sed -nE "s/.*pid=([0-9]+) tier=$next_tier .*/\\1/p") + [ -n "$target_pid" ] && [ -n "$next_pid" ] && [ "$target_pid" != "$next_pid" ] || { echo "::error::$next_tier did not run in a replacement worker after $target_tier failed"; exit 1; } + else + [ "$pids" = 1 ] || { echo "::error::$target_tier is last but proof used $pids Playwright workers"; exit 1; } + fi + echo "$target_tier: real break failed; every other tier passed in its own fresh-page test" + + # Renders what ran and what it found - to the job log as an aligned + # table and to the run's Summary panel as markdown - so a dispatch + # answers "which branch, against what, with what verdict" without + # opening the artifact. + - name: Publish results table + if: always() + shell: bash + env: + BRANCH_TESTED: ${{ inputs.branch || github.ref_name }} + COMFYUI_REF_USED: ${{ inputs.comfyui_ref || env.COMFYUI_PIN }} + GREP_FILTER: ${{ inputs.grep }} + run: python3 scripts/cicd/custom-nodes-summary.py + + - name: Upload Playwright report and raw diagnostics + if: always() + uses: actions/upload-artifact@v6 + with: + # Shard-scoped: artifact names are unique per run, so matrix jobs + # sharing one name cannot overwrite each other's reports. + name: ${{ github.event_name == 'workflow_dispatch' && matrix.proof_row != '0' && format('playwright-report-custom-nodes-{0}-row{1}', env.SHARD_SLUG, matrix.proof_row) || format('playwright-report-custom-nodes-{0}', env.SHARD_SLUG) }} + path: | + playwright-report/ + custom-nodes-results.json + custom-nodes.log + tier-isolation-proof-evidence.txt + tier-isolation-proof-identity.txt + retention-days: 7 + if-no-files-found: error + + custom-nodes-e2e-status: + name: custom-nodes-e2e-status + if: always() + needs: custom_nodes_e2e + runs-on: ubuntu-latest + steps: + - name: Enforce suite result + shell: bash + env: + E2E_RESULT: ${{ needs.custom_nodes_e2e.result }} + run: | + set -euo pipefail + [ "$E2E_RESULT" = success ] || { echo "::error::custom-node matrix ended with $E2E_RESULT"; exit 1; } diff --git a/.github/workflows/ci-tests-e2e-coverage.yaml b/.github/workflows/ci-tests-e2e-coverage.yaml index 73c970368ab..bf9e77cbb81 100644 --- a/.github/workflows/ci-tests-e2e-coverage.yaml +++ b/.github/workflows/ci-tests-e2e-coverage.yaml @@ -1,13 +1,30 @@ name: 'CI: E2E Coverage' on: + workflow_call: + inputs: + source_branch: + required: true + type: string + source_pr: + required: false + type: string + source_run_id: + required: true + type: string + source_sha: + required: true + type: string + secrets: + CODECOV_TOKEN: + required: true workflow_run: workflows: ['CI: Tests E2E'] types: - completed concurrency: - group: e2e-coverage-${{ github.event.workflow_run.head_sha }} + group: e2e-coverage-${{ inputs.source_sha || github.event.workflow_run.head_sha }} cancel-in-progress: true permissions: @@ -18,6 +35,10 @@ jobs: # Not gated on full success: fail-fast:false means one flaky shard flips the whole run to failure, which would skip merging every other shard's real coverage. if: > github.repository == 'Comfy-Org/ComfyUI_frontend' && + (github.event_name == 'workflow_call' || + github.event.workflow_run.event != 'pull_request' || + github.event.workflow_run.head_repository.full_name != github.repository || + github.event.workflow_run.actor.login == 'dependabot[bot]') && github.event.workflow_run.conclusion != 'cancelled' runs-on: ubuntu-latest timeout-minutes: 10 @@ -28,6 +49,7 @@ jobs: - name: Checkout sources for genhtml uses: actions/checkout@v7 with: + ref: ${{ inputs.source_sha || github.event.workflow_run.head_sha }} sparse-checkout: | src packages @@ -35,7 +57,7 @@ jobs: - name: Download all shard coverage data uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12 with: - run_id: ${{ github.event.workflow_run.id }} + run_id: ${{ inputs.source_run_id || github.event.workflow_run.id }} name: e2e-coverage-shard-.* name_is_regexp: true path: temp/coverage-shards @@ -122,6 +144,9 @@ jobs: with: files: coverage/playwright/coverage.lcov flags: e2e + override_branch: ${{ inputs.source_branch || github.event.workflow_run.head_branch }} + override_commit: ${{ inputs.source_sha || github.event.workflow_run.head_sha }} + override_pr: ${{ inputs.source_pr || github.event.workflow_run.pull_requests[0].number }} token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: false diff --git a/.github/workflows/ci-tests-e2e.yaml b/.github/workflows/ci-tests-e2e.yaml index 73d1cf1e693..14839851cda 100644 --- a/.github/workflows/ci-tests-e2e.yaml +++ b/.github/workflows/ci-tests-e2e.yaml @@ -223,6 +223,25 @@ jobs: [[ "$SHARDED" != "success" || "$BROWSERS" != "success" || ( "$VIDEO" != "success" && "$VIDEO" != "skipped" ) ]] && echo "E2E failed" && exit 1 echo "E2E passed" + upload-e2e-coverage: + needs: [changes, playwright-tests-chromium-sharded] + if: >- + ${{ + always() && + needs.changes.outputs.should-run == 'true' && + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.fork == false && + github.event.pull_request.user.login != 'dependabot[bot]' && + needs.playwright-tests-chromium-sharded.result != 'cancelled' + }} + uses: ./.github/workflows/ci-tests-e2e-coverage.yaml + with: + source_branch: ${{ github.head_ref || github.ref_name }} + source_pr: ${{ format('{0}', github.event.pull_request.number) }} + source_run_id: ${{ format('{0}', github.run_id) }} + source_sha: ${{ github.event.pull_request.head.sha || github.sha }} + secrets: inherit + # Records video only for spec files newly added in this PR, to keep cost bounded. playwright-video-new-tests: needs: setup diff --git a/.pinact.yaml b/.pinact.yaml index c06f41a161f..72ae227905b 100644 --- a/.pinact.yaml +++ b/.pinact.yaml @@ -12,6 +12,10 @@ files: ignore_actions: - name: actions/cache ref: v5 + - name: actions/cache/restore + ref: v5 + - name: actions/cache/save + ref: v5 - name: actions/checkout ref: v7 - name: actions/setup-node diff --git a/browser_tests/README.md b/browser_tests/README.md index ca7d6ae6ca9..5401350a845 100644 --- a/browser_tests/README.md +++ b/browser_tests/README.md @@ -206,6 +206,15 @@ flowchart TD E -- No, coordinates actions
across the app --> H[fixtures/helpers/] ``` +### Custom-node regression suite + +`tests/customNodes/` holds the manifest-driven suite that proves community +custom-node packs load, render in both renderers (LiteGraph canvas and Vue +Nodes 2.0), and execute real workflows. It has its own prerequisites, pnpm +scripts (`pnpm test:custom-nodes` and variants), and a one-JSON-row process +for adding packs - see +[docs/custom-node-regression-suite.md](../docs/custom-node-regression-suite.md). + ## Writing Tests ### Golden rules diff --git a/browser_tests/assets/customNodes/core_primitive_preview_run.json b/browser_tests/assets/customNodes/core_primitive_preview_run.json new file mode 100644 index 00000000000..86bb8b96028 --- /dev/null +++ b/browser_tests/assets/customNodes/core_primitive_preview_run.json @@ -0,0 +1,53 @@ +{ + "last_node_id": 2, + "last_link_id": 1, + "nodes": [ + { + "id": 1, + "type": "PrimitiveInt", + "pos": { "0": 20, "1": 60 }, + "size": { "0": 250, "1": 100 }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [1], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PrimitiveInt" + }, + "widgets_values": [5, "fixed"] + }, + { + "id": 2, + "type": "PreviewAny", + "pos": { "0": 340, "1": 60 }, + "size": { "0": 220, "1": 60 }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 1 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewAny" + } + } + ], + "links": [[1, 1, 0, 2, 0, "INT"]], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/browser_tests/assets/customNodes/core_smoke.json b/browser_tests/assets/customNodes/core_smoke.json new file mode 100644 index 00000000000..2f5fd36701b --- /dev/null +++ b/browser_tests/assets/customNodes/core_smoke.json @@ -0,0 +1,53 @@ +{ + "last_node_id": 2, + "last_link_id": 1, + "nodes": [ + { + "id": 1, + "type": "PrimitiveInt", + "pos": { "0": 20, "1": 60 }, + "size": { "0": 250, "1": 80 }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [1], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PrimitiveInt" + }, + "widgets_values": [42, "fixed"] + }, + { + "id": 2, + "type": "PreviewAny", + "pos": { "0": 340, "1": 60 }, + "size": { "0": 220, "1": 60 }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 1 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewAny" + } + } + ], + "links": [[1, 1, 0, 2, 0, "INT"]], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/browser_tests/assets/customNodes/customscripts_string_show_run.json b/browser_tests/assets/customNodes/customscripts_string_show_run.json new file mode 100644 index 00000000000..d98627d1856 --- /dev/null +++ b/browser_tests/assets/customNodes/customscripts_string_show_run.json @@ -0,0 +1,60 @@ +{ + "last_node_id": 2, + "last_link_id": 1, + "nodes": [ + { + "id": 1, + "type": "StringFunction|pysssss", + "pos": { "0": 20, "1": 60 }, + "size": { "0": 300, "1": 240 }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [1], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "StringFunction|pysssss" + }, + "widgets_values": ["append", "yes", "hello", " world", ""] + }, + { + "id": 2, + "type": "ShowText|pysssss", + "pos": { "0": 380, "1": 60 }, + "size": { "0": 220, "1": 80 }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "text", + "type": "STRING", + "link": 1 + } + ], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": null, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ShowText|pysssss" + } + } + ], + "links": [[1, 1, 0, 2, 0, "STRING"]], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/browser_tests/assets/customNodes/essentials_math_display_run.json b/browser_tests/assets/customNodes/essentials_math_display_run.json new file mode 100644 index 00000000000..2b1253e0489 --- /dev/null +++ b/browser_tests/assets/customNodes/essentials_math_display_run.json @@ -0,0 +1,61 @@ +{ + "last_node_id": 2, + "last_link_id": 1, + "nodes": [ + { + "id": 1, + "type": "SimpleMathInt+", + "pos": { "0": 20, "1": 60 }, + "size": { "0": 250, "1": 60 }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [1], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "SimpleMathInt+" + }, + "widgets_values": [5] + }, + { + "id": 2, + "type": "DisplayAny", + "pos": { "0": 340, "1": 60 }, + "size": { "0": 220, "1": 80 }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "input", + "type": "*", + "link": 1 + } + ], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": null, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DisplayAny" + }, + "widgets_values": ["raw value"] + } + ], + "links": [[1, 1, 0, 2, 0, "INT"]], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/browser_tests/assets/customNodes/impact_primitives_run.json b/browser_tests/assets/customNodes/impact_primitives_run.json new file mode 100644 index 00000000000..1b420fedfd0 --- /dev/null +++ b/browser_tests/assets/customNodes/impact_primitives_run.json @@ -0,0 +1,98 @@ +{ + "last_node_id": 4, + "last_link_id": 2, + "nodes": [ + { + "id": 1, + "type": "ImpactInt", + "pos": { "0": 20, "1": 60 }, + "size": { "0": 250, "1": 60 }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [1], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ImpactInt" + }, + "widgets_values": [42] + }, + { + "id": 2, + "type": "PreviewAny", + "pos": { "0": 340, "1": 60 }, + "size": { "0": 220, "1": 60 }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 1 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewAny" + } + }, + { + "id": 3, + "type": "ImpactFloat", + "pos": { "0": 20, "1": 220 }, + "size": { "0": 250, "1": 60 }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "FLOAT", + "type": "FLOAT", + "links": [2], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ImpactFloat" + }, + "widgets_values": [3.14] + }, + { + "id": 4, + "type": "PreviewAny", + "pos": { "0": 340, "1": 220 }, + "size": { "0": 220, "1": 60 }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 2 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewAny" + } + } + ], + "links": [ + [1, 1, 0, 2, 0, "INT"], + [2, 3, 0, 4, 0, "FLOAT"] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/browser_tests/assets/customNodes/kjnodes_constants_run.json b/browser_tests/assets/customNodes/kjnodes_constants_run.json new file mode 100644 index 00000000000..7d3fb744eaf --- /dev/null +++ b/browser_tests/assets/customNodes/kjnodes_constants_run.json @@ -0,0 +1,98 @@ +{ + "last_node_id": 4, + "last_link_id": 2, + "nodes": [ + { + "id": 1, + "type": "INTConstant", + "pos": { "0": 20, "1": 60 }, + "size": { "0": 250, "1": 60 }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [1], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "INTConstant" + }, + "widgets_values": [42] + }, + { + "id": 2, + "type": "PreviewAny", + "pos": { "0": 340, "1": 60 }, + "size": { "0": 220, "1": 60 }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 1 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewAny" + } + }, + { + "id": 3, + "type": "FloatConstant", + "pos": { "0": 20, "1": 220 }, + "size": { "0": 250, "1": 60 }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "FLOAT", + "links": [2], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "FloatConstant" + }, + "widgets_values": [3.14] + }, + { + "id": 4, + "type": "PreviewAny", + "pos": { "0": 340, "1": 220 }, + "size": { "0": 220, "1": 60 }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 2 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewAny" + } + } + ], + "links": [ + [1, 1, 0, 2, 0, "INT"], + [2, 3, 0, 4, 0, "FLOAT"] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/browser_tests/assets/customNodes/vhs_video_pipeline_cloud_run.json b/browser_tests/assets/customNodes/vhs_video_pipeline_cloud_run.json new file mode 100644 index 00000000000..363c67084ab --- /dev/null +++ b/browser_tests/assets/customNodes/vhs_video_pipeline_cloud_run.json @@ -0,0 +1,107 @@ +{ + "last_node_id": 3, + "last_link_id": 2, + "nodes": [ + { + "id": 1, + "type": "VHS_LoadVideo", + "pos": { "0": 20, "1": 60 }, + "size": { "0": 320, "1": 260 }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": null + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": [1], + "slot_index": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": ["plain_video.mp4", 0, 0, 0, 0, 0, 1] + }, + { + "id": 2, + "type": "VHS_VideoInfo", + "pos": { "0": 400, "1": 60 }, + "size": { "0": 240, "1": 260 }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "link": 1 + } + ], + "outputs": [ + { + "name": "source_fps🟨", + "type": "FLOAT", + "links": [2], + "slot_index": 0 + }, + { "name": "source_frame_count🟨", "type": "INT", "links": null }, + { "name": "source_duration🟨", "type": "FLOAT", "links": null }, + { "name": "source_width🟨", "type": "INT", "links": null }, + { "name": "source_height🟨", "type": "INT", "links": null }, + { "name": "loaded_fps🟦", "type": "FLOAT", "links": null }, + { "name": "loaded_frame_count🟦", "type": "INT", "links": null }, + { "name": "loaded_duration🟦", "type": "FLOAT", "links": null }, + { "name": "loaded_width🟦", "type": "INT", "links": null }, + { "name": "loaded_height🟦", "type": "INT", "links": null } + ], + "properties": { + "Node name for S&R": "VHS_VideoInfo" + } + }, + { + "id": 3, + "type": "PreviewAny", + "pos": { "0": 700, "1": 60 }, + "size": { "0": 220, "1": 60 }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 2 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewAny" + } + } + ], + "links": [ + [1, 1, 3, 2, 0, "VHS_VIDEOINFO"], + [2, 2, 0, 3, 0, "FLOAT"] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/browser_tests/assets/customNodes/vhs_video_pipeline_run.json b/browser_tests/assets/customNodes/vhs_video_pipeline_run.json new file mode 100644 index 00000000000..9599b687cff --- /dev/null +++ b/browser_tests/assets/customNodes/vhs_video_pipeline_run.json @@ -0,0 +1,107 @@ +{ + "last_node_id": 3, + "last_link_id": 2, + "nodes": [ + { + "id": 1, + "type": "VHS_LoadVideoPath", + "pos": { "0": 20, "1": 60 }, + "size": { "0": 320, "1": 260 }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": null + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": [1], + "slot_index": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideoPath" + }, + "widgets_values": ["input/plain_video.mp4", 0, 0, 0, 0, 0, 1] + }, + { + "id": 2, + "type": "VHS_VideoInfo", + "pos": { "0": 400, "1": 60 }, + "size": { "0": 240, "1": 260 }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "link": 1 + } + ], + "outputs": [ + { + "name": "source_fps🟨", + "type": "FLOAT", + "links": [2], + "slot_index": 0 + }, + { "name": "source_frame_count🟨", "type": "INT", "links": null }, + { "name": "source_duration🟨", "type": "FLOAT", "links": null }, + { "name": "source_width🟨", "type": "INT", "links": null }, + { "name": "source_height🟨", "type": "INT", "links": null }, + { "name": "loaded_fps🟦", "type": "FLOAT", "links": null }, + { "name": "loaded_frame_count🟦", "type": "INT", "links": null }, + { "name": "loaded_duration🟦", "type": "FLOAT", "links": null }, + { "name": "loaded_width🟦", "type": "INT", "links": null }, + { "name": "loaded_height🟦", "type": "INT", "links": null } + ], + "properties": { + "Node name for S&R": "VHS_VideoInfo" + } + }, + { + "id": 3, + "type": "PreviewAny", + "pos": { "0": 700, "1": 60 }, + "size": { "0": 220, "1": 60 }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 2 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewAny" + } + } + ], + "links": [ + [1, 1, 3, 2, 0, "VHS_VIDEOINFO"], + [2, 2, 0, 3, 0, "FLOAT"] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/browser_tests/assets/customNodes/was_number_text_run.json b/browser_tests/assets/customNodes/was_number_text_run.json new file mode 100644 index 00000000000..13073c3bb9c --- /dev/null +++ b/browser_tests/assets/customNodes/was_number_text_run.json @@ -0,0 +1,124 @@ +{ + "last_node_id": 4, + "last_link_id": 3, + "nodes": [ + { + "id": 1, + "type": "Constant Number", + "pos": { "0": 20, "1": 60 }, + "size": { "0": 250, "1": 100 }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "NUMBER", + "type": "NUMBER", + "links": [1], + "slot_index": 0 + }, + { + "name": "FLOAT", + "type": "FLOAT", + "links": null, + "slot_index": 1 + }, + { + "name": "INT", + "type": "INT", + "links": null, + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "Constant Number" + }, + "widgets_values": ["integer", 7] + }, + { + "id": 2, + "type": "Number to Text", + "pos": { "0": 340, "1": 60 }, + "size": { "0": 220, "1": 60 }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "number", + "type": "NUMBER", + "link": 1 + } + ], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [2, 3], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "Number to Text" + } + }, + { + "id": 3, + "type": "Text to Console", + "pos": { "0": 640, "1": 60 }, + "size": { "0": 250, "1": 80 }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "text", + "type": "STRING", + "link": 2 + } + ], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": null, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "Text to Console" + }, + "widgets_values": ["Text Output"] + }, + { + "id": 4, + "type": "PreviewAny", + "pos": { "0": 640, "1": 200 }, + "size": { "0": 220, "1": 60 }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 3 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewAny" + } + } + ], + "links": [ + [1, 1, 0, 2, 0, "NUMBER"], + [2, 2, 0, 3, 0, "STRING"], + [3, 2, 0, 4, 0, "STRING"] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/browser_tests/assets/widgets/all_load_widgets.json b/browser_tests/assets/widgets/all_load_widgets.json index a2d05c03f5b..5fbb0b48447 100644 --- a/browser_tests/assets/widgets/all_load_widgets.json +++ b/browser_tests/assets/widgets/all_load_widgets.json @@ -44,7 +44,7 @@ "properties": { "Node name for S&R": "LoadVideo" }, - "widgets_values": [null, "image"] + "widgets_values": ["this-video-does-not-exist-deadbeef.mp4", "image"] }, { "id": 3, diff --git a/browser_tests/fixtures/ComfyPage.ts b/browser_tests/fixtures/ComfyPage.ts index f5ac04f0fac..f4be11b1895 100644 --- a/browser_tests/fixtures/ComfyPage.ts +++ b/browser_tests/fixtures/ComfyPage.ts @@ -14,12 +14,24 @@ import { EMPTY_BILLING_PLANS, LEGACY_PERSONAL_BILLING_STATUS } from '@e2e/fixtures/data/cloudWorkspace' -import { ZERO_BALANCE } from '@e2e/fixtures/data/subscriptionFixtures' +import { + UNSUBSCRIBED, + ZERO_BALANCE +} from '@e2e/fixtures/data/subscriptionFixtures' import { ComfyActionbar } from '@e2e/fixtures/components/Actionbar' import { ComfyTemplates } from '@e2e/fixtures/components/Templates' import { ComfyMouse } from '@e2e/fixtures/ComfyMouse' import { TestIds } from '@e2e/fixtures/selectors' import { comfyExpect } from '@e2e/fixtures/utils/customMatchers' +import { + installCustomNodeBlankStartup, + runWithCollectedCleanup +} from '@e2e/fixtures/utils/customNodeSuite' +import { + collectConsoleErrors, + recordStartupConsoleErrors +} from '@e2e/fixtures/utils/consoleErrorCollector' +import { trackVisibleErrors } from '@e2e/fixtures/utils/errorSurfaces' import { assetPath } from '@e2e/fixtures/utils/paths' import { nextFrame, sleep } from '@e2e/fixtures/utils/timing' import { mockWorkspace, workspace } from '@e2e/fixtures/utils/workspaceMocks' @@ -286,8 +298,16 @@ export class ComfyPage { data: { username } }) - if (resp.status() !== 200) - throw new Error(`Failed to create user: ${await resp.text()}`) + if (resp.status() !== 200) { + const body = await resp.text() + // Persistent backends (Comfy Desktop server user storage) keep the user + // across runs and do not list it via GET /api/users, so a duplicate means + // it already exists. Returns the username since the generated id is not + // retrievable here; only reached on single-user / default-resolving backends. + if (resp.status() === 400 && body.includes('Duplicate username.')) + return username + throw new Error(`Failed to create user: ${body}`) + } return await resp.json() } @@ -357,15 +377,57 @@ export class ComfyPage { * `WorkflowHelper.reloadAndWaitForApp()`. */ async waitForAppReady() { - await this.page.waitForFunction( - // window.app => GraphCanvas ready - // window.app.extensionManager => GraphView ready - () => window.app?.extensionManager - ) - await this.page.locator('.p-blockui-mask').waitFor({ state: 'hidden' }) + const readyFuseMs = 300_000 + try { + await this.page.waitForFunction( + // window.app => GraphCanvas ready + // window.app.extensionManager => GraphView ready + () => window.app?.extensionManager, + null, + { timeout: readyFuseMs } + ) + await this.page + .locator('.p-blockui-mask') + .waitFor({ state: 'hidden', timeout: readyFuseMs }) + } catch (error) { + const state = await this.describeUnreadyApp() + throw new Error(`app never became ready: ${state}`, { cause: error }) + } await this.nextFrame() } + /** + * Why the app is not ready, in one line, for the failure message. A bare + * "timeout exceeded" cannot tell a stalled sign-in from a crashed boot from + * a backend that never answered, which is the difference between a + * five-minute fix and a day of guessing - so every unready-app failure + * carries this. Best-effort by construction: it runs on an already-failing + * page, so it reports what it could not read rather than throwing over it. + */ + private async describeUnreadyApp(): Promise { + try { + const state = await this.page.evaluate(() => ({ + url: location.href, + title: document.title, + hasApp: !!window.app, + hasExtensionManager: !!window.app?.extensionManager, + blockUiVisible: !!document.querySelector('.p-blockui-mask'), + signInVisible: !!document.querySelector( + '[data-testid*="sign-in"], [class*="SignIn"], form[action*="signin"]' + ), + bodyText: document.body?.innerText?.slice(0, 300) ?? '' + })) + return ( + `url=${state.url} title=${JSON.stringify(state.title)} ` + + `window.app=${state.hasApp} extensionManager=${state.hasExtensionManager} ` + + `blockUiMask=${state.blockUiVisible} signInView=${state.signInVisible} ` + + `body=${JSON.stringify(state.bodyText)}` + ) + } catch (probeError) { + return `page state unreadable (${probeError instanceof Error ? probeError.message : String(probeError)})` + } + } + /** @deprecated Use standalone `assetPath` from `browser_tests/fixtures/utils/assetPath` directly. */ public assetPath(fileName: string) { return assetPath(fileName) @@ -514,6 +576,8 @@ export const comfyPageFixture = base.extend<{ await page.coverage.startJSCoverage({ resetOnNavigation: false }) await use(page) + // A closed page has no coverage to collect + if (page.isClosed()) return const coverage = await page.coverage.stopJSCoverage() const mcr = MCR({ @@ -532,14 +596,29 @@ export const comfyPageFixture = base.extend<{ const { parallelIndex } = testInfo const username = `playwright-test-${parallelIndex}` - const userId = await comfyPage.setupUser(username) - comfyPage.userIds[parallelIndex] = userId + const isCustomNodes = testInfo.project.name === 'custom-nodes' + const needsPerf = + testInfo.tags.includes('@perf') || testInfo.tags.includes('@audit') + const startupErrorCollector = isCustomNodes + ? collectConsoleErrors(page) + : undefined + let perfStarted = false + const cleanups: (() => Promise)[] = [] + if (needsPerf) + cleanups.push(async () => { + if (perfStarted) await comfyPage.perf.dispose() + }) + if (startupErrorCollector) + cleanups.push(async () => startupErrorCollector.stop()) - const isVueNodes = testInfo.tags.includes('@vue-nodes') - comfyPage.isVueNodes = isVueNodes + const run = async () => { + const userId = await comfyPage.setupUser(username) + comfyPage.userIds[parallelIndex] = userId - try { - await comfyPage.setupSettings({ + const isVueNodes = testInfo.tags.includes('@vue-nodes') + comfyPage.isVueNodes = isVueNodes + + const startupSettings: Record = { 'Comfy.UseNewMenu': 'Top', // Hide canvas menu/info/selection toolbox by default. 'Comfy.Graph.CanvasInfo': false, @@ -565,52 +644,70 @@ export const comfyPageFixture = base.extend<{ 'Comfy.RightSidePanel.ShowErrorsTab': false, ...(isVueNodes && { 'Comfy.VueNodes.Enabled': true }), ...initialSettings - }) - } catch (e) { - console.error(e) - } + } + try { + await comfyPage.setupSettings(startupSettings) + } catch (e) { + console.error(e) + } + if (testInfo.tags.includes('@cloud')) { + const context = page.context() + await context.route('**/api/auth/session', (route) => + route.fulfill({ status: 204 }) + ) + await context.route('**/api/billing/status', (route) => + route.fulfill({ json: LEGACY_PERSONAL_BILLING_STATUS }) + ) + await context.route('**/api/billing/balance', (route) => + route.fulfill({ json: EMPTY_BILLING_BALANCE }) + ) + await context.route('**/api/billing/plans', (route) => + route.fulfill({ json: EMPTY_BILLING_PLANS }) + ) + await context.route('**/customers/cloud-subscription-status', (route) => + route.fulfill({ json: UNSUBSCRIBED }) + ) + await context.route('**/customers/balance', (route) => + route.fulfill({ json: ZERO_BALANCE }) + ) + await mockWorkspace(context, workspace('personal', 'owner'), []) + } + if (testInfo.tags.includes('@cloud') || testInfo.tags.includes('@auth')) { + await comfyPage.cloudAuth.mockAuth() + } - if (testInfo.tags.includes('@cloud')) { - const context = page.context() - await context.route('**/api/auth/session', (route) => - route.fulfill({ status: 204 }) - ) - await context.route('**/api/billing/status', (route) => - route.fulfill({ json: LEGACY_PERSONAL_BILLING_STATUS }) - ) - await context.route('**/api/billing/balance', (route) => - route.fulfill({ json: EMPTY_BILLING_BALANCE }) - ) - await context.route('**/api/billing/plans', (route) => - route.fulfill({ json: EMPTY_BILLING_PLANS }) - ) - await context.route('**/customers/balance', (route) => - route.fulfill({ json: ZERO_BALANCE }) - ) - await mockWorkspace(context, workspace('personal', 'owner'), []) - } + if (isCustomNodes) await installCustomNodeBlankStartup(page) + if (isCustomNodes) await trackVisibleErrors(page) - if (testInfo.tags.includes('@cloud') || testInfo.tags.includes('@auth')) { - await comfyPage.cloudAuth.mockAuth() - } + if (Object.keys(initialFeatureFlags).length > 0) { + await comfyPage.featureFlags.seedFlags(initialFeatureFlags) + } - if (Object.keys(initialFeatureFlags).length > 0) { - await comfyPage.featureFlags.seedFlags(initialFeatureFlags) - } + await comfyPage.setup() - await comfyPage.setup() + if (startupErrorCollector) { + startupErrorCollector.stop() + recordStartupConsoleErrors(page, startupErrorCollector.errors) + } - if (isVueNodes) { - await comfyPage.vueNodes.waitForNodes() - } + if (isCustomNodes) { + await comfyExpect + .poll(() => comfyPage.nodeOps.getGraphNodesCount()) + .toBe(0) + } - const needsPerf = - testInfo.tags.includes('@perf') || testInfo.tags.includes('@audit') - if (needsPerf) await comfyPage.perf.init() + if (isVueNodes) { + await comfyPage.vueNodes.waitForNodes() + } - await use(comfyPage) + if (needsPerf) { + await comfyPage.perf.init() + perfStarted = true + } - if (needsPerf) await comfyPage.perf.dispose() + await use(comfyPage) + } + await runWithCollectedCleanup(run, cleanups) }, comfyMouse: async ({ comfyPage }, use) => { const comfyMouse = new ComfyMouse(comfyPage) diff --git a/browser_tests/fixtures/VueNodeHelpers.ts b/browser_tests/fixtures/VueNodeHelpers.ts index 18e6d301c82..080b93d5f3f 100644 --- a/browser_tests/fixtures/VueNodeHelpers.ts +++ b/browser_tests/fixtures/VueNodeHelpers.ts @@ -58,6 +58,22 @@ export class VueNodeHelpers { ) } + getOutputSlotRow(nodeId: string, slotIndex: number): Locator { + return this.getNodeLocator(nodeId) + .locator('.lg-slot--output') + .filter({ + has: this.page.locator( + `[data-slot-key="${getSlotKey(toNodeId(nodeId), slotIndex, false)}"]` + ) + }) + } + + getOutputSlotConnectionDot(nodeId: string, slotIndex: number): Locator { + return this.getOutputSlotRow(nodeId, slotIndex).getByTestId( + TestIds.node.slotConnectionDot + ) + } + /** * Get locator for Vue nodes by the node's title (displayed name in the header). * Matches against the actual title element, not the full node body. diff --git a/browser_tests/fixtures/customNode/ComfyTarget.ts b/browser_tests/fixtures/customNode/ComfyTarget.ts new file mode 100644 index 00000000000..2f67195d6d6 --- /dev/null +++ b/browser_tests/fixtures/customNode/ComfyTarget.ts @@ -0,0 +1,364 @@ +import type { Page } from '@playwright/test' + +import type { PromptResponse } from '@/schemas/apiSchema' + +import type { ObjectInfo } from '@e2e/fixtures/customNode/objectInfoValidator' +import type { + ExecutionError, + PromptEvent, + RunResult +} from '@e2e/fixtures/customNode/runResult' +import { classifyRun } from '@e2e/fixtures/customNode/runResult' +import { onPromptIdResponse } from '@e2e/fixtures/utils/customNodeSuite' + +export interface RawEvent { + type: string + node?: string | null + // execution_cached carries every cache-served node id (apiSchema + // zExecutionCachedWsMessage); NodeId is number | string on the wire. + nodes?: (string | number)[] + prompt_id?: string + output?: unknown + exception_message?: string + exception_type?: string + node_id?: string + node_type?: string + traceback?: string[] +} + +const TERMINAL = [ + 'execution_success', + 'execution_error', + 'execution_interrupted' +] + +// The /prompt rejection body is the apiSchema PromptResponse shape +// ({ error: string | {message}, node_errors: { : { class_type, +// errors: [{ details, message }] } } }). Flatten it to a single line naming +// the node class and the failing input so a VALIDATION_FAIL result is +// actionable instead of an empty object. Exported for a pure unit test: the +// happy path never runs it, so without a test a regression here would rot the +// diagnostic back to `{}` silently. +export function summarizePromptError(body: unknown): string | undefined { + const payload = body as Partial | null + if (!payload || typeof payload !== 'object') return undefined + const parts: string[] = [] + const topError = payload.error + if (typeof topError === 'string') { + if (topError) parts.push(topError) + } else if (topError?.message) parts.push(topError.message) + for (const [nodeId, nodeError] of Object.entries(payload.node_errors ?? {})) { + const cls = nodeError.class_type || nodeId + for (const err of nodeError.errors ?? []) { + const detail = err.details || err.message + if (detail) parts.push(`${cls}: ${detail}`) + } + } + return parts.length > 0 ? parts.join('; ') : undefined +} + +// A non-2xx /prompt response, with attribution decided by its status: 400 is +// the backend REJECTING this graph (pack-attributable validation), 5xx is the +// backend FAILING (an environment fault that is never a per-node verdict). +// errorType is the body's typed provenance when the backend supplies one +// (Cloud infra faults carry e.g. DATABASE_ERROR; an OSS validation-time crash +// or a proxy 5xx arrives untyped, which is itself diagnostic). +interface PromptRejection { + status: number + summary?: string + errorType?: string +} + +function extractPromptErrorType(body: unknown): string | undefined { + const error = (body as Partial | null)?.error + if (typeof error !== 'object' || error === null) return undefined + const type = (error as { type?: unknown }).type + return typeof type === 'string' && type ? type : undefined +} + +const describeRejection = (rejection: PromptRejection): string => + rejection.summary ?? `HTTP ${rejection.status} prompt submission failed` + +const SERVER_SIDE_FAULT_PREFIX = 'prompt submission failed server-side' + +// Callers that accumulate per-node verdicts use this to catch the fault, +// record it, and still report the failures found before it - a late 5xx must +// never mask real regressions from earlier batches. +export function isServerSideFault(error: unknown): error is Error { + return ( + error instanceof Error && error.message.startsWith(SERVER_SIDE_FAULT_PREFIX) + ) +} + +const serverSideFault = (rejection: PromptRejection): Error => + new Error( + `${SERVER_SIDE_FAULT_PREFIX} (HTTP ${rejection.status} POST /prompt)` + + (rejection.summary ? ` - ${rejection.summary}` : '') + + (rejection.errorType ? ` [type: ${rejection.errorType}]` : '') + + ' - backend/environment fault, not a pack validation reject' + ) + +export function toPromptEvent(raw: RawEvent): PromptEvent { + if (raw.type === 'executing') + return { type: 'executing', node: raw.node ?? null } + if (raw.type === 'executed') + return { type: 'executed', node: raw.node ?? null, output: raw.output } + if (raw.type === 'execution_cached') + return { type: 'execution_cached', nodes: (raw.nodes ?? []).map(String) } + if (raw.type === 'execution_error' || raw.type === 'execution_interrupted') { + const error: ExecutionError = { + exceptionMessage: raw.exception_message?.trimEnd(), + exceptionType: raw.exception_type, + nodeId: raw.node_id, + nodeType: raw.node_type, + traceback: raw.traceback + } + return { type: raw.type, error } + } + return { type: raw.type as 'execution_start' | 'execution_success' } +} + +/** + * Drives a real ComfyUI backend through the running frontend. The verdict logic + * lives in the pure `classifyRun`; this class is only the in-page IO plumbing. + */ +export class LocalDesktopTarget { + async getObjectInfo(page: Page): Promise { + return await page.evaluate(async () => { + const defs = await window.app!.api.getNodeDefs() + const out: Record< + string, + { input?: { required?: Record } } + > = {} + for (const [name, def] of Object.entries(defs)) { + const required = ( + def as { input?: { required?: Record } } + ).input?.required + out[name] = { input: { required } } + } + return out + }) + } + + async runWorkflow( + page: Page, + opts: { + expectedNodeIds: string[] + graphNodeIds?: string[] + proofOutputNodeByExpectedNode?: Record + timeoutMs: number + } + ): Promise { + // A prior run's terminal event can arrive after its sink was read (late + // websocket delivery, or a timed-out prompt finishing during this run). + // Remember every prompt id already observed and ignore its events here, + // so one node's failure is never attributed to the next node tested. + const seenPromptIds = await page.evaluate( + (types) => { + const sink = window as unknown as { + __cnEvents: RawEvent[] + __cnSeenPromptIds?: string[] + __cnTapInstalled?: boolean + } + const seen = new Set(sink.__cnSeenPromptIds ?? []) + for (const event of sink.__cnEvents ?? []) + if (event.prompt_id) seen.add(event.prompt_id) + sink.__cnSeenPromptIds = [...seen] + sink.__cnEvents = [] + if (sink.__cnTapInstalled) return sink.__cnSeenPromptIds + sink.__cnTapInstalled = true + for (const type of types) + (window.app!.api as EventTarget).addEventListener( + type, + (event: Event) => { + const detail: unknown = (event as CustomEvent).detail + // `executing` dispatches a bare node-id string (api.ts + // dispatchCustomEvent('executing', msg.data.node)); the other + // events dispatch object payloads. + sink.__cnEvents.push( + detail !== null && typeof detail === 'object' + ? { type, ...(detail as Record) } + : { type, node: (detail as string | undefined) ?? null } + ) + } + ) + return sink.__cnSeenPromptIds + }, + [ + 'execution_start', + ...TERMINAL, + 'executing', + 'execution_cached', + 'executed' + ] + ) + + // Positively identify THIS attempt: the /prompt POST response body + // carries the prompt_id the backend assigned. When captured it becomes + // the primary event filter; the seen-set above and the graph-membership + // check below stay as defense in depth (capture can lose a race with a + // transient refusal, and `executing` events carry no prompt id at all). + let capturedPromptId: string | undefined + // A backend rejection answers /prompt with a non-2xx body carrying + // { error, node_errors }. app.queuePrompt swallows it and just returns + // false, so without capturing it here the verdict names nothing. The + // status is kept alongside the summarized body because it decides + // attribution (see PromptRejection). + let capturedRejection: PromptRejection | undefined + let capturedResponseSequence = 0 + const { detach: stopCapture, settled: captureSettled } = onPromptIdResponse( + page, + (promptId, body, status, sequence) => { + if (sequence < capturedResponseSequence) return + capturedResponseSequence = sequence + capturedPromptId = promptId + capturedRejection = + status >= 400 + ? { + status, + summary: summarizePromptError(body), + errorType: extractPromptErrorType(body) + } + : undefined + } + ) + + // app.queuePrompt (NOT api.queuePrompt: that submits an empty prompt). + // false = validation reject (emits no events), but pack JS hooking the + // queue can refuse transiently - retry once; real rejects fail twice. + // Pack JS can also THROW mid-graphToPrompt on a graph shape it does not + // expect; catch in-page so one bad node classifies as VALIDATION_FAIL + // (with the exception text) instead of aborting the whole tier. + const queueOnce = () => + page.evaluate(async () => { + try { + return await window.app!.queuePrompt(0) + } catch (error) { + // Never an empty string: an empty __cnThrew would nullish-coalesce + // wrong downstream and blank the VALIDATION_FAIL message. + return { __cnThrew: String(error) || 'pack threw an empty error' } + } + }) + const refused = ( + result: unknown + ): result is false | { __cnThrew: string } => + result === false || + (typeof result === 'object' && result !== null && '__cnThrew' in result) + let queued = await queueOnce() + if (refused(queued)) { + await new Promise((resolve) => setTimeout(resolve, 250)) + queued = await queueOnce() + if (refused(queued)) { + await captureSettled() + stopCapture() + // A captured 5xx outranks a client-side throw: the backend + // demonstrably failed this submission server-side. + if (capturedRejection !== undefined && capturedRejection.status >= 500) + throw serverSideFault(capturedRejection) + return { + outcome: 'VALIDATION_FAIL', + executedNodes: [], + outputsByNode: {}, + // A throw carries its own text; a bare `false` reject leaves only + // the backend's node_errors captured off the /prompt response. + clientError: + (typeof queued === 'object' ? queued.__cnThrew : undefined) ?? + (capturedRejection && describeRejection(capturedRejection)) + } + } + } + + // The submission resolved, so the /prompt response is in flight or done; + // give its body-parse a bounded beat before snapshotting the id. + await captureSettled() + const captureDeadline = Date.now() + 2_000 + while ( + capturedPromptId === undefined && + capturedRejection === undefined && + Date.now() < captureDeadline + ) { + await new Promise((resolve) => setTimeout(resolve, 50)) + await captureSettled() + } + if (capturedRejection !== undefined) { + stopCapture() + if (capturedRejection.status >= 500) + throw serverSideFault(capturedRejection) + return { + outcome: 'VALIDATION_FAIL', + executedNodes: [], + outputsByNode: {}, + clientError: describeRejection(capturedRejection) + } + } + // A silent permanent miss would degrade every run to the legacy filters + // with no signal - make the fallback observable in the runner output. + if (capturedPromptId === undefined) + console.warn( + '[customNodes] /prompt response id capture missed; falling back to seen-set filtering' + ) + + await page + .waitForFunction( + ([terminal, seen, graphIds, promptId]) => { + const events = + ( + window as unknown as { + __cnEvents?: { + type: string + prompt_id?: string + node_id?: string + }[] + } + ).__cnEvents ?? [] + return events.some( + (event) => + terminal.includes(event.type) && + (promptId !== null + ? event.prompt_id === promptId + : !(event.prompt_id && seen.includes(event.prompt_id)) && + (graphIds === null || + event.node_id === undefined || + graphIds.includes(event.node_id))) + ) + }, + [ + TERMINAL, + seenPromptIds ?? [], + opts.graphNodeIds ?? null, + capturedPromptId ?? null + ] as const, + { timeout: opts.timeoutMs } + ) + .catch((error: unknown) => { + // Only a Playwright wait timeout means "no terminal event"; surface any + // other fault instead of masquerading it as a run TIMEOUT. + if (error instanceof Error && error.name === 'TimeoutError') return + stopCapture() + throw error + }) + stopCapture() + + const raw = ( + await page.evaluate( + () => + (window as unknown as { __cnEvents?: RawEvent[] }).__cnEvents ?? [] + ) + ).filter((event) => + // Positive id match when captured (events without a prompt_id - bare + // `executing` strings - stay, and graph membership still vets them); + // otherwise the legacy seen-set exclusion. + capturedPromptId !== undefined + ? event.prompt_id === undefined || event.prompt_id === capturedPromptId + : !(event.prompt_id && (seenPromptIds ?? []).includes(event.prompt_id)) + ) + const timedOut = !raw.some((event) => TERMINAL.includes(event.type)) + return classifyRun({ + events: raw.map(toPromptEvent), + expectedNodeIds: opts.expectedNodeIds, + proofOutputNodeByExpectedNode: opts.proofOutputNodeByExpectedNode, + graphNodeIds: opts.graphNodeIds, + timedOut + }) + } +} diff --git a/browser_tests/fixtures/customNode/autoRun.ts b/browser_tests/fixtures/customNode/autoRun.ts new file mode 100644 index 00000000000..b47f25e13d0 --- /dev/null +++ b/browser_tests/fixtures/customNode/autoRun.ts @@ -0,0 +1,201 @@ +// Classifies which nodes can execute with no hand-authored fixture; the +// rest are recorded with the reason, never silently dropped. +import { chunk } from 'es-toolkit' + +import type { RawNodeDef } from '@e2e/fixtures/customNode/typePairing' + +type AutoRunClass = + // Widgets cover every required input and a terminus exists. + | 'AUTO_RUNNABLE' + // Every required socket is synthesizable from a model-free producer. + | 'CHAINABLE' + // A required socket type has no model-free producer (MODEL, CLIP, SEGS...). + | 'NEEDS_WIRES' + // A required combo has zero options (empty model/file scan). + | 'NEEDS_MODELS' + // No outputs and not an OUTPUT_NODE - nothing the executor could watch. + | 'NO_OBSERVABLE_OUTPUT' + +export interface RequiredSocket { + name: string + type: string +} + +export interface AutoRunVerdict { + key: string + verdict: AutoRunClass + // Wire output 0 to PreviewAny (false = the node is its own terminus). + needsPreviewSink?: boolean + // CHAINABLE: sockets to satisfy from SYNTH_PRODUCERS, in declaration order. + requiredSockets?: RequiredSocket[] + reason: string +} + +// Model-free producers for each synthesizable socket type. NUMBER is a WAS +// type with a WAS producer, so each entry is validated against the live defs +// before it counts as synthesizable. +export const SYNTH_PRODUCERS: Record< + string, + { nodeType: string; outputIndex: number } +> = { + IMAGE: { nodeType: 'EmptyImage', outputIndex: 0 }, + LATENT: { nodeType: 'EmptyLatentImage', outputIndex: 0 }, + MASK: { nodeType: 'SolidMask', outputIndex: 0 }, + INT: { nodeType: 'PrimitiveInt', outputIndex: 0 }, + FLOAT: { nodeType: 'PrimitiveFloat', outputIndex: 0 }, + STRING: { nodeType: 'PrimitiveString', outputIndex: 0 }, + BOOLEAN: { nodeType: 'PrimitiveBoolean', outputIndex: 0 }, + AUDIO: { nodeType: 'EmptyAudio', outputIndex: 0 }, + NUMBER: { nodeType: 'Constant Number', outputIndex: 0 }, + '*': { nodeType: 'PrimitiveInt', outputIndex: 0 } +} + +const WIDGET_TYPES = new Set(['INT', 'FLOAT', 'STRING', 'BOOLEAN']) + +function classifyInput(spec: unknown): 'widget' | 'socket' | 'empty-combo' { + const specArray: unknown[] = Array.isArray(spec) ? spec : [spec] + const rawType = specArray[0] + const options = specArray[1] as + | { forceInput?: boolean; options?: unknown; widgetType?: string } + | undefined + // forceInput beats every form, combos included: no widget materializes, + // so no default exists to run on - the input must be wired. + if (options?.forceInput) return 'socket' + if (Array.isArray(rawType)) + return rawType.length > 0 ? 'widget' : 'empty-combo' + if (typeof rawType !== 'string') return 'socket' + if (rawType === 'COMBO') { + // Transformed (V2-schema) defs carry combos as the literal 'COMBO' with + // the option list in the opts object. No static list (empty, or a + // `remote` lazy combo) means the default value cannot be verified + // runnable at plan time - same bucket as an empty model scan. + return Array.isArray(options?.options) && options.options.length > 0 + ? 'widget' + : 'empty-combo' + } + return options?.widgetType !== undefined || WIDGET_TYPES.has(rawType) + ? 'widget' + : 'socket' +} + +function socketType(spec: unknown): string { + const specArray: unknown[] = Array.isArray(spec) ? spec : [spec] + return String(specArray[0]) +} + +export function classifyAutoRunnable( + key: string, + def: RawNodeDef & { output_node?: boolean }, + synthTypes: ReadonlySet +): AutoRunVerdict { + const sockets: RequiredSocket[] = [] + for (const [name, spec] of Object.entries(def.input?.required ?? {})) { + const kind = classifyInput(spec) + if (kind === 'empty-combo') + return { + key, + verdict: 'NEEDS_MODELS', + reason: `required combo "${name}" has no options on this backend` + } + if (kind === 'socket') { + const type = socketType(spec) + // Union-typed sockets ("IMAGE,MASK") accept any member (same semantics + // isTypeCompatible applies in typePairing), so one synthesizable member + // makes the socket wireable. Push the MEMBER, not the union string - + // the chain builder synthesizes a producer for exactly this type. + const member = type + .split(',') + .map((candidate) => candidate.trim()) + .find((candidate) => synthTypes.has(candidate)) + if (member === undefined) + return { + key, + verdict: 'NEEDS_WIRES', + reason: `required input "${name}" (${type}) has no model-free producer` + } + sockets.push({ name, type: member }) + } + } + const terminus = + def.output_node === true + ? { needsPreviewSink: false, note: 'node is its own terminus' } + : (def.output ?? []).length > 0 + ? { needsPreviewSink: true, note: 'output 0 -> PreviewAny' } + : null + if (!terminus) + return { + key, + verdict: 'NO_OBSERVABLE_OUTPUT', + reason: 'no outputs and not an OUTPUT_NODE - nothing observable to queue' + } + if (sockets.length === 0) + return { + key, + verdict: 'AUTO_RUNNABLE', + needsPreviewSink: terminus.needsPreviewSink, + reason: `widgets satisfy all required inputs; ${terminus.note}` + } + return { + key, + verdict: 'CHAINABLE', + needsPreviewSink: terminus.needsPreviewSink, + requiredSockets: sockets, + reason: `${sockets.length} required socket(s) synthesized from model-free producers; ${terminus.note}` + } +} + +export function planAutoRuns( + defs: Record, + packNodeKeys: string[] +): AutoRunVerdict[] { + // A producer only counts if the backend actually registers it. + const synthTypes = new Set( + Object.entries(SYNTH_PRODUCERS) + .filter(([, producer]) => producer.nodeType in defs) + .map(([type]) => type) + ) + return packNodeKeys.map((key) => + classifyAutoRunnable(key, defs[key], synthTypes) + ) +} + +// Independent chains per prompt so one bad node fails a batch, not the tier. +export function batchAutoRunnable( + verdicts: AutoRunVerdict[], + batchSize: number +): AutoRunVerdict[][] { + const runnable = verdicts.filter( + (verdict) => + verdict.verdict === 'AUTO_RUNNABLE' || verdict.verdict === 'CHAINABLE' + ) + return chunk(runnable, batchSize) +} + +export const AUTO_RUN_WIDGET_INPUTS: Record< + string, + Record> +> = { + 'ComfyUI-VideoHelperSuite': { + VHS_LoadAudio: { audio_file: 'input/plain_audio.wav' }, + VHS_LoadAudioUpload: { audio: 'plain_audio.wav' }, + VHS_LoadImagePath: { image: 'input/test_upload_image.webp' }, + VHS_LoadImages: { directory: 'images' }, + VHS_LoadImagesPath: { directory: 'input/images' }, + VHS_LoadVideoFFmpegPath: { video: 'input/plain_video.mp4' }, + VHS_LoadVideoPath: { video: 'input/plain_video.mp4' } + } +} + +export const CLOUD_RUN_EXCLUSIONS: Record< + string, + Record +> = { + 'ComfyUI-VideoHelperSuite': { + VHS_SelectLatest: { + reason: + 'its Python execute method is deliberately unreachable and requires pack JS to replace the node during prompt conversion', + restore: + 'add a curated prompt-conversion fixture for VHS_SelectLatest and remove this entry when the transformed workflow executes' + } + } +} diff --git a/browser_tests/fixtures/customNode/connectivityExpectations.ts b/browser_tests/fixtures/customNode/connectivityExpectations.ts new file mode 100644 index 00000000000..6ec2034d358 --- /dev/null +++ b/browser_tests/fixtures/customNode/connectivityExpectations.ts @@ -0,0 +1,247 @@ +export interface ConnectivityExpectations { + isolatedNodeTypes: Record + connectRejected: PairExpectationGroup[] + deterministicSlotContractMismatch: PairExpectationGroup[] + dynamicSlotCleanupStalled: PairExpectationGroup[] + roundtripLost: PairExpectationGroup[] + zeroPairDragExpectedNodeCounts: Partial> +} + +export interface PairExpectationGroup { + id: string + pack: string + pairs: string[] + reason: string + restore: string +} + +export function pairExpectationKeys(groups: PairExpectationGroup[]): string[] { + return groups.flatMap((group) => group.pairs) +} + +export function pairExpectationNodeTypes( + groups: PairExpectationGroup[] +): string[] { + const nodeTypes = new Set() + for (const key of pairExpectationKeys(groups)) { + const sides = key.split(' -> ') + if (sides.length !== 2) throw new Error(`${key}: invalid pair key`) + for (const side of sides) { + const separator = side.lastIndexOf('.') + if (separator <= 0 || separator === side.length - 1) + throw new Error(`${key}: invalid pair endpoint`) + nodeTypes.add(side.slice(0, separator)) + } + } + return [...nodeTypes].sort() +} + +export const pairEndpointPacks: Readonly> = { + ACN_SparseCtrlLoaderAdvanced: 'comfyui-advanced-controlnet', + ADE_AnimateDiffCombine: 'comfyui-animatediff-evolved', + ADE_ValueScheduling: 'comfyui-animatediff-evolved', + AddLabel: 'ComfyUI-KJNodes', + AddNoise: 'core', + AddTextPrefix: 'core', + AdjustBrightness: 'core', + AnimaLLLiteApply: 'core', + 'Basic data handling: Boolean And': 'basic_data_handling', + 'Basic data handling: CastToInt': 'basic_data_handling', + CompositorTools3: 'comfyui-enricos-nodes', + FL_CodeNode: 'ComfyUI_Fill-Nodes', + FL_NodeLoader: 'ComfyUI_Fill-Nodes', + FL_NodePackLoader: 'ComfyUI_Fill-Nodes', + FL_TimeLine: 'ComfyUI_Fill-Nodes', + FL_VideoBatchSplitter: 'ComfyUI_Fill-Nodes', + 'MathExpression|pysssss': 'ComfyUI-Custom-Scripts', + MultiImageLoader: 'WhatDreamsCost-ComfyUI', + VHS_SelectLatest: 'comfyui-videohelpersuite', + Vewd: 'vewd' +} + +export function pairEndpointOwnershipIssues( + requiredNodeTypes: string[], + nodes: Array<{ type: string; pack: string }>, + installedPacks: ReadonlySet +): string[] { + const installed = new Set( + [...installedPacks].map((pack) => pack.toLowerCase()) + ) + return requiredNodeTypes.flatMap((nodeType) => { + const expectedPack = pairEndpointPacks[nodeType] + if (!expectedPack) + return [`${nodeType}: no endpoint pack attribution exists`] + if (expectedPack !== 'core' && !installed.has(expectedPack.toLowerCase())) + return [] + const node = nodes.find((candidate) => candidate.type === nodeType) + if (!node) return [`${nodeType}: not registered by ${expectedPack}`] + return node.pack.toLowerCase() === expectedPack.toLowerCase() + ? [] + : [`${nodeType}: expected ${expectedPack}, observed ${node.pack}`] + }) +} + +export const connectivityExpectations: ConnectivityExpectations = { + isolatedNodeTypes: { + FL_CodeNode: { + pack: 'ComfyUI_Fill-Nodes', + reason: + 'its deferred dynamic-slot cleanup is context-sensitive after repeated same-page graph resets' + }, + TimerNodeKJ: { + pack: 'ComfyUI-KJNodes', + reason: + 'an earlier KJNodes creation race can corrupt shared editor state before this node mounts' + } + }, + connectRejected: [ + { + id: 'math-expression-text-veto', + pack: 'ComfyUI-Custom-Scripts', + pairs: ['AddTextPrefix.texts -> MathExpression|pysssss.expression'], + reason: + 'MathExpression rejects this materialized STRING connection despite matching object_info types', + restore: + 'fix the MathExpression connection veto and remove this entry when the pair connects' + }, + { + id: 'fill-node-loader-trigger-veto', + pack: 'ComfyUI_Fill-Nodes', + pairs: [ + 'FL_NodeLoader.TRIGGER -> FL_NodeLoader.trigger', + 'FL_NodeLoader.TRIGGER -> FL_NodePackLoader.trigger', + 'FL_NodePackLoader.TRIGGER -> FL_NodeLoader.trigger' + ], + reason: + 'the Fill loader nodes reject their own declared TRIGGER connections', + restore: + 'fix the loader connection vetoes and remove this entry when all three pairs connect' + } + ], + deterministicSlotContractMismatch: [ + { + id: 'multi-image-loader-trimmed-outputs', + pack: 'WhatDreamsCost-ComfyUI', + pairs: Array.from( + { length: 50 }, + (_, index) => + `MultiImageLoader.image_${index + 1} -> ADE_AnimateDiffCombine.images` + ), + reason: + 'MultiImageLoader trims its 50 declared image outputs to the loaded count', + restore: + 'make object_info describe only materialized outputs and remove this entry' + }, + { + id: 'video-batch-splitter-trimmed-outputs', + pack: 'ComfyUI_Fill-Nodes', + pairs: Array.from( + { length: 16 }, + (_, index) => + `FL_VideoBatchSplitter.batch_${index + 5} -> AdjustBrightness.images` + ), + reason: + 'FL_VideoBatchSplitter exposes only the default 4 of 20 declared outputs', + restore: + 'make object_info describe only materialized outputs and remove this entry' + } + ], + dynamicSlotCleanupStalled: [ + { + id: 'fill-code-node-dynamic-cleanup-hang', + pack: 'ComfyUI_Fill-Nodes', + pairs: [ + 'Basic data handling: Boolean And.BOOLEAN -> FL_CodeNode.use_file', + 'Basic data handling: Boolean And.BOOLEAN -> FL_CodeNode.run_always' + ], + reason: + 'FL_CodeNode enters a synchronous loop when its 25ms dynamic-slot cleanup repeatedly removes the fixed use_file or run_always index', + restore: + 'advance or remove the current dynamic slot during cleanup and remove this entry when both pairs complete' + } + ], + roundtripLost: [ + { + id: 'timeline-node-loss', + pack: 'ComfyUI_Fill-Nodes', + pairs: ['FL_TimeLine.MODEL -> AddNoise.model'], + reason: + 'FL_TimeLine replaces node.serialize and loses graph identity on reload', + restore: + 'preserve LiteGraph serialization and remove this entry when the pair survives reload' + }, + { + id: 'vewd-dynamic-input-reload', + pack: 'vewd', + pairs: [ + 'AddTextPrefix.texts -> Vewd.folder', + 'AddTextPrefix.texts -> Vewd.filename_prefix', + 'AddTextPrefix.texts -> Vewd.selected_media', + 'ADE_ValueScheduling.INT -> Vewd.max_frames' + ], + reason: 'Vewd rebuilds its dynamic inputs during configure', + restore: + 'preserve these four links during configure and remove this entry' + }, + { + id: 'vhs-select-latest-dynamic-reload', + pack: 'comfyui-videohelpersuite', + pairs: [ + 'VHS_SelectLatest.Filename -> AddTextPrefix.texts', + 'AddTextPrefix.texts -> VHS_SelectLatest.filename_prefix', + 'AddTextPrefix.texts -> VHS_SelectLatest.filename_postfix', + 'VHS_SelectLatest.Filename -> AddLabel.font_color' + ], + reason: 'VHS_SelectLatest rebuilds its dynamic slots during configure', + restore: + 'preserve these four links during configure and remove this entry' + }, + { + id: 'fill-code-node-dynamic-reload', + pack: 'ComfyUI_Fill-Nodes', + pairs: [ + 'AddTextPrefix.texts -> FL_CodeNode.code_input', + 'AddTextPrefix.texts -> FL_CodeNode.file' + ], + reason: + 'FL_CodeNode treats code_input and file as removable during configure', + restore: 'preserve both links during configure and remove this entry' + }, + { + id: 'fill-timeline-dynamic-reload', + pack: 'ComfyUI_Fill-Nodes', + pairs: [ + 'AnimaLLLiteApply.MODEL -> FL_TimeLine.model', + 'AddTextPrefix.texts -> FL_TimeLine.timeline_data', + 'Basic data handling: CastToInt.INT -> FL_TimeLine.video_width', + 'Basic data handling: CastToInt.INT -> FL_TimeLine.video_height', + 'Basic data handling: CastToInt.INT -> FL_TimeLine.number_animation_frames', + 'Basic data handling: CastToInt.INT -> FL_TimeLine.frames_per_second' + ], + reason: + 'FL_TimeLine replaces node.serialize and drops its declared inputs during configure', + restore: 'preserve these six links during configure and remove this entry' + }, + { + id: 'advanced-controlnet-motion-link-reload', + pack: 'comfyui-advanced-controlnet', + pairs: [ + 'CompositorTools3.tools -> ACN_SparseCtrlLoaderAdvanced.use_motion' + ], + reason: + 'ACN_SparseCtrlLoaderAdvanced drops its optional motion link during configure', + restore: + 'preserve the use_motion link during configure and remove this entry' + } + ], + // Packs too small to wire to themselves: the drag sweep needs a producer and + // a consumer inside one pack, and these register one or two nodes. The count + // is the assertion - a pack that grows past it has pairs to find and the + // entry must go. + zeroPairDragExpectedNodeCounts: { + 'comfyui-impact-subpack': 1, + 'comfyui-qwenmultiangle': 1, + 'comfyui-string-converter': 2, + 'comfyui-workflow-prettier': 1 + } +} diff --git a/browser_tests/fixtures/customNode/consoleErrorLedger.ts b/browser_tests/fixtures/customNode/consoleErrorLedger.ts new file mode 100644 index 00000000000..d4607c257d7 --- /dev/null +++ b/browser_tests/fixtures/customNode/consoleErrorLedger.ts @@ -0,0 +1,412 @@ +interface AllowlistRule { + global?: boolean + id: string + pattern: RegExp + reason: string + requiredStartupId?: string + restore: string +} + +interface ConnectivityRule extends AllowlistRule { + requiredConnectivityId?: string +} + +// Pack-attributed console noise with no visible error surface. Shared by +// the all-nodes tiers and the curated run tier so one ledger covers every +// surface a pack's script can emit on. Filter-guarded: a pattern suppresses +// matching errors for its pack only. Most entries are environment-conditional +// and remain review-stale. +const CONSOLE_ERROR_ALLOWLIST: Record = { + 'comfyui-itools': [ + { + id: 'itools-no-saved-paint-file', + pattern: + /Error: File not found.*\/extensions\/comfyui-itools\/makadi\/SmartPaintArea\.js/, + global: true, + reason: + 'the paint widget logs the API no-saved-drawing response from SmartPaintArea.js', + restore: + 'handle the no-saved-drawing response without console.error and remove this entry' + }, + { + id: 'itools-missing-paint-file-request', + pattern: + /Failed to load resource.*404.*api\/itools\/request_the_paint_file/, + global: true, + reason: + 'the paint widget requests its backing file on creation and the route 404s until one is saved', + restore: + 'return an empty successful response before a paint file exists and remove this entry' + } + ], + 'ComfyUI-Impact-Pack': [ + { + id: 'impact-sam-editor-legacy-clipspace-import', + pattern: + /\[vite:preloadError\].*\/extensions\/ComfyUI-Impact-Pack\/impact-sam-editor\.js.*Failed to fetch dynamically imported module/, + global: true, + requiredStartupId: 'impact-sam-editor-legacy-clipspace-import', + reason: + 'impact-sam-editor.js imports the removed extensions/core/clipspace.js module', + restore: + 'replace the legacy ClipspaceDialog import and remove this entry when impact-sam-editor.js imports successfully' + }, + { + id: 'impact-root-relative-media-preview', + // Media/text widgets preview their value via root-relative URLs at + // creation; 404s on a backend whose root does not serve the file. + pattern: + /Failed to load resource.*404.*(000_custom_node_probe\.png|00[01]_custom_node_probe\.wav|beach\.jpg|example\.png|plain_video\.mp4|file\.txt)/, + reason: 'media widget previews its value via a root-relative URL', + restore: + 'resolve media previews through the backend view endpoint and remove this entry' + }, + { + id: 'impact-preview-bridge-empty-id', + // PreviewBridge widgets fetch their internal preview id on configure; + // a bare backend has no image behind it. + pattern: /Failed to load resource.*400.*api\/impact\/get\/pb_id_image/, + reason: 'PreviewBridge fetches its preview id on configure', + restore: + 'avoid fetching a PreviewBridge id before it has backing media and remove this entry' + }, + { + id: 'impact-set-value-probe-preview', + // The save/reload tier writes `_cn` probe values; media widgets + // preview them as URLs and 404. + pattern: /Failed to load resource.*404.*_cn/, + reason: 'set-and-stick probe value previewed by a media widget', + restore: + 'make the media widget tolerate an unavailable configured value and remove this entry' + } + ], + 'ComfyUI-KJNodes': [ + { + id: 'kj-empty-loader-preview', + // Image/video loader previews fetch their combo value at creation; + // on a backend with an empty input dir the value is undefined and the + // preview 404s (and retries with a fresh rand). Console-only noise, + // no visible error; upstream-report candidate. + pattern: + /Failed to load resource.*\/api\/view\?type=input&filename=undefined/, + reason: 'loader preview fetches undefined filename on empty input dir', + restore: + 'skip the preview request when no loader filename exists and remove this entry' + }, + { + id: 'kj-repeat-editor-menu', + // The SHARED editor base (editor_base.js:727 createContextMenu) + // replaceChild-es a menu element it assumes is attached; repeat + // instantiation within one page (the wiring sweep creates a node once + // per planned pair) finds it gone and throws at construction - for + // EVERY editor subclass (SplineEditor and PointsEditor observed), so + // the pattern matches the mechanism, not one class name. Single + // creation is clean - the mount tier passes. Latent pack bug, + // surfaced 2026-07-18 when new core partner nodes (HeyGen/Gemini) + // grew the corpus and reshuffled the sweep's pair plan. Console-only; + // upstream-report candidate. + pattern: + /Error creating \w+Editor: TypeError: Cannot read properties of null \(reading 'replaceChild'\)/, + reason: + 'KJNodes editor_base createContextMenu crashes on repeat instantiation (all editor subclasses)', + restore: + 'make repeated editor menu construction idempotent and remove this entry' + } + ], + 'ComfyUI-Custom-Scripts': [ + { + id: 'custom-scripts-null-combo-result', + // betterCombos.js:473 checks `typeof ret === "object" && "content" in + // ret`; typeof null is "object", so a null ret during save/reload + // throws `Cannot use 'in' operator to search for 'content' in null` + // as an uncaught page error - invisible until pageerror collection + // landed. Pack-owned and deterministic; upstream-report candidate. + pattern: /Cannot use 'in' operator to search for 'content' in null/, + reason: 'betterCombos.js missing null check throws during save/reload', + restore: + 'guard the null betterCombos result before reading content and remove this entry' + } + ], + 'ComfyUI-UltraShape1': [ + { + id: 'ultrashape-mesh-placeholder-preview', + pattern: + /Failed to load resource.*404.*api\/view\?type=input&filename=%28upload\+a\+mesh\+file%29&subfolder=&rand=/, + global: true, + reason: + 'mesh preview requests the literal upload-placeholder value on a backend with no mesh input', + restore: + 'skip mesh preview requests for the upload placeholder and remove this entry' + } + ], + 'WhatDreamsCost-ComfyUI': [ + { + id: 'ltx-director-guide-syntax', + pattern: + /\[vite:preloadError\].*Unexpected token '\}'\} \[http:\/\/localhost:8188\/assets\//, + global: true, + requiredStartupId: 'ltx-director-guide-syntax', + reason: + 'js/ltx_director_guide.js at the pinned commit has an extra closing brace and cannot parse', + restore: + 'fix js/ltx_director_guide.js and remove this entry when startup imports it without a preload error' + }, + { + id: 'whatdreams-set-value-probe-preview', + pattern: + /Failed to load resource.*404.*api\/view\?filename=_cn&type=input/, + reason: 'set-and-stick probe value previewed by the media loader', + restore: + 'make the audio loader tolerate an unavailable configured value and remove this entry' + } + ], + 'ComfyUI_Fill-Nodes': [ + { + id: 'fill-empty-metadata-node', + pattern: + /Failed to load resource.*404.*https:\/\/api\.comfy\.org\/comfy-nodes\/\/node/, + reason: 'pack metadata lookup requests an empty node name on creation', + restore: + 'skip metadata requests without a node name and remove this entry' + } + ], + ComfyUI_LayerStyle_Advance: [ + { + id: 'duplicate-color-overlay', + pattern: + /\[vite:preloadError\].*Extension named 'ColorOverlay' already registered\./, + global: true, + requiredStartupId: 'duplicate-color-overlay', + reason: + 'this pack and comfyui_layerstyle both register the ColorOverlay extension in their pinned dz_node_palette.js', + restore: + 'give the extensions distinct names and remove this entry when both packs load without a duplicate-registration error' + } + ], + 'comfyui-sam3': [ + { + id: 'sam3-editor-legacy-clipspace-import', + pattern: + /\[vite:preloadError\].*\/extensions\/comfyui-sam3\/sam3-editor\.js.*Failed to fetch dynamically imported module/, + global: true, + requiredStartupId: 'sam3-editor-legacy-clipspace-import', + reason: + 'sam3-editor.js imports the removed extensions/core/clipspace.js module', + restore: + 'replace the legacy ClipspaceDialog import and remove this entry when sam3-editor.js imports successfully' + } + ] +} + +// The breadth sweep intentionally creates and configures thousands of pack +// nodes in one page. These exact pack-owned mechanisms are allowed only in +// that tier. Deterministic mechanisms opt into required observation so a +// fixed or renamed mechanism immediately makes the ledger stale instead of +// staying hidden; environment-state mechanisms remain exact but conditional. +const CONNECTIVITY_ERROR_ALLOWLIST: Record = { + 'ComfyUI-KJNodes': [ + { + id: 'kj-points-empty-bbox-json', + pattern: + /Error parsing stored points: SyntaxError: Unexpected end of JSON input[\s\S]*\/extensions\/ComfyUI-KJNodes\/js\/editors\/point_editor_canvas\.js/, + reason: + 'PointsEditor parses an empty bbox widget when the sweep configures the node', + restore: + 'treat an empty bbox widget as an empty point set and remove this entry' + } + ], + 'ComfyUI-LTXVideo': [ + { + id: 'ltx-sparse-track-null-image-size', + pattern: + /Cannot read properties of null \(reading 'imgH'\)[\s\S]*\/extensions\/ComfyUI-LTXVideo\/js\/sparse_track_editor\.js/, + reason: + 'the sparse track editor computes its size before image state initializes', + requiredConnectivityId: 'ltx-sparse-track-null-image-size', + restore: + 'defer sparse-track sizing until image state exists and remove this entry' + } + ], + 'ComfyUI-Custom-Scripts': [ + { + id: 'custom-scripts-missing-lora-examples', + pattern: + /Failed to load resource.*status of 404\b.*http:\/\/localhost:8188\/api\/pysssss\/examples\/loras%2FNone(?:[\s\]]|$)/, + reason: + 'betterCombos conditionally requests examples for its literal None default and the pack route returns 404 when no matching lora exists', + restore: + 'return an empty example list for the None default and remove this entry' + } + ], + 'ComfyUI-VideoHelperSuite': [ + { + id: 'vhs-removed-link-target-id', + pattern: + /Cannot read properties of undefined \(reading 'target_id'\)[\s\S]*\/extensions\/comfyui-videohelpersuite\/js\/VHS\.core\.js/i, + reason: + 'VHS_SelectLatest can read a link before graph configure restores it; the exact ROUNDTRIP_LOST contract remains mandatory', + restore: + 'guard removed links during VHS file refresh and remove this entry' + } + ] +} + +function foldedRulesFor( + ledger: Record, + pack: string +): T[] { + const folded = pack.toLowerCase() + return Object.entries(ledger) + .filter(([ledgered]) => ledgered.toLowerCase() === folded) + .flatMap(([, rules]) => rules) +} + +// Some backends install packs under a lower-cased dirname +// (comfyui-impact-pack for ComfyUI-Impact-Pack), so an exact-key lookup +// ledgers nothing there and the pack's known noise reds the run. Fold the +// key so one entry covers both targets; fold-equal keys merge rather than +// shadow each other. +function allowlistRulesFor(pack: string): AllowlistRule[] { + return foldedRulesFor(CONSOLE_ERROR_ALLOWLIST, pack) +} + +function connectivityRulesForPacks(packs: string[]): ConnectivityRule[] { + return packs.flatMap((pack) => + foldedRulesFor(CONNECTIVITY_ERROR_ALLOWLIST, pack) + ) +} + +function withoutMatches(rules: AllowlistRule[], errors: string[]): string[] { + return errors.filter( + (error) => !rules.some((rule) => ruleMatches(rule, error)) + ) +} + +function ruleMatches(rule: AllowlistRule, error: string): boolean { + return rule.pattern.test(error) +} + +export function unallowlistedErrors(pack: string, errors: string[]): string[] { + return withoutMatches(allowlistRulesFor(pack), errors) +} + +export function unallowlistedGlobalExtensionErrorsForPacks( + packs: string[], + errors: string[] +): string[] { + const rules = packs.flatMap((pack) => + allowlistRulesFor(pack).filter((rule) => rule.global) + ) + return withoutMatches(rules, errors) +} + +export function customExtensionStartupErrors( + errors: readonly string[] +): string[] { + return errors.filter( + (error) => + /\/extensions\/(?!core\/)/i.test(error) || + /(?:error (?:loading|calling)|failed to load) extension\b/i.test(error) || + /\[vite:preloadError\]/.test(error) + ) +} + +export function staleRequiredStartupErrorRulesForPacks( + packs: string[], + errors: string[] +): string[] { + return packs.flatMap((pack) => + allowlistRulesFor(pack) + .filter( + (rule) => + rule.requiredStartupId !== undefined && + !errors.some((error) => ruleMatches(rule, error)) + ) + .map((rule) => `${pack}/${rule.requiredStartupId}`) + ) +} + +export function consoleErrorExclusionsForPacks(packs: string[]) { + return packs.flatMap((pack) => [ + ...allowlistRulesFor(pack).map((rule) => ({ + label: `${pack} console ${rule.id}`, + pack, + reason: rule.reason, + restore: rule.restore, + scope: 'startup and pack operations', + tier: 'S8' as const, + mode: rule.requiredStartupId + ? ('expected-failure' as const) + : ('conditional-console' as const) + })), + ...foldedRulesFor(CONNECTIVITY_ERROR_ALLOWLIST, pack).map((rule) => ({ + label: `${pack} connectivity console ${rule.id}`, + pack, + reason: rule.reason, + restore: rule.restore, + scope: 'S4 connectivity sweep', + tier: 'S8' as const, + mode: rule.requiredConnectivityId + ? ('expected-failure' as const) + : ('conditional-console' as const) + })) + ]) +} + +// For the cross-pack wiring sweep, where an error's owning pack cannot be +// read off the collector: an error is ledgered if any pack WHOSE NODES ARE +// IN THE SWEEP owns a matching pattern. Scoped to the installed packs, so a +// pack absent from the corpus can never vouch for an error. +export function unallowlistedErrorsForPacks( + packs: string[], + errors: string[] +): string[] { + return packs.reduce( + (remaining, pack) => unallowlistedErrors(pack, remaining), + errors + ) +} + +export function unallowlistedConnectivityErrorsForPacks( + packs: string[], + errors: string[] +): string[] { + return withoutMatches( + connectivityRulesForPacks(packs), + unallowlistedErrorsForPacks(packs, errors) + ) +} + +export function staleRequiredConnectivityErrorRulesForPacks( + packs: string[], + errors: string[] +): string[] { + return connectivityRulesForPacks(packs) + .filter( + (rule) => + rule.requiredConnectivityId !== undefined && + !errors.some((error) => ruleMatches(rule, error)) + ) + .map((rule) => rule.requiredConnectivityId!) +} + +// Execution errors surface on the tiers that actually queue prompts (the +// curated run and the auto-run tier). The mount, persistence, and wiring +// tiers queue nothing, so a prompt-execution error arriving in their console +// collector is an async stray from a prior tier's still-draining execution - +// the same "not this test" principle the event-attribution filter uses +// (ARCHITECTURE section 9). It is filtered from the non-executing tiers only; +// the executing tiers still assert on it. This is not error suppression: the +// visible error SURFACES (overlay/dialog/toast) are still asserted separately +// by expectNoVisibleErrors. +const FOREIGN_EXECUTION_NOISE: RegExp[] = [ + /PromptExecutionError/, + /Prompt execution failed/, + // The browser logs a rejected prompt submission as a failed resource load + // on /api/prompt. Only the executing tiers POST there, so this line in a + // mount/persistence/wiring collector is a prior tier's async submission. + /Failed to load resource.*\/api\/prompt/ +] +export function isForeignExecutionNoise(error: string): boolean { + return FOREIGN_EXECUTION_NOISE.some((pattern) => pattern.test(error)) +} diff --git a/browser_tests/fixtures/customNode/failureReport.ts b/browser_tests/fixtures/customNode/failureReport.ts new file mode 100644 index 00000000000..da271b78dc9 --- /dev/null +++ b/browser_tests/fixtures/customNode/failureReport.ts @@ -0,0 +1,12 @@ +export function failureSummary( + label: string, + failures: string[], + attachment: string +): string { + if (failures.length === 0) return label + const preview = failures + .slice(0, 10) + .map((failure, index) => ` ${index + 1}. ${failure.split('\n')[0]}`) + .join('\n') + return `${label}: ${failures.length} failure(s); first ${Math.min(10, failures.length)} below, full list in ${attachment}:\n${preview}` +} diff --git a/browser_tests/fixtures/customNode/interactionProfiles.ts b/browser_tests/fixtures/customNode/interactionProfiles.ts new file mode 100644 index 00000000000..2231b11c6fd --- /dev/null +++ b/browser_tests/fixtures/customNode/interactionProfiles.ts @@ -0,0 +1,255 @@ +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + writeFileSync +} from 'node:fs' +import { fileURLToPath } from 'node:url' + +// S13 differential interaction profiles: the def-driven tiers are +// structurally blind to what pack JS does IN RESPONSE to an interaction, +// and the curated tiers (S12 autogrow, S15 outputs) cover hand-picked +// nodes only. This tier locks the OBSERVED interaction behavior of every +// registered node - instantiate, connect-first, connect-last, disconnect - +// as shape DELTAS, without needing to understand any pack: whatever a +// node's JS does today is the baseline, and a frontend change that alters +// it reds against the committed delta. Deltas (not absolute shapes) keep +// baselines invariant to def changes a pin bump legitimately makes. + +const PROFILE_DIR = fileURLToPath( + new URL('./interactionProfiles/', import.meta.url) +) + +// One facet entry per slot/widget: `kind:name:type`, model order ignored +// (entries are sorted) so reordering alone is not drift. +export interface LogicalShape { + inputs: string[] + outputs: string[] + widgets: string[] +} + +// The symmetric difference of two shapes: entries the interaction added +// (`+kind:name:type`) or removed (`-kind:name:type`), sorted. Empty means +// the interaction changed nothing - itself a locked observation. +export type ShapeDelta = string[] + +// A probe that could not run carries its mechanism instead of a delta: +// NO_PRODUCER = no synthesizable model-free source matches the input type; +// NO_INPUTS = the node declares no connectable inputs (instantiate-only). +type ProbeResult = ShapeDelta | 'NO_PRODUCER' | 'NO_INPUTS' + +export interface NodeInteractionProfile { + connectFirst: ProbeResult + // SAME_AS_FIRST when the node has exactly one connectable input. + connectLast: ProbeResult | 'SAME_AS_FIRST' + // Delta from disconnecting what connectLast (or connectFirst) attached, + // measured against the connected shape - autogrow shrink lives here. + disconnect: ProbeResult +} + +export interface PackInteractionProfileFile { + recordedAt: { core: string; pin: string } + schema: 1 + nodes: Record +} + +// Nodes whose interaction deltas are not reproducible run-to-run, keyed by +// the MECHANISM (the console ledger discipline): registration-guarded in +// the spec, announced in run output, omitted from baselines. +// Empty until a record/compare cycle observes real instability - entries +// are earned by evidence, never pre-emptively. +export const INTERACTION_UNSTABLE_NODES: Record< + string, + Record +> = {} + +export function diffShapes( + before: LogicalShape, + after: LogicalShape +): ShapeDelta { + const delta: string[] = [] + for (const facet of ['inputs', 'outputs', 'widgets'] as const) { + const beforeSet = new Set(before[facet]) + const afterSet = new Set(after[facet]) + for (const entry of afterSet) + if (!beforeSet.has(entry)) delta.push(`+${entry}`) + for (const entry of beforeSet) + if (!afterSet.has(entry)) delta.push(`-${entry}`) + } + return delta.sort() +} + +function profilePath(pack: string): string { + return `${PROFILE_DIR}${pack}.json` +} + +/** + * A recorded baseline exists for this pack AND was recorded at this ref. + * + * S13 diffs a pack's live interaction shape against a committed recording, so + * it can only cover packs that have one. Recording is a real artifact - the six + * committed here run 1.7KB to 27KB each - and the cloud pack set deliberately + * has none, so those packs sit outside S13 rather than being skipped inside it. + * + * The ref check is what stops a core recording being reused for the same pack + * at a different cloud pin, where the baseline describes different code and + * every legitimate version difference would read as drift. + * + * Directory listing rather than existsSync: the filenames are mixed-case and + * the cloud rows are lowercase registry dirnames, so a case-insensitive + * filesystem (macOS) matched five packs that Linux CI would match two of. + */ +export function hasCommittedProfile(pack: string, ref: string): boolean { + const match = readdirSync(PROFILE_DIR).find( + (name) => name.toLowerCase() === `${pack.toLowerCase()}.json` + ) + if (!match) return false + const parsed: unknown = JSON.parse( + readFileSync(`${PROFILE_DIR}${match}`, 'utf-8') + ) + assertProfileFile(parsed, `${PROFILE_DIR}${match}`) + return parsed.recordedAt.pin === ref +} + +const PROBES = ['connectFirst', 'connectLast', 'disconnect'] as const + +function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.length > 0 +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function isProbeResult(value: unknown): value is ProbeResult { + if (value === 'NO_PRODUCER' || value === 'NO_INPUTS') return true + return ( + Array.isArray(value) && value.every((entry) => typeof entry === 'string') + ) +} + +// The offending field path, or null when the file is a valid profile set. +// A malformed baseline must name itself here: left unchecked it surfaces as +// an unrelated TypeError deep inside comparePackProfiles. +function invalidProfileField(parsed: unknown): string | null { + if (!isPlainObject(parsed)) return 'root (expected a JSON object)' + if (parsed.schema !== 1) return 'schema (expected 1)' + if (!isPlainObject(parsed.recordedAt)) + return 'recordedAt (expected { core, pin })' + if (!isNonEmptyString(parsed.recordedAt.core)) + return 'recordedAt.core (expected a non-empty string)' + if (!isNonEmptyString(parsed.recordedAt.pin)) + return 'recordedAt.pin (expected a non-empty string)' + if (!isPlainObject(parsed.nodes)) + return 'nodes (expected an object keyed by node type)' + for (const [node, profile] of Object.entries(parsed.nodes)) { + if (!isPlainObject(profile)) + return `nodes.${node} (expected a profile object)` + for (const probe of PROBES) { + const value = profile[probe] + if (isProbeResult(value)) continue + if (probe === 'connectLast' && value === 'SAME_AS_FIRST') continue + const markers = + probe === 'connectLast' + ? "'NO_PRODUCER', 'NO_INPUTS', or 'SAME_AS_FIRST'" + : "'NO_PRODUCER' or 'NO_INPUTS'" + return `nodes.${node}.${probe} (expected a string[] delta, ${markers})` + } + } + return null +} + +function assertProfileFile( + parsed: unknown, + path: string +): asserts parsed is PackInteractionProfileFile { + const field = invalidProfileField(parsed) + if (field !== null) + throw new Error( + `${path} is not a valid interaction profile file - ${field} - re-record it (docs/custom-node-regression-suite.md Step 5d)` + ) +} + +// null = no baseline recorded yet; compare mode must red on that (an +// uncovered pack is the failure mode this suite bans), record mode expects it. +export function loadPackProfiles( + pack: string +): PackInteractionProfileFile | null { + const path = profilePath(pack) + if (!existsSync(path)) return null + const parsed: unknown = JSON.parse(readFileSync(path, 'utf-8')) + assertProfileFile(parsed, path) + return parsed +} + +export function recordPackProfiles( + pack: string, + nodes: Record, + recordedAt: { core: string; pin: string } +): void { + mkdirSync(PROFILE_DIR, { recursive: true }) + const file: PackInteractionProfileFile = { recordedAt, schema: 1, nodes } + writeFileSync(profilePath(pack), JSON.stringify(file, null, 2) + '\n') +} + +function probesEqual( + a: NodeInteractionProfile, + b: NodeInteractionProfile +): string[] { + const problems: string[] = [] + for (const probe of PROBES) { + const expected = a[probe] + const actual = b[probe] + const same = + typeof expected === 'string' || typeof actual === 'string' + ? expected === actual + : expected.length === actual.length && + expected.every((entry, i) => entry === actual[i]) + if (!same) + problems.push( + `${probe}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}` + ) + } + return problems +} + +// Fail-closed compare, one problem string per drifted node: missing +// baseline file, node missing from baseline (new node = re-record), node +// gone from the live corpus (stale baseline), or any probe delta drifting. +export function comparePackProfiles(input: { + pack: string + observed: Record + committed: PackInteractionProfileFile | null +}): string[] { + const { pack, observed, committed } = input + if (committed === null) + return [ + `S13: no committed interaction profiles for '${pack}' - record them ` + + `(CN_INTERACTION=record run, commit the fixture; docs/custom-node-regression-suite.md Step 5d)` + ] + const provenance = `(baseline recorded at core ${committed.recordedAt.core}, pin ${committed.recordedAt.pin})` + const unstable = INTERACTION_UNSTABLE_NODES[pack] ?? {} + const problems: string[] = [] + for (const [node, expected] of Object.entries(committed.nodes)) { + if (node in unstable) continue + const actual = observed[node] + if (actual === undefined) { + problems.push( + `${pack}/${node}: baseline entry but the node was not probed - stale baseline, re-record ${provenance}` + ) + continue + } + for (const problem of probesEqual(expected, actual)) + problems.push( + `${pack}/${node}: interaction delta drifted - ${problem}. A frontend ` + + `change altered what this node's JS does on interaction ${provenance}` + ) + } + for (const node of Object.keys(observed)) + if (!(node in committed.nodes) && !(node in unstable)) + problems.push( + `${pack}/${node}: probed but no baseline entry - new node, re-record ${provenance}` + ) + return problems +} diff --git a/browser_tests/fixtures/customNode/interactionProfiles/ComfyUI-Custom-Scripts.json b/browser_tests/fixtures/customNode/interactionProfiles/ComfyUI-Custom-Scripts.json new file mode 100644 index 00000000000..bc0e519924d --- /dev/null +++ b/browser_tests/fixtures/customNode/interactionProfiles/ComfyUI-Custom-Scripts.json @@ -0,0 +1,74 @@ +{ + "recordedAt": { + "core": "b08e6cf35fac50d3ca8470dffb3f9a1fbb7187d2", + "pin": "609f3afaa74b2f88ef9ce8d939626065e3247469" + }, + "schema": 1, + "nodes": { + "CheckpointLoader|pysssss": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ConstrainImage|pysssss": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ConstrainImageforVideo|pysssss": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LoadText|pysssss": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LoraLoader|pysssss": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MathExpression|pysssss": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PlaySound|pysssss": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Repeater|pysssss": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ReroutePrimitive|pysssss": { + "connectFirst": ["+output:*:INT", "-output:*:*"], + "connectLast": "SAME_AS_FIRST", + "disconnect": ["+output:*:*", "-output:*:INT"] + }, + "SaveText|pysssss": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ShowText|pysssss": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "StringFunction|pysssss": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SystemNotification|pysssss": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + } + } +} diff --git a/browser_tests/fixtures/customNode/interactionProfiles/ComfyUI-Impact-Pack.json b/browser_tests/fixtures/customNode/interactionProfiles/ComfyUI-Impact-Pack.json new file mode 100644 index 00000000000..6203c1d2f48 --- /dev/null +++ b/browser_tests/fixtures/customNode/interactionProfiles/ComfyUI-Impact-Pack.json @@ -0,0 +1,1071 @@ +{ + "recordedAt": { + "core": "b08e6cf35fac50d3ca8470dffb3f9a1fbb7187d2", + "pin": "429d0159ad429e64d2b3916e6e7be9c22d025c3c" + }, + "schema": 1, + "nodes": { + "AddMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "AnyPipeToBasic": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "BasicPipeToDetailerPipe": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "BasicPipeToDetailerPipeSDXL": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "BboxDetectorCombined_v2": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "BboxDetectorSEGS": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "BitwiseAndMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "BitwiseAndMaskForEach": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "BlackPatchRetryHookProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CfgScheduleHookProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CLIPSegDetectorProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CombineRegionalPrompts": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "CoreMLDetailerHookProvider": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "CustomSamplerDetailerHookProvider": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "DenoiseScheduleHookProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "DenoiseSchedulerDetailerHookProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "DetailerForEach": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "DetailerForEachAutoRetry": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "DetailerForEachDebug": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "DetailerForEachDebugPipe": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "DetailerForEachPipe": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "DetailerForEachPipeForAnimateDiff": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "DetailerHookCombine": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "DetailerPipeToBasicPipe": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "EditBasicPipe": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "EditDetailerPipe": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "EditDetailerPipeSDXL": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "EmptySegs": { + "connectFirst": "NO_INPUTS", + "connectLast": "NO_INPUTS", + "disconnect": "NO_INPUTS" + }, + "FaceDetailer": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "FaceDetailerPipe": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "FromBasicPipe": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "FromBasicPipe_v2": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "FromDetailerPipe": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "FromDetailerPipe_v2": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "FromDetailerPipeSDXL": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "GITSSchedulerFuncProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageListToImageBatch": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImageMaskSwitch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageReceiver": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageSender": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactAssembleSEGS": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactBoolean": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactCombineConditionings": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactCompare": { + "connectFirst": [], + "connectLast": [ + "+input:a:INT", + "+input:b:INT", + "-input:a:*", + "-input:b:*" + ], + "disconnect": [] + }, + "ImpactConcatConditionings": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactConditionalBranch": { + "connectFirst": [], + "connectLast": [ + "+input:ff_value:INT", + "+input:tt_value:INT", + "+output:*:INT", + "-input:ff_value:*", + "-input:tt_value:*", + "-output:*:*" + ], + "disconnect": [] + }, + "ImpactConditionalBranchSelMode": { + "connectFirst": [], + "connectLast": [ + "+input:ff_value:INT", + "+input:tt_value:INT", + "+output:*:INT", + "-input:ff_value:*", + "-input:tt_value:*", + "-output:*:*" + ], + "disconnect": [] + }, + "ImpactConditionalStopIteration": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactControlBridge": { + "connectFirst": [ + "+input:value:INT", + "+output:value:INT", + "-input:value:*", + "-output:value:*" + ], + "connectLast": [ + "+input:value:INT", + "+output:value:INT", + "-input:value:*", + "-output:value:*" + ], + "disconnect": [] + }, + "ImpactControlNetApplyAdvancedSEGS": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactControlNetApplySEGS": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactControlNetClearSEGS": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactConvertDataType": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactCount_Elts_in_SEGS": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactDecomposeSEGS": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactDilate_Mask_SEG_ELT": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactDilateMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactDilateMaskInSEGS": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactDummyInput": { + "connectFirst": "NO_INPUTS", + "connectLast": "NO_INPUTS", + "disconnect": "NO_INPUTS" + }, + "ImpactEdit_SEG_ELT": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactExecutionOrderController": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactFlattenMask": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactFloat": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactFrom_SEG_ELT": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactFrom_SEG_ELT_bbox": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactFrom_SEG_ELT_crop_region": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactGaussianBlurMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactGaussianBlurMaskInSEGS": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactHFTransformersClassifierProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactIfNone": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactImageBatchToImageList": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactImageInfo": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactInt": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactInversedSwitch": { + "connectFirst": [ + "+input:input:INT", + "+input:sel_mode:INT", + "+output:output1:INT", + "-input:input:*", + "-input:sel_mode:BOOLEAN", + "-output:output1:*" + ], + "connectLast": [ + "+input:input:INT", + "+input:sel_mode:INT", + "+output:output1:INT", + "-input:input:*", + "-input:sel_mode:BOOLEAN", + "-output:output1:*" + ], + "disconnect": [] + }, + "ImpactIPAdapterApplySEGS": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactIsNotEmptySEGS": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactKSamplerAdvancedBasicPipe": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactKSamplerBasicPipe": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactLatentInfo": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactListBridge": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactLogger": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactLogicalOperators": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactMakeAnyList": { + "connectFirst": [ + "+input:value1:INT", + "+input:value2:INT", + "+output:INT:INT", + "-input:value1:*", + "-output:*:*" + ], + "connectLast": "SAME_AS_FIRST", + "disconnect": ["-input:value2:INT"] + }, + "ImpactMakeImageBatch": { + "connectFirst": ["+input:image2:IMAGE"], + "connectLast": "SAME_AS_FIRST", + "disconnect": ["-input:image2:IMAGE"] + }, + "ImpactMakeImageList": { + "connectFirst": ["+input:image2:IMAGE"], + "connectLast": "SAME_AS_FIRST", + "disconnect": ["-input:image2:IMAGE"] + }, + "ImpactMakeMaskBatch": { + "connectFirst": ["+input:mask2:MASK"], + "connectLast": "SAME_AS_FIRST", + "disconnect": ["-input:mask2:MASK"] + }, + "ImpactMakeMaskList": { + "connectFirst": ["+input:mask2:MASK"], + "connectLast": "SAME_AS_FIRST", + "disconnect": ["-input:mask2:MASK"] + }, + "ImpactMakeTileSEGS": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactMinMax": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactNeg": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactNegativeConditioningPlaceholder": { + "connectFirst": "NO_INPUTS", + "connectLast": "NO_INPUTS", + "disconnect": "NO_INPUTS" + }, + "ImpactNodeSetMuteState": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactQueueTrigger": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactQueueTriggerCountdown": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactRemoteBoolean": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactRemoteInt": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactSAM2VideoDetectorSEGS": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactScaleBy_BBOX_SEG_ELT": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactSchedulerAdapter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactSegsAndMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactSegsAndMaskForEach": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactSEGSClassify": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactSEGSConcat": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactSEGSIntersectionFilter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactSEGSLabelAssign": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactSEGSLabelFilter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactSEGSMerge": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactSEGSNMSFilter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactSEGSOrderedFilter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactSEGSPicker": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactSEGSRangeFilter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactSEGSToMaskBatch": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactSEGSToMaskList": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImpactSelectNthItemOfAnyList": { + "connectFirst": [ + "+input:any_list:INT", + "+output:*:INT", + "-input:any_list:*", + "-output:*:*" + ], + "connectLast": [ + "+input:any_list:INT", + "+output:*:INT", + "-input:any_list:*", + "-output:*:*" + ], + "disconnect": [] + }, + "ImpactSetWidgetValue": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactSimpleDetectorSEGS": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactSimpleDetectorSEGS_for_AD": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactSimpleDetectorSEGSPipe": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactSleep": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactStringSelector": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactSwitch": { + "connectFirst": [], + "connectLast": [ + "+input:input1:INT", + "+input:input2:INT", + "+output:INT:INT", + "-input:input1:*", + "-output:selected_value:*" + ], + "disconnect": ["-input:input2:INT"] + }, + "ImpactValueReceiver": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactValueSender": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactWildcardEncode": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImpactWildcardProcessor": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "IterativeImageUpscale": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "IterativeLatentUpscale": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "KSamplerAdvancedProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "KSamplerProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LamaRemoverDetailerHookProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LatentPixelScale": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LatentReceiver": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LatentSender": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LatentSwitch": { + "connectFirst": [], + "connectLast": [ + "+input:input1:INT", + "+input:input2:INT", + "+output:INT:INT", + "-input:input1:*", + "-output:selected_value:*" + ], + "disconnect": ["-input:input2:INT"] + }, + "MaskDetailerPipe": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MaskListToMaskBatch": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "MaskRectArea": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MaskRectAreaAdvanced": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MasksToMaskList": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "MaskToSEGS": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MaskToSEGS_for_AnimateDiff": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MediaPipeFaceMeshToSEGS": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "NoiseInjectionDetailerHookProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "NoiseInjectionHookProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ONNXDetectorProvider": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ONNXDetectorSEGS": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PixelKSampleHookCombine": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PixelKSampleUpscalerProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PixelKSampleUpscalerProviderPipe": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PixelTiledKSampleUpscalerProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PixelTiledKSampleUpscalerProviderPipe": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PreviewBridge": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PreviewBridgeLatent": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PreviewDetailerHookProvider": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ReencodeLatent": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ReencodeLatentPipe": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "RegionalPrompt": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "RegionalSampler": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "RegionalSamplerAdvanced": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "RemoveImageFromSEGS": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "RemoveNoiseMask": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "SAMDetectorCombined": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SAMDetectorSegmented": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SAMLoader": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SegmDetectorCombined_v2": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SegmDetectorSEGS": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SEGSDetailer": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SEGSDetailerForAnimateDiff": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SEGSLabelFilterDetailerHookProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SEGSOrderedFilterDetailerHookProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SEGSPaste": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SEGSPreview": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SEGSPreviewCNet": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "SEGSRangeFilterDetailerHookProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SEGSSwitch": { + "connectFirst": [], + "connectLast": [ + "+input:input1:INT", + "+input:input2:INT", + "+output:INT:INT", + "-input:input1:*", + "-output:selected_value:*" + ], + "disconnect": ["-input:input2:INT"] + }, + "SegsToCombinedMask": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "SEGSToImageList": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SEGSUpscaler": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SEGSUpscalerPipe": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SetDefaultImageForSEGS": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "StepsScheduleHookProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "StringListToString": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SubtractMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SubtractMaskForEach": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "TiledKSamplerProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ToBasicPipe": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ToBinaryMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ToDetailerPipe": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ToDetailerPipeSDXL": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "TwoAdvancedSamplersForMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "TwoSamplersForMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "TwoSamplersForMaskUpscalerProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "TwoSamplersForMaskUpscalerProviderPipe": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "UnsamplerDetailerHookProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "UnsamplerHookProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VariationNoiseDetailerHookProvider": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "WildcardPromptFromString": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + } + } +} diff --git a/browser_tests/fixtures/customNode/interactionProfiles/ComfyUI-KJNodes.json b/browser_tests/fixtures/customNode/interactionProfiles/ComfyUI-KJNodes.json new file mode 100644 index 00000000000..95c1e24b5b1 --- /dev/null +++ b/browser_tests/fixtures/customNode/interactionProfiles/ComfyUI-KJNodes.json @@ -0,0 +1,1229 @@ +{ + "recordedAt": { + "core": "b08e6cf35fac50d3ca8470dffb3f9a1fbb7187d2", + "pin": "e27a505b3ba6ce42687fe00500deda103d9d6071" + }, + "schema": 1, + "nodes": { + "AddLabel": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "AddNoiseToTrackPath": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "AppendInstanceDiffusionTracking": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "AppendStringsToList": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ApplyRifleXRoPE_HunuyanVideo": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ApplyRifleXRoPE_WanVideo": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "AudioConcatenate": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "BatchCLIPSeg": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "BatchCropFromMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "BatchCropFromMaskAdvanced": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "BatchUncrop": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "BatchUncropAdvanced": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "BboxToInt": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "BboxVisualize": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "BlockifyMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "BOOLConstant": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "CameraPoseVisualizer": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CFGZeroStarAndInit": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CheckpointLoaderKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CheckpointPerturbWeights": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ColorMatch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ColorMatchV2": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ColorToMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ConditioningMultiCombine": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ConditioningSetMaskAndCombine": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ConditioningSetMaskAndCombine3": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ConditioningSetMaskAndCombine4": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ConditioningSetMaskAndCombine5": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CondPassThrough": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ConsolidateMasksKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ContextWindowsVisualizerKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CreateAudioMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CreateFadeMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CreateFadeMaskAdvanced": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CreateFluidMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CreateGradientFromCoords": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CreateGradientMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CreateInstanceDiffusionTracking": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CreateMagicMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CreateShapeImageOnPath": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CreateShapeMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CreateShapeMaskOnPath": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CreateTextMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CreateTextOnPath": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CreateVoronoiMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CrossFadeImages": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CrossFadeImagesMulti": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CustomControlNetWeightsFluxFromList": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CustomSigmas": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CutAndDragOnPath": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "DecodeAndSaveVideo": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "DifferentialDiffusionAdvanced": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "DiffusionModelLoaderKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "DiffusionModelSelector": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "DiTBlockLoraLoader": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "DownloadAndLoadCLIPSeg": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "DrawInstanceDiffusionTracking": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "DrawMaskOnImage": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "DummyOut": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "EmptyLatentImageCustomPresets": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "EmptyLatentImagePresets": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "EncodeVideoComponents": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "EndRecordCUDAMemoryHistory": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "FastPreview": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "FastPreviewBatch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "FilterZeroMasksAndCorrespondingImages": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "FlipSigmasAdjusted": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "FloatConstant": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "FloatToMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "FloatToSigmas": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "FluxBlockLoraSelect": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "GenerateNoise": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "GetImageRangeFromBatch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "GetImagesFromBatchIndexed": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "GetImageSizeAndCount": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "GetLatentRangeFromBatch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "GetLatentsFromBatchIndexed": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "GetLatentSizeAndCount": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "GetMaskSizeAndCount": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "GetPreviewOverrideFramesKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "GetTrackRange": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "GGUFLoaderKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "GLIGENTextBoxApplyBatchCoords": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "GradientToFloat": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "GrowMaskWithBlur": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "HDRPreviewKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "HunyuanVideoBlockLoraSelect": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "HunyuanVideoEncodeKeyframesToCond": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Ideogram4OptimizationsKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Ideogram4PromptBuilderKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageAddMulti": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageAndMaskPreview": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageBatchExtendWithOverlap": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageBatchFilter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageBatchJoinWithTransition": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageBatchMulti": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageBatchRepeatInterleaving": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageBatchTestPattern": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageConcanate": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageConcatFromBatch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageConcatMulti": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageCropByMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageCropByMaskAndResize": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageCropByMaskBatch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageGrabPIL": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageGridComposite2x2": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageGridComposite3x3": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageGridtoBatch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageNoiseAugmentation": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageNormalize_Neg1_To_1": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImagePadForOutpaintMasked": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImagePadForOutpaintTargetSize": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImagePadKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImagePass": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImagePrepForICLora": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageResizeKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageResizeKJv2": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageSharpenKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageTensorList": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageTransformByNormalizedAmplitude": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageTransformKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageUncropByMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageUpscaleWithModelBatched": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "InjectNoiseToLatent": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "InsertImageBatchByIndexes": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "InsertImagesToBatchIndexed": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "InsertLatentToIndexed": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "INTConstant": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "InterpolateCoords": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "JoinStringMulti": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "JoinStrings": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Krea2PromptWeight": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LatentInpaintTTM": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LazySwitchKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LeapfusionHunyuanI2VPatcher": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LoadAndResizeImage": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LoadImagesFromFolderKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LoadVideosFromFolder": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LoraExtractKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LoraReduceRankKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LTX2_NAG": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LTX2AttentionTunerPatch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LTX2AudioLatentNormalizingSampling": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LTX2BlockLoraSelect": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LTX2LoraLoaderAdvanced": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LTX2MemoryEfficientSageAttentionPatch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LTX2SamplingPreviewOverride": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LTXVAddGuideMulti": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LTXVAddGuidesFromBatch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LTXVAudioVideoMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LTXVChunkFeedForward": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LTXVEnhanceAVideoKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LTXVImgToVideoInplaceKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MaskBatchMulti": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MaskOrImageToWeight": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MergeImageChannels": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ModelMemoryUsageFactorOverride": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ModelMemoryUseReportPatch": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ModelPassThrough": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ModelPatchTorchSettings": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ModelPreviewOverrideKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ModelSaveKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "NABLA_AttentionKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "NormalizedAmplitudeToFloatList": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "NormalizedAmplitudeToMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "OffsetMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "OffsetMaskByNormalizedAmplitude": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PadImageBatchInterleaved": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PatchFlashAttentionKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PatchModelPatcherOrder": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PathchSageAttentionKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PiDColorBiasCorrection": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PlaySoundKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PlotCoordinates": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PointsEditor": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PreviewAnimation": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PreviewImageOrMask": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "PreviewLatentNoiseMask": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "RandomImageFromBatch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "RemapImageRange": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "RemapMaskRange": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ReplaceImagesInBatch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ResizeMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ReverseImageBatch": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "RoundMask": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "SamplerSelfRefineVideo": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SaveImageKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SaveImageWithAlpha": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SaveStringKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ScaleBatchPromptSchedule": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ScheduledCFGGuidance": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Screencap_mss": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ScreencapStream": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SeparateMasks": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SetShakkerLabsUnionControlNetType": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ShuffleImageBatch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SigmasToFloat": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "SimpleCalculatorKJ": { + "connectFirst": [], + "connectLast": "NO_PRODUCER", + "disconnect": [] + }, + "SkipLayerGuidanceWanVideo": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Sleep": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SomethingToString": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SoundReactive": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SplineEditor": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SplitBboxes": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SplitImageChannels": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "StableZero123_BatchSchedule": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "StartRecordCUDAMemoryHistory": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "StringConstant": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "StringConstantMultiline": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "StringToFloatList": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "StyleModelApplyAdvanced": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Superprompt": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SV3D_BatchSchedule": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "TimerNodeKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "TorchCompileControlNet": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "TorchCompileCosmosModel": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "TorchCompileLTXModel": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "TorchCompileModelAdvanced": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "TorchCompileModelFluxAdvanced": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "TorchCompileModelFluxAdvancedV2": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "TorchCompileModelHyVideo": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "TorchCompileModelQwenImage": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "TorchCompileModelWanVideo": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "TorchCompileModelWanVideoV2": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "TorchCompileVAE": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "TransitionImagesInBatch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "TransitionImagesMulti": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VAEDecodeLoopKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VAELoaderKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VAEMergeKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VisualizeCUDAMemoryHistory": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "VisualizeSigmasKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VRAM_Debug": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Wan21BlockLoraSelect": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "WanChunkFeedForward": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "WanImageToVideoSVIPro": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "WanVideoEnhanceAVideoKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "WanVideoMemoryEfficientSageAttentionPatch": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "WanVideoNAG": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "WanVideoTeaCacheKJ": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "WebcamCaptureCV2": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "WeightScheduleConvert": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "WeightScheduleExtend": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "WidgetToString": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + } + } +} diff --git a/browser_tests/fixtures/customNode/interactionProfiles/ComfyUI-VideoHelperSuite.json b/browser_tests/fixtures/customNode/interactionProfiles/ComfyUI-VideoHelperSuite.json new file mode 100644 index 00000000000..7f68778fdf8 --- /dev/null +++ b/browser_tests/fixtures/customNode/interactionProfiles/ComfyUI-VideoHelperSuite.json @@ -0,0 +1,209 @@ +{ + "recordedAt": { + "core": "b08e6cf35fac50d3ca8470dffb3f9a1fbb7187d2", + "pin": "4ee72c065db22c9d96c2427954dc69e7b908444b" + }, + "schema": 1, + "nodes": { + "VHS_AudioToVHSAudio": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "VHS_BatchManager": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "VHS_DuplicateImages": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_DuplicateLatents": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_DuplicateMasks": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_GetImageCount": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "VHS_GetLatentCount": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "VHS_GetMaskCount": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "VHS_LoadAudio": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_LoadAudioUpload": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_LoadImagePath": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_LoadImages": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_LoadImagesPath": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_LoadVideo": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_LoadVideoFFmpeg": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_LoadVideoFFmpegPath": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_LoadVideoPath": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_MergeImages": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_MergeLatents": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_MergeMasks": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_PruneOutputs": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_SelectEveryNthImage": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_SelectEveryNthLatent": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_SelectEveryNthMask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_SelectFilename": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_SelectImages": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_SelectLatents": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_SelectLatest": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_SelectMasks": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_SplitImages": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_SplitLatents": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_SplitMasks": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_Unbatch": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "VHS_VAEDecodeBatched": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_VAEEncodeBatched": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_VHSAudioToAudio": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "VHS_VideoCombine": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VHS_VideoInfo": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "VHS_VideoInfoLoaded": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "VHS_VideoInfoSource": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + } + } +} diff --git a/browser_tests/fixtures/customNode/interactionProfiles/ComfyUI_essentials.json b/browser_tests/fixtures/customNode/interactionProfiles/ComfyUI_essentials.json new file mode 100644 index 00000000000..30ef6751a08 --- /dev/null +++ b/browser_tests/fixtures/customNode/interactionProfiles/ComfyUI_essentials.json @@ -0,0 +1,434 @@ +{ + "recordedAt": { + "core": "b08e6cf35fac50d3ca8470dffb3f9a1fbb7187d2", + "pin": "9d9f4bedfc9f0321c19faf71855e228c93bd0dc9" + }, + "schema": 1, + "nodes": { + "ApplyCLIPSeg+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "BatchCount+": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "CLIPTextEncodeSDXL+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ConditioningCombineMultiple+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ConsoleDebug+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "DebugTensorShape+": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "DisplayAny": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "DrawText+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "FluxAttentionSeeker+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "FluxBlocksBuster+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "FluxSamplerParams+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "GetImageSize+": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "GuidanceTimestepping+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageApplyLUT+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageBatchMultiple+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageBatchToList+": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImageCASharpening+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageColorMatch+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageColorMatchAdobe+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageComposite+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageCompositeFromMaskBatch+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageCrop+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageDesaturate+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageEnhanceDifference+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageExpandBatch+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageFlip+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageFromBatch+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageHistogramMatch+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageListToBatch+": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImagePosterize+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImagePreviewFromLatent+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageRandomTransform+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageRemoveAlpha+": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "ImageRemoveBackground+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageResize+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageSeamCarving+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageSmartSharpen+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageTile+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageToDevice+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ImageUntile+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "InjectLatentNoise+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "KSamplerVariationsStochastic+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "KSamplerVariationsWithNoise+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "LoadCLIPSegModels+": { + "connectFirst": "NO_INPUTS", + "connectLast": "NO_INPUTS", + "disconnect": "NO_INPUTS" + }, + "LorasForFluxParams+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MaskBatch+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MaskBlur+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MaskBoundingBox+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MaskExpandBatch+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MaskFix+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MaskFlip+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MaskFromBatch+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MaskFromColor+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MaskFromList+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MaskFromRGBCMYBW+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MaskFromSegmentation+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MaskPreview+": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "MaskSmooth+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ModelCompile+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "ModelSamplingSD3Advanced+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "NoiseFromImage+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PixelOEPixelize+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "PlotParameters+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "RemBGSession+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "RemoveLatentMask+": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "SamplerSelectHelper+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SchedulerSelectHelper+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SD3AttentionSeekerLG+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SD3AttentionSeekerT5+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SD3NegativeConditioning+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SDXLEmptyLatentSizePicker+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SimpleComparison+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SimpleCondition+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SimpleMath+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SimpleMathBoolean+": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "SimpleMathCondition+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SimpleMathDual+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SimpleMathFloat+": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "SimpleMathInt+": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "SimpleMathPercent+": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "SimpleMathSlider+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SimpleMathSliderLowRes+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "TextEncodeForSamplerParams+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "TransitionMask+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "TransparentBGSession+": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + } + } +} diff --git a/browser_tests/fixtures/customNode/interactionProfiles/was-node-suite-comfyui.json b/browser_tests/fixtures/customNode/interactionProfiles/was-node-suite-comfyui.json new file mode 100644 index 00000000000..1967e19cd0c --- /dev/null +++ b/browser_tests/fixtures/customNode/interactionProfiles/was-node-suite-comfyui.json @@ -0,0 +1,1109 @@ +{ + "recordedAt": { + "core": "b08e6cf35fac50d3ca8470dffb3f9a1fbb7187d2", + "pin": "ea935d1044ae5a26efa54ebeb18fe9020af49a45" + }, + "schema": 1, + "nodes": { + "Blend Latents": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "BLIP Analyze Image": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "BLIP Model Loader": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Boolean To Text": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Bounded Image Blend": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Bounded Image Blend with Mask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Bounded Image Crop": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Bounded Image Crop with Mask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Bus Node": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Cache Node": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Checkpoint Loader": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Checkpoint Loader (Simple)": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "CLIP Input Switch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CLIP Vision Input Switch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CLIPSeg Batch Masking": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CLIPSeg Masking": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CLIPSeg Model Loader": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "CLIPSEG2": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "CLIPTextEncode (NSP)": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Conditioning Input Switch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Constant Number": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Control Net Model Input Switch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Convert Masks to Images": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Create Grid Image": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Create Grid Image from Batch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Create Morph Image": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Create Morph Image from Path": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Create Video from Path": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Debug Number to Console": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Dictionary to Console": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Diffusers Hub Model Down-Loader": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Diffusers Model Loader": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Export API": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Hex to HSL": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "HSL to Hex": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Image Analyze": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Aspect Ratio": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Batch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Blank": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Blend": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Blend by Mask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Blending Mode": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Bloom Filter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Bounds": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Image Bounds to Console": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Canny Filter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Chromatic Aberration": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Color Palette": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Crop Face": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Crop Location": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Crop Square Location": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Displacement Warp": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Dragan Photography Filter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Edge Detection Filter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image fDOF Filter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Film Grain": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Filter Adjustments": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Flip": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Generate Gradient": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Gradient Map": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image High Pass Filter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image History Loader": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Image Input Switch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Levels Adjustment": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Load": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Lucy Sharpen": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Median Filter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Mix RGB Channels": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Monitor Effects Filter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Nova Filter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Padding": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Paste Crop": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Paste Crop by Location": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Paste Face": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Perlin Noise": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Perlin Power Fractal": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Pixelate": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Power Noise": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Rembg (Remove Background)": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Remove Background (Alpha)": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Remove Color": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Resize": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Rotate": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Rotate Hue": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Save": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Seamless Texture": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Select Channel": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Select Color": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Send HTTP": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Shadows and Highlights": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Size to Number": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Image SSAO (Ambient Occlusion)": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image SSDO (Direct Occlusion)": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Stitch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Style Filter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Threshold": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Tiled": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image to Latent Mask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image to Noise": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image to Seed": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Image Transpose": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Image Voronoi Noise Filter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Images to Linear": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Images to RGB": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Inset Image Bounds": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Integer place counter": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "KSampler (WAS)": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "KSampler Cycle": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Latent Batch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Latent Input Switch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Latent Noise Injection": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Latent Size to Number": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Latent Upscale by Factor (WAS)": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Load Cache": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Load Image Batch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Load Lora": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Load Text File": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Logic Boolean": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Logic Boolean Primitive": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Logic Comparison AND": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Logic Comparison OR": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Logic Comparison XOR": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Logic NOT": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Lora Input Switch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Lora Loader": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Mask Arbitrary Region": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Mask Batch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Mask Batch to Mask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Mask Ceiling Region": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Mask Crop Dominant Region": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Mask Crop Minority Region": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Mask Crop Region": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Mask Dilate Region": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Mask Dominant Region": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Mask Erode Region": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Mask Fill Holes": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Mask Floor Region": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Mask Gaussian Region": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Mask Invert": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Mask Minority Region": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Mask Paste Region": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Mask Rect Area": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Mask Rect Area (Advanced)": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Mask Smooth Region": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Mask Threshold Region": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Masks Add": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Masks Combine Batch": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Masks Combine Regions": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Masks Subtract": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MiDaS Depth Approximation": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MiDaS Mask Image": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "MiDaS Model Loader": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Model Input Switch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Number Counter": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Number Input Condition": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Number Input Switch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Number Multiple Of": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Number Operation": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Number PI": { + "connectFirst": "NO_INPUTS", + "connectLast": "NO_INPUTS", + "disconnect": "NO_INPUTS" + }, + "Number to Float": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Number to Int": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Number to Seed": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Number to String": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Number to Text": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Prompt Multiple Styles Selector": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Prompt Styles Selector": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Random Number": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SAM Image Mask": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SAM Model Loader": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "SAM Parameters": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "SAM Parameters Combine": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Samples Passthrough (Stat System)": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Save Text File": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Seed": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "String to Text": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Tensor Batch to Image": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text Add Token by Input": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text Add Tokens": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text Compare": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text Concatenate": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text Contains": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text Dictionary Convert": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Text Dictionary Get": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text Dictionary Keys": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Text Dictionary New": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text Dictionary To Text": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Text Dictionary Update": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text File History Loader": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text Find": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text Find and Replace": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text Find and Replace by Dictionary": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text Find and Replace Input": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text Input Switch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text List": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text List Concatenate": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text List to Text": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text Load Line From File": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text Multiline": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Text Multiline (Code Compatible)": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Text Parse A1111 Embeddings": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Text Parse Noodle Soup Prompts": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text Parse Tokens": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Text Random Line": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text Random Prompt": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Text Shuffle": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text Sort": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text String": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text String Truncate": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text to Conditioning": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text to Console": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Text to Number": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Text to String": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "True Random.org Number Generator": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "unCLIP Checkpoint Loader": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Upscale Model Loader": { + "connectFirst": [], + "connectLast": "SAME_AS_FIRST", + "disconnect": [] + }, + "Upscale Model Switch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "VAE Input Switch": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Video Dump Frames": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Write to GIF": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + }, + "Write to Video": { + "connectFirst": [], + "connectLast": [], + "disconnect": [] + } + } +} diff --git a/browser_tests/fixtures/customNode/manifest.ts b/browser_tests/fixtures/customNode/manifest.ts new file mode 100644 index 00000000000..5ddd9512c46 --- /dev/null +++ b/browser_tests/fixtures/customNode/manifest.ts @@ -0,0 +1,724 @@ +import { existsSync, readFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' + +// Lazy: import.meta.url is not a file: URL under vitest happy-dom. +function dataPath(basename: string): string { + return fileURLToPath(new URL(`../data/${basename}`, import.meta.url)) +} + +const VALID_TIERS = ['load', 'run', 'connectivity'] as const + +export type CustomNodeTier = (typeof VALID_TIERS)[number] + +// Which pack list to test. The local gate tests the CLOUD pack list against a +// LOCAL backend, so this must NOT also select backend behaviour: Cloud's +// coreDisabledNodes describe Cloud, and applying them here would expect nodes +// to be absent that this backend serves. +const VALID_MANIFESTS = ['core', 'cloud'] as const +// Separate from the manifest: the local gate runs CLOUD's pack list against a +// LOCAL backend, and the two disagree about which nodes exist because Cloud +// disables some. localExpectations records that difference; it must not apply +// when these same specs run against Cloud. +const VALID_BACKENDS = ['local', 'cloud'] as const +const CLOUD_LOCAL_CALIBRATION_SHARDS = 5 + +interface SharedNodeExpectations { + pack: string + tiers: CustomNodeTier[] + // Frontend-format workflow (path relative to browser_tests/) loaded and queued + // by the run tier. Run the backend with --cache-none, or repeat runs classify + // PARTIAL when cached nodes skip executing. + workflow: string + // Runtime class_type / object_info keys, NOT Python class names (e.g. rgthree + // registers "Power Primitive (rgthree)", not RgthreePowerPrimitive). + expectedNodes: string[] + expectedRunnableCount?: number + timeoutMs: number +} + +export interface CoreManifestEntry extends SharedNodeExpectations { + repo: string + pin: string + // Frontend extension names the pack's JS registers at boot (via + // app.registerExtension), calibrated from the pinned source. Asserted + // against window.app.extensions in the load tier: backend nodes can + // register while the pack's frontend JS silently fails to load (wrong + // web dir, a loadExtensions regression), and every JS-dependent + // assertion in this suite would then quietly test vanilla nodes. + // Empty array = no healthy boot registration is expected. Source-proven + // broken registrations are tracked by the explicit broken-extension ledger. + expectedExtensions: string[] + // Exact number of nodes the pack registers at its pin, calibrated from the + // gating CI's object_info. The all-nodes tiers derive their corpus from the + // live backend, so without this a pack that silently registers fewer nodes + // (a broken sub-import, a core change breaking registration) shrinks + // coverage while CI stays green. Any delta - either direction - fails until + // the count is deliberately recalibrated alongside a pin/core change. + expectedNodeCount: number + requiresGpu: boolean + requiresModels: string[] + // Nodes that cannot execute on pure defaults. Asserted both ways: an + // unlisted failure is a regression, a listed clean run is a stale entry. + cannotRunAlone?: string[] +} + +// Rows are generated by `pnpm gen:cloud-manifest `. +export interface CloudManifestEntry extends SharedNodeExpectations { + deployRef: string + disabledNodes: Record + expectedExtensions: string[] + expectedNodeCount: number + webDirectory?: string + cannotRunAlone?: string[] +} + +export interface CloudManifestSource { + repository: string + ref: string + path: string + importedAt: string +} + +export interface CloudManifest { + source: CloudManifestSource + coreDisabledNodes: Record + packs: CloudManifestEntry[] + // yaml packs the snapshot has no nodes for. Recorded rather than dropped: + // the snapshot cannot distinguish "pack registers nothing" from "the dirname + // mapping broke", so the list is a reviewed record - a pack appearing here + // has no cloud coverage, and the list growing is the signal to look. + unjoinedYamlPacks: string[] +} + +export const AUTOGROW_CASES = [ + { + pack: 'ComfyUI-Impact-Pack', + extensionName: 'Comfy.Impack', + extensionPathPack: 'comfyui-impact-pack', + consumerType: 'ImpactMakeImageList', + producerType: 'EmptyImage', + producerSlot: 'IMAGE' + } +] as const + +export const FRONTEND_ASSET_EXCLUSIONS: Record< + string, + { + deployRef: string + reason: string + restore: string + webDirectory: string + } +> = { + 'ComfyUI-DepthAnythingV3': { + deployRef: 'ComfyUI-DepthAnythingV3@0.1.2', + webDirectory: 'web', + reason: + 'the registry artifact declares WEB_DIRECTORY="./web" but contains no web/ directory', + restore: + 'publish an artifact containing web/ and remove this entry when the served extension path appears' + } +} + +function extensionPathPack(value: string): string | undefined { + try { + const segments = new URL(value, 'http://comfy.invalid').pathname + .split('/') + .filter(Boolean) + .flatMap((segment) => { + try { + return decodeURIComponent(segment).split('/').filter(Boolean) + } catch { + return [segment] + } + }) + const extensionsIndex = segments.findIndex( + (segment) => segment.toLowerCase() === 'extensions' + ) + return extensionsIndex >= 0 ? segments[extensionsIndex + 1] : undefined + } catch { + return undefined + } +} + +export function servesFrontendAssetsForPack( + servedExtensionPaths: readonly string[], + pack: string +): boolean { + return servedExtensionPaths.some( + (value) => extensionPathPack(value)?.toLowerCase() === pack.toLowerCase() + ) +} + +export function staleAutogrowApplicabilityIssues( + entry: { pack: string; expectedExtensions: readonly string[] }, + registeredExtensions: readonly string[], + servedExtensionPaths: readonly string[] +): string[] { + return AUTOGROW_CASES.flatMap((autogrowCase) => { + if ( + autogrowCase.pack.toLowerCase() !== entry.pack.toLowerCase() || + entry.expectedExtensions.includes(autogrowCase.extensionName) + ) + return [] + + const served = servesFrontendAssetsForPack( + servedExtensionPaths, + autogrowCase.extensionPathPack + ) + return [ + ...(served + ? [ + `frontend assets for "${autogrowCase.extensionName}" are now served - restore expectedExtensions and autogrow coverage` + ] + : []), + ...(registeredExtensions.includes(autogrowCase.extensionName) + ? [ + `frontend extension "${autogrowCase.extensionName}" is now registered - restore expectedExtensions and autogrow coverage` + ] + : []) + ] + }) +} + +function workflowFileExists(workflow: string): boolean { + try { + return existsSync( + fileURLToPath(new URL(`../../${workflow}`, import.meta.url)) + ) + } catch { + return false + } +} + +function sharedIssues(entry: SharedNodeExpectations): string[] { + const missing: string[] = [] + // CI installs the pack into custom_nodes/, and node attribution keys + // on that directory name via python_module - so pack must be a safe, + // plain path segment, not just non-empty. + if ( + typeof entry.pack !== 'string' || + !/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(entry.pack) + ) + missing.push('pack (must be a plain path segment)') + // workflow may be an empty string until the pack gains a run-tier fixture. + if (typeof entry.workflow !== 'string') missing.push('workflow') + else if ( + Array.isArray(entry.tiers) && + entry.tiers.includes('run') && + (entry.workflow === '' || !workflowFileExists(entry.workflow)) + ) + missing.push('workflow (committed file required when tiers includes "run")') + if (!Array.isArray(entry.expectedNodes) || entry.expectedNodes.length === 0) + missing.push('expectedNodes') + if (!Array.isArray(entry.tiers) || entry.tiers.length === 0) + missing.push('tiers') + // A typo like "connectivty" would otherwise pass and silently drop that + // tier's coverage - the exact drift this manifest exists to catch. + else if (entry.tiers.some((tier) => !VALID_TIERS.includes(tier))) + missing.push(`tiers (unknown value; allowed: ${VALID_TIERS.join(', ')})`) + else if ( + entry.tiers.includes('run') && + (!Number.isInteger(entry.expectedRunnableCount) || + (entry.expectedRunnableCount ?? 0) <= 0) + ) + missing.push('expectedRunnableCount (positive integer for run tier)') + else if ( + !entry.tiers.includes('run') && + entry.expectedRunnableCount !== undefined + ) + missing.push('expectedRunnableCount (only valid for run tier)') + if (!Number.isFinite(entry.timeoutMs) || entry.timeoutMs <= 0) + missing.push('timeoutMs') + return missing +} + +function isNonEmptyString(value: unknown): boolean { + return typeof value === 'string' && value.length > 0 +} + +function isLabelList(value: unknown): boolean { + return ( + Array.isArray(value) && + value.length > 0 && + value.every(isNonEmptyString) && + new Set(value).size === value.length + ) +} + +function invalidRecordOf( + map: unknown, + isValidValue: (value: unknown) => boolean +): boolean { + return ( + typeof map !== 'object' || + map === null || + Array.isArray(map) || + Object.values(map).some((value) => !isValidValue(value)) + ) +} + +function calibrationIssues( + entry: Pick< + CoreManifestEntry, + 'expectedExtensions' | 'expectedNodeCount' | 'cannotRunAlone' + > +): string[] { + const missing: string[] = [] + // Explicitly required (an empty array deliberately expects no healthy boot + // registration; known-broken registrations use their separate ledger) so a + // new pack row cannot silently opt out of the extension-loaded assert. + if ( + !Array.isArray(entry.expectedExtensions) || + !entry.expectedExtensions.every(isNonEmptyString) || + new Set(entry.expectedExtensions).size !== entry.expectedExtensions.length + ) + missing.push('expectedExtensions (unique non-empty extension names)') + if ( + !Number.isInteger(entry.expectedNodeCount) || + entry.expectedNodeCount <= 0 + ) + missing.push('expectedNodeCount (positive integer, calibrated at the pin)') + if ( + entry.cannotRunAlone !== undefined && + (!Array.isArray(entry.cannotRunAlone) || + !entry.cannotRunAlone.every(isNonEmptyString) || + new Set(entry.cannotRunAlone).size !== entry.cannotRunAlone.length) + ) + missing.push('cannotRunAlone (unique non-empty node keys)') + return missing +} + +function throwIfIncomplete( + label: string, + pack: unknown, + index: number, + missing: string[] +): void { + if (missing.length > 0) + throw new Error( + `${label} entry ${index} (${typeof pack === 'string' ? pack : '?'}) missing: ${missing.join(', ')}` + ) +} + +// Exported for the pure spec's validation cases; production callers go +// through loadManifest. +export function assertCoreEntry(entry: CoreManifestEntry, index: number): void { + const missing = sharedIssues(entry) + // CI clones from repo, so an empty value must fail here, not mid-clone. + if (!isNonEmptyString(entry.repo)) missing.push('repo') + if (!/^[0-9a-f]{40}$/.test(entry.pin ?? '')) + missing.push('pin (full 40-char commit SHA required)') + missing.push(...calibrationIssues(entry)) + if (!Array.isArray(entry.requiresModels)) missing.push('requiresModels') + if (typeof entry.requiresGpu !== 'boolean') missing.push('requiresGpu') + if ( + entry.tiers.includes('run') && + (entry.requiresGpu || + (Array.isArray(entry.requiresModels) && entry.requiresModels.length > 0)) + ) + missing.push('run tier (must be executable on the model-free CPU gate)') + throwIfIncomplete('custom-node manifest', entry.pack, index, missing) +} + +const DEPLOY_REF_URL = /^https?:\/\/\S+@[0-9a-f]{40}$/ +const DEPLOY_REF_REGISTRY = /^[A-Za-z0-9][A-Za-z0-9._-]*@\S+$/ + +export function assertCloudEntry( + entry: CloudManifestEntry, + index: number +): void { + const missing = sharedIssues(entry) + if ( + typeof entry.deployRef !== 'string' || + !( + DEPLOY_REF_URL.test(entry.deployRef) || + DEPLOY_REF_REGISTRY.test(entry.deployRef) + ) + ) + missing.push('deployRef (git URL@sha or registry id@version)') + if (invalidRecordOf(entry.disabledNodes, isLabelList)) + missing.push('disabledNodes (node key -> non-empty unique label list)') + if ( + entry.webDirectory !== undefined && + (typeof entry.webDirectory !== 'string' || + !/^(?!.*(?:^|\/)\.\.(?:\/|$))[^/][A-Za-z0-9._/-]*$/.test( + entry.webDirectory + )) + ) + missing.push('webDirectory (safe relative path)') + missing.push(...calibrationIssues(entry)) + throwIfIncomplete('custom-node cloud manifest', entry.pack, index, missing) +} + +function readEnum( + name: string, + allowed: T, + fallback: T[number] +): T[number] { + const raw = process.env[name] || fallback + const hit = allowed.find((candidate) => candidate === raw) + if (hit === undefined) + throw new Error( + `${name}='${raw}' is not valid (allowed: ${allowed.join(', ')}; unset defaults to ${fallback})` + ) + return hit +} + +export function customNodesManifest(): (typeof VALID_MANIFESTS)[number] { + return readEnum('CUSTOM_NODES_MANIFEST', VALID_MANIFESTS, 'core') +} + +function customNodesBackend(): (typeof VALID_BACKENDS)[number] { + return readEnum('CUSTOM_NODES_BACKEND', VALID_BACKENDS, 'local') +} + +interface LocalExpectation { + expectedNodeCount: number + expectedExtensions?: string[] + reason: string +} + +function loadLocalExpectations(): Record { + if (customNodesBackend() !== 'local' || customNodesManifest() !== 'cloud') + return {} + const path = dataPath('cloud/localExpectations.json') + if (!existsSync(path)) return {} + const parsed: unknown = JSON.parse(readFileSync(path, 'utf-8')) + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) + throw new Error(`${path} must be a JSON object keyed by pack name`) + const entries = parsed as Record + const manifestPacks = new Set( + readCloudManifest().packs.map(({ pack }) => pack) + ) + for (const [pack, entry] of Object.entries(entries)) { + if (!manifestPacks.has(pack)) + throw new Error(`${path}: ${pack} is not in the cloud manifest`) + if ( + typeof entry !== 'object' || + entry === null || + !Number.isInteger(entry.expectedNodeCount) || + entry.expectedNodeCount <= 0 || + !isNonEmptyString(entry.reason) || + (entry.expectedExtensions !== undefined && + (!Array.isArray(entry.expectedExtensions) || + entry.expectedExtensions.length !== + new Set(entry.expectedExtensions).size || + !entry.expectedExtensions.every(isNonEmptyString))) + ) + throw new Error(`${path}: ${pack} has an invalid local expectation`) + } + return entries +} + +/** + * How many nodes this pack registers on the backend under test. + * + * The cloud manifest is generated from Cloud's object_info, and a local + * ComfyUI serves the nodes Cloud disables - so the manifest number is right + * for Cloud and wrong here. Nodes are not disabled locally to imitate Cloud; + * what a local backend actually serves is recorded instead. + */ +export function expectedNodeCountFor( + entry: CoreManifestEntry | CloudManifestEntry +): number { + return ( + loadLocalExpectations()[entry.pack]?.expectedNodeCount ?? + entry.expectedNodeCount + ) +} + +/** + * Frontend extensions this pack registers on the backend under test. + * + * Cloud serves no frontend assets for some packs and records an empty list; a + * local install serves them, and staleAutogrowApplicabilityIssues correctly + * reds when coverage was withheld on a premise that no longer holds here. + */ +export function expectedExtensionsFor( + entry: CoreManifestEntry | CloudManifestEntry +): string[] { + return ( + loadLocalExpectations()[entry.pack]?.expectedExtensions ?? + entry.expectedExtensions + ) +} + +/** Entries whose local count has caught up with the manifest are stale. */ +export function staleLocalExpectations(): string[] { + const local = loadLocalExpectations() + return loadFullManifest() + .filter( + (entry) => + local[entry.pack] !== undefined && + local[entry.pack].expectedNodeCount === entry.expectedNodeCount + ) + .map((entry) => entry.pack) +} + +/** + * Which slice of the manifest this run owns, as `index/total`, 1-based. + */ +function manifestShard(): { index: number; total: number } | null { + const raw = process.env.CUSTOM_NODES_SHARD + if (!raw) return null + const parsed = /^(\d+)\/(\d+)$/.exec(raw) + if (!parsed) + throw new Error( + `CUSTOM_NODES_SHARD='${raw}' is not index/total (e.g. 3/10)` + ) + const index = Number(parsed[1]) + const total = Number(parsed[2]) + if (index < 1 || total < 1 || index > total) + throw new Error( + `CUSTOM_NODES_SHARD='${raw}' is out of range (need 1 <= index <= total)` + ) + return { index, total } +} + +/** + * Balanced by node count, not by pack count. + * + * Equal pack counts are not equal work: expectedNodeCount runs from 1 to 285 + * with a median of 6, and S1/S2/S3 walk every node, so a few packs dominate a + * slice. Round-robin striping ignored that and produced a 4.86x spread on run + * 31849196357 - 1,220 nodes in one shard against 251 in another - and since a + * matrix costs its slowest shard, that one straggler was the whole run. + * + * Greedy longest-processing-time: heaviest pack first, always into the + * lightest shard. Deterministic for the pinned manifest; ties preserve its + * reviewed order. + */ +export function shardOf( + entries: readonly T[], + weightOf: (entry: T) => number = () => 1 +): T[] { + const shard = manifestShard() + if (!shard) return [...entries] + const bins: T[][] = Array.from({ length: shard.total }, () => []) + const load = new Array(shard.total).fill(0) + const ordered = [...entries] + .map((entry, index) => ({ entry, index })) + .sort((a, b) => weightOf(b.entry) - weightOf(a.entry) || a.index - b.index) + for (const { entry } of ordered) { + const lightest = load.indexOf(Math.min(...load)) + bins[lightest].push(entry) + load[lightest] += weightOf(entry) + } + return bins[shard.index - 1] +} + +export function assertCloudManifestShape( + parsed: unknown, + sourcePath: string +): CloudManifest { + const manifest = parsed as CloudManifest + if ( + typeof manifest !== 'object' || + manifest === null || + !isCloudManifestSource(manifest.source) || + invalidRecordOf(manifest.coreDisabledNodes, isLabelList) || + !Array.isArray(manifest.packs) || + manifest.packs.length === 0 || + // Required, and an empty array is the deliberate "every yaml pack joined" + // declaration - the live assert reads this list, so a missing or malformed + // one would silently assert nothing. + !Array.isArray(manifest.unjoinedYamlPacks) || + !manifest.unjoinedYamlPacks.every(isNonEmptyString) || + new Set(manifest.unjoinedYamlPacks).size !== + manifest.unjoinedYamlPacks.length + ) + throw new Error( + `${sourcePath} is malformed (expected { source, coreDisabledNodes, packs, unjoinedYamlPacks } with at least one pack): regenerate it via 'pnpm gen:cloud-manifest'` + ) + manifest.packs.forEach(assertCloudEntry) + return manifest +} + +function isIsoDate(value: unknown): value is string { + if (typeof value !== 'string') return false + const timestamp = Date.parse(`${value}T00:00:00Z`) + return ( + !Number.isNaN(timestamp) && + new Date(timestamp).toISOString().slice(0, 10) === value + ) +} + +function isCloudManifestSource(value: unknown): value is CloudManifestSource { + if (typeof value !== 'object' || value === null || Array.isArray(value)) + return false + const source = value as Record + if ( + typeof source.repository !== 'string' || + !/^https:\/\/github\.com\/[^/]+\/[^/]+$/.test(source.repository) || + typeof source.ref !== 'string' || + !/^[0-9a-f]{40}$/.test(source.ref) || + typeof source.path !== 'string' || + source.path.length === 0 || + !isIsoDate(source.importedAt) + ) + return false + return true +} + +function readCloudManifest(): CloudManifest { + const cloudPath = dataPath('customNodeManifest.cloud.json') + if (!existsSync(cloudPath)) + throw new Error( + `CUSTOM_NODES_MANIFEST=cloud but ${cloudPath} does not exist. ` + + `The cloud manifest is generated, never hand-authored: run ` + + `'pnpm gen:cloud-manifest ', which joins ` + + `the vendored browser_tests/` + + `fixtures/data/cloud/supported_nodes.yaml with a Cloud /object_info ` + + `snapshot. That snapshot is the Phase-1 probe output; until the ` + + `probe has run, the cloud suite cannot run.` + ) + return assertCloudManifestShape( + JSON.parse(readFileSync(cloudPath, 'utf-8')), + cloudPath + ) +} + +function readCoreManifest(): CoreManifestEntry[] { + const path = dataPath('customNodeManifest.core.json') + const parsed: unknown = JSON.parse(readFileSync(path, 'utf-8')) + if (!Array.isArray(parsed)) + throw new Error( + `custom-node manifest ${path} must be a JSON array of entries, got ${parsed === null ? 'null' : typeof parsed}` + ) + const entries: CoreManifestEntry[] = parsed + entries.forEach(assertCoreEntry) + return entries +} + +export function loadAllManifestPackNames(): string[] { + return [ + ...readCoreManifest().map((entry) => entry.pack), + ...readCloudManifest().packs.map((entry) => entry.pack) + ] +} + +export function loadUnjoinedYamlPacks(): string[] { + return customNodesManifest() === 'cloud' + ? readCloudManifest().unjoinedYamlPacks + : [] +} + +/** + * What this pack is pinned to, whatever form its manifest uses. + * + * Core rows carry a git commit; cloud rows carry the deployRef Cloud actually + * ships - `owner/repo@sha` for git-sourced packs, `id@version` for registry + * packs, which the registry distributes as a CDN artifact and for which no + * commit exists. Callers recording provenance want the identity, not the form. + */ +export function packIdentity( + entry: CoreManifestEntry | CloudManifestEntry +): string { + return 'deployRef' in entry ? entry.deployRef : entry.pin +} + +interface QuarantineDetails { + reason: string + evidence: string + upstreamFix: string + since: string +} + +export type QuarantinedPack = QuarantineDetails & + ( + | { class: 'unfetchable-ref'; failurePattern?: never } + | { class: 'unsatisfiable-requirement'; failurePattern: string } + | { class: 'requires-gpu-runner'; failurePattern?: never } + ) + +// Packs this local CPU gate cannot exercise. Network-verifiable source failures +// are rechecked by the workflow; hardware exclusions remain visible in every +// Actions summary until a GPU-backed shard restores their coverage. +export function loadPackQuarantine(): Record { + const path = dataPath('cloud/packQuarantine.json') + if (!existsSync(path)) return {} + const parsed: unknown = JSON.parse(readFileSync(path, 'utf-8')) + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) + throw new Error(`${path} must be a JSON object keyed by pack name`) + const entries = parsed as Record + const manifestPacks = new Set( + readCloudManifest().packs.map(({ pack }) => pack) + ) + const classes = new Set([ + 'unfetchable-ref', + 'unsatisfiable-requirement', + 'requires-gpu-runner' + ]) + for (const [pack, entry] of Object.entries(entries)) { + if (!manifestPacks.has(pack)) + throw new Error(`${path}: ${pack} is not in the cloud manifest`) + if ( + typeof entry !== 'object' || + entry === null || + !classes.has(entry.class) || + !isNonEmptyString(entry.reason) || + !isNonEmptyString(entry.evidence) || + (entry.class === 'unsatisfiable-requirement' && + !isNonEmptyString(entry.failurePattern)) || + !isNonEmptyString(entry.upstreamFix) || + !isIsoDate(entry.since) + ) + throw new Error(`${path}: ${pack} has an invalid quarantine entry`) + } + return entries +} + +export function loadManifest(): (CoreManifestEntry | CloudManifestEntry)[] { + const shard = manifestShard() + if ( + customNodesManifest() === 'cloud' && + customNodesBackend() === 'local' && + shard !== null && + shard.total !== CLOUD_LOCAL_CALIBRATION_SHARDS + ) + throw new Error( + `cloud/local expectations are calibrated for ${CLOUD_LOCAL_CALIBRATION_SHARDS} shards, got ${shard.total}; recalibrate the exact node, persistence, and connectivity contracts before changing the matrix` + ) + // Assign over the FULL manifest, then drop the excluded packs. Packing the + // filtered list instead made shard composition depend on the exclusion + // ledgers: moving four packs out of quarantine moved 26 of 81 packs to a + // different shard. All packs in a shard share one Python environment, so a + // pack's neighbours decide which of its optional imports resolve and how + // many node classes it registers - WanVideoWrapper measured 141, 142 and 146 + // across three runs without its own pin changing. Composition now moves only + // when the manifest does. + const quarantined = + customNodesManifest() === 'cloud' ? loadPackQuarantine() : {} + return shardOf(loadFullManifest(), (entry) => entry.expectedNodeCount).filter( + (entry) => !(entry.pack in quarantined) + ) +} + +/** Every pack the manifest declares, ignoring the shard slice. */ +export function loadFullManifest(): (CoreManifestEntry | CloudManifestEntry)[] { + if (customNodesManifest() === 'cloud') return readCloudManifest().packs + return readCoreManifest() +} + +export function loadApplicableAutogrowCases() { + const full = loadFullManifest() + const shard = new Set(loadManifest().map((entry) => entry.pack)) + return AUTOGROW_CASES.flatMap((autogrowCase) => { + const manifestEntry = full.find( + (entry) => entry.pack.toLowerCase() === autogrowCase.pack.toLowerCase() + ) + // Stale AUTOGROW_CASES is a manifest-wide fact and must fail everywhere; + // whether this shard runs the pack is a separate question below. + if (!manifestEntry) + throw new Error( + `${autogrowCase.pack} is not a manifest pack - fix AUTOGROW_CASES` + ) + if (!shard.has(manifestEntry.pack)) return [] + return expectedExtensionsFor(manifestEntry).includes( + autogrowCase.extensionName + ) + ? [{ autogrowCase, manifestEntry }] + : [] + }) +} diff --git a/browser_tests/fixtures/customNode/objectInfoValidator.ts b/browser_tests/fixtures/customNode/objectInfoValidator.ts new file mode 100644 index 00000000000..34d83321247 --- /dev/null +++ b/browser_tests/fixtures/customNode/objectInfoValidator.ts @@ -0,0 +1,13 @@ +interface ObjectInfoNode { + input?: { required?: Record } +} +export type ObjectInfo = Record + +// Names from `expectedNodes` absent from the backend's object_info. Empty +// result = every expected node is registered on this backend. +export function missingExpectedNodes( + objectInfo: ObjectInfo, + expectedNodes: string[] +): string[] { + return expectedNodes.filter((name) => !(name in objectInfo)) +} diff --git a/browser_tests/fixtures/customNode/outputHashes.ts b/browser_tests/fixtures/customNode/outputHashes.ts new file mode 100644 index 00000000000..5ed657ed7ba --- /dev/null +++ b/browser_tests/fixtures/customNode/outputHashes.ts @@ -0,0 +1,214 @@ +import { createHash } from 'node:crypto' +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { dirname } from 'node:path' + +// S15 output-regression tier: content hashes for curated run outputs. +// +// A frontend regression can corrupt what a workflow PRODUCES while the run +// still ends in execution_success (a serialization change that drifts a +// widget value without invalidating it). The run tier proves "it ran"; this +// tier proves "it produced the same outputs": one digest per sink over its +// canonicalized ui payload, with PNG file refs hashed by pixel content +// (IDAT chunks only - ComfyUI embeds the prompt as tEXt metadata, so +// whole-file hashes would false-fail on byte-identical pixels). + +export interface OutputImageRef { + filename: string + subfolder: string + type: string +} + +// The real ComfyUI output ref shape carries type and/or subfolder beside +// filename; requiring one prevents a value payload that merely CONTAINS a +// filename field from being collapsed (and its sibling keys dropped). +function isFileRef( + value: unknown +): value is { filename: string; subfolder?: string; type?: string } { + if (typeof value !== 'object' || value === null) return false + const ref = value as Record + return ( + typeof ref.filename === 'string' && + (typeof ref.type === 'string' || typeof ref.subfolder === 'string') + ) +} + +// Deterministic canonical form of a sink's ui payload. The curated corpus's +// sinks are mostly VALUE displays (text/number payloads) - the displayed +// value IS the output, so the payload itself is what gets hashed (record run +// 30316904957: zero `images` arrays across all six curated workflows). +// File refs embed run-varying counters (ComfyUI_00001_.png), so a ref +// canonicalizes to its extension plus, for PNGs, the pixel hash of the +// fetched file - content-stable where content is checkable, position-stable +// where it is not (video containers embed timestamps). +async function canonicalize( + value: unknown, + fetchFile: (ref: OutputImageRef) => Promise +): Promise { + if (Array.isArray(value)) { + const out = [] + for (const item of value) out.push(await canonicalize(item, fetchFile)) + return out + } + if (isFileRef(value)) { + const ext = (value.filename.match(/\.[a-z0-9]+$/i)?.[0] ?? '').toLowerCase() + const ref: OutputImageRef = { + filename: value.filename, + subfolder: value.subfolder ?? '', + type: value.type ?? 'output' + } + // Sibling keys (format, frame_rate, ...) are content - keep them; only + // the run-varying filename collapses to its extension. + const rest: Record = {} + for (const key of Object.keys(value).sort()) + if (key !== 'filename') + rest[key] = await canonicalize( + (value as Record)[key], + fetchFile + ) + if (ext === '.png') + return { ...rest, file: ext, pixels: hashPngPixels(await fetchFile(ref)) } + return { ...rest, file: ext } + } + if (typeof value === 'object' && value !== null) { + const out: Record = {} + for (const key of Object.keys(value).sort()) + out[key] = await canonicalize( + (value as Record)[key], + fetchFile + ) + return out + } + return value +} + +// One digest per sink node: sha256 over the canonicalized payload JSON. +export async function hashSinkPayloads( + outputsByNode: Record, + fetchFile: (ref: OutputImageRef) => Promise +): Promise> { + const observed: Record = {} + for (const nodeId of Object.keys(outputsByNode).sort()) { + const canonical = await canonicalize(outputsByNode[nodeId], fetchFile) + observed[nodeId] = `sha256:${createHash('sha256') + .update(JSON.stringify(canonical)) + .digest('hex')}` + } + return observed +} + +const PNG_SIGNATURE = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]) + +// sha256 over the concatenated IDAT payloads - the compressed pixel stream, +// byte-stable for identical pixels regardless of tEXt/iTXt metadata. Throws +// on non-PNG bytes: a sink expected to emit PNG that emits something else IS +// a regression, not a skippable case. +export function hashPngPixels(file: Buffer): string { + if (!file.subarray(0, 8).equals(PNG_SIGNATURE)) + throw new Error( + `not a PNG (starts ${file.subarray(0, 8).toString('hex')}) - S15 hashes cover PNG sinks only` + ) + const hash = createHash('sha256') + let offset = 8 + let sawIdat = false + let sawIend = false + while (offset + 8 <= file.length) { + const length = file.readUInt32BE(offset) + const chunkType = file.toString('latin1', offset + 4, offset + 8) + if (offset + 12 + length > file.length) + throw new Error( + `PNG chunk ${chunkType} overruns the buffer - truncated file?` + ) + if (chunkType === 'IDAT') { + hash.update(file.subarray(offset + 8, offset + 8 + length)) + sawIdat = true + } + if (chunkType === 'IEND') { + sawIend = true + break + } + offset += 12 + length + } + if (!sawIdat) throw new Error('PNG has no IDAT chunk - truncated file?') + if (!sawIend) throw new Error('PNG has no IEND chunk - truncated file?') + return `sha256:${hash.digest('hex')}` +} + +export interface CuratedOutputHashes { + // Hashes are only comparable against the environment that recorded them + // (the S14 geometry convention): pinned core + pack pins determine sink + // payloads, and a drift red must name where its baseline came from. + recordedAt: { core: string; run: string } + schema: 1 + workflows: Record> +} + +// Read-merge-write: record mode deliberately fails each test, and Playwright +// restarts the worker after a failure, so in-memory accumulation resets per +// test - proven by record run 30316246562, whose artifact held only the last +// test's entry. The file on disk is the accumulator. +export function recordObservedHashes( + filePath: string, + workflowKey: string, + observed: Record +): void { + const existing: CuratedOutputHashes = existsSync(filePath) + ? JSON.parse(readFileSync(filePath, 'utf-8')) + : { + recordedAt: { + core: process.env.CN_OUTPUT_HASHES_CORE ?? 'unpinned-local', + run: process.env.GITHUB_RUN_ID ?? 'local' + }, + schema: 1, + workflows: {} + } + existing.workflows[workflowKey] = observed + mkdirSync(dirname(filePath), { recursive: true }) + writeFileSync(filePath, JSON.stringify(existing, null, 2)) +} + +export function compareOutputHashes(input: { + workflowKey: string + observed: Record + committed: CuratedOutputHashes +}): string[] { + const { workflowKey, observed, committed } = input + if (committed.schema !== 1 || !committed.recordedAt?.core) + throw new Error( + 'curatedOutputHashes fixture is not schema 1 with recordedAt provenance - re-record it (docs/custom-node-regression-suite.md Step 5c)' + ) + const provenance = `(baseline recorded at core ${committed.recordedAt.core}, run ${committed.recordedAt.run})` + const expected = committed.workflows[workflowKey] + if (!expected) + return [ + `S15: no committed hashes for '${workflowKey}' - a curated run workflow ` + + `must enroll its outputs (RECORD_OUTPUT_HASHES=1 run, commit the ` + + `fixture) or carry a ledger entry with a mechanism` + ] + if (Object.keys(expected).length === 0) + return [ + `S15: '${workflowKey}' has no committed output hashes - add an observable ` + + `sink before recording its baseline` + ] + const problems: string[] = [] + for (const [key, digest] of Object.entries(expected)) { + const actual = observed[key] + if (actual === undefined) + problems.push( + `${workflowKey} ${key}: committed hash but the output is gone - the ` + + `sink no longer produces this output` + ) + else if (actual !== digest) + problems.push( + `${workflowKey} ${key}: output hash changed - expected ${digest}, got ` + + `${actual}. A frontend change altered what this workflow produces ` + + provenance + ) + } + for (const key of Object.keys(observed)) + if (!(key in expected)) + problems.push( + `${workflowKey} ${key}: new output with no committed hash - enroll it ` + + `(a new output appearing is also a behavior change)` + ) + return problems +} diff --git a/browser_tests/fixtures/customNode/packLedger.ts b/browser_tests/fixtures/customNode/packLedger.ts new file mode 100644 index 00000000000..9d3e097098a --- /dev/null +++ b/browser_tests/fixtures/customNode/packLedger.ts @@ -0,0 +1,53 @@ +type NodeLedger = Record + +function matchingEntries( + ledger: Record>, + pack: string +): Array<[string, NodeLedger]> { + const folded = pack.toLowerCase() + return Object.entries(ledger).filter( + ([ledgered]) => ledgered.toLowerCase() === folded + ) +} + +export function packLedgerFor( + ledger: Record>, + pack: string +): NodeLedger { + const matches = matchingEntries(ledger, pack) + if (matches.length > 1) + throw new Error( + `pack ledger has duplicate case-insensitive keys for ${pack}: ${matches.map(([key]) => key).join(', ')}` + ) + return matches[0]?.[1] ?? {} +} + +export function assertPackLedgerKeys( + name: string, + ledger: Record>, + manifestPacks: string[] +): void { + const foldedManifest = new Set( + manifestPacks.map((pack) => pack.toLowerCase()) + ) + const unmatched = Object.keys(ledger).filter( + (pack) => !foldedManifest.has(pack.toLowerCase()) + ) + const duplicates = Object.keys(ledger).filter( + (pack, index, packs) => + packs.findIndex( + (candidate) => candidate.toLowerCase() === pack.toLowerCase() + ) !== index + ) + if (unmatched.length > 0 || duplicates.length > 0) + throw new Error( + `${name} has invalid pack keys: ${[ + unmatched.length > 0 ? `not in manifest: ${unmatched.join(', ')}` : '', + duplicates.length > 0 + ? `case-insensitive duplicates: ${duplicates.join(', ')}` + : '' + ] + .filter(Boolean) + .join('; ')}` + ) +} diff --git a/browser_tests/fixtures/customNode/runResult.ts b/browser_tests/fixtures/customNode/runResult.ts new file mode 100644 index 00000000000..507f21abe8c --- /dev/null +++ b/browser_tests/fixtures/customNode/runResult.ts @@ -0,0 +1,139 @@ +type CustomNodeOutcome = + | 'NOT_INSTALLED' + | 'IMPORT_ERROR' + | 'MISSING_NODE' + | 'VALIDATION_FAIL' + | 'EXECUTION_ERROR' + | 'PARTIAL' + | 'TIMEOUT' + | 'PASS' + +export interface ExecutionError { + exceptionMessage?: string + exceptionType?: string + nodeId?: string + nodeType?: string + traceback?: string[] +} + +export type PromptEvent = + | { type: 'execution_start' } + | { type: 'executing'; node: string | null } + | { type: 'execution_cached'; nodes: string[] } + | { type: 'executed'; node: string | null; output?: unknown } + | { type: 'execution_success' } + | { type: 'execution_error'; error: ExecutionError } + | { type: 'execution_interrupted'; error?: ExecutionError } + +export interface RunResult { + outcome: CustomNodeOutcome + executedNodes: string[] + // ui payloads from `executed` events, keyed by node id - proof that data + // reached each output node, not just that execution finished. + outputsByNode: Record + error?: ExecutionError + // Set when queuePrompt THREW client-side (pack JS hooking the queue can + // crash on a graph shape it does not expect); carries the exception text + // so the failing node self-identifies in the report. + clientError?: string +} + +export function describeRunOutcome(result: RunResult): string { + if (!result.error) return result.outcome + const identity = [result.error.nodeType, result.error.exceptionType] + .filter(Boolean) + .join(': ') + const detail = [identity, result.error.exceptionMessage] + .filter(Boolean) + .join(' - ') + return `${result.outcome} (${detail || 'error event carried no details'})` +} + +// A node's output is present for this prompt via one of two disjoint signals: +// `executing` fires solely for nodes that actually ran (execution.py:493), and +// `execution_cached` names, once per prompt, every node the backend served from +// cache instead. Counting only the first makes a verdict track the backend's +// cache temperature - a node that PASSed on a cold cache reads PARTIAL on the +// next run. The `executed` message stays excluded: it replays for cached nodes +// without naming them, so it cannot distinguish the two. +function executedNodesFrom(events: PromptEvent[]): string[] { + const executed = new Set() + for (const event of events) { + if (event.type === 'executing' && event.node !== null) + executed.add(event.node) + if (event.type === 'execution_cached') + for (const node of event.nodes) executed.add(node) + } + return [...executed] +} + +function outputsFrom(events: PromptEvent[]): Record { + const outputs: Record = {} + for (const event of events) { + if (event.type === 'executed' && event.node !== null) + outputs[event.node] = event.output + } + return outputs +} + +export function classifyRun(input: { + events: PromptEvent[] + expectedNodeIds: string[] + proofOutputNodeByExpectedNode?: Record + // All node ids in the queued graph. An error naming a node outside it is a + // stray from another prompt (late websocket delivery, or a duplicate queue + // from the client-flap retry) and must not be pinned on this run. + graphNodeIds?: string[] + timedOut?: boolean +}): RunResult { + const { + events, + expectedNodeIds, + proofOutputNodeByExpectedNode = {}, + graphNodeIds, + timedOut = false + } = input + const executedNodes = executedNodesFrom(events) + const outputsByNode = outputsFrom(events) + + if (timedOut) return { outcome: 'TIMEOUT', executedNodes, outputsByNode } + + const failure = events.find( + ( + event + ): event is Extract< + PromptEvent, + { type: 'execution_error' | 'execution_interrupted' } + > => + (event.type === 'execution_error' || + event.type === 'execution_interrupted') && + (graphNodeIds === undefined || + event.error?.nodeId === undefined || + graphNodeIds.includes(event.error.nodeId)) + ) + if (failure) + return { + outcome: 'EXECUTION_ERROR', + executedNodes, + outputsByNode, + error: failure.error + } + + if (!events.some((event) => event.type === 'execution_success')) + return { outcome: 'TIMEOUT', executedNodes, outputsByNode } + + const ranEveryExpected = expectedNodeIds.every((node) => { + if (executedNodes.includes(node)) return true + const proofNode = proofOutputNodeByExpectedNode[node] + return ( + proofNode !== undefined && + Object.hasOwn(outputsByNode, proofNode) && + outputsByNode[proofNode] != null + ) + }) + return { + outcome: ranEveryExpected ? 'PASS' : 'PARTIAL', + executedNodes, + outputsByNode + } +} diff --git a/browser_tests/fixtures/customNode/tierNodeExclusions.ts b/browser_tests/fixtures/customNode/tierNodeExclusions.ts new file mode 100644 index 00000000000..bcda0f37837 --- /dev/null +++ b/browser_tests/fixtures/customNode/tierNodeExclusions.ts @@ -0,0 +1,102 @@ +export type CustomNodeNodeExclusionTier = + | 'S1' + | 'S2' + | 'S3' + | 'S4' + | 'S5' + | 'S6' + | 'S7' + | 'S8' + +export interface TierNodeExclusionTarget { + identity: string + pack: string +} + +export interface TierNodeExclusion { + identity: string + nodeType: string + pack: string + reason: string + restore: string + ticket: string + tiers: readonly CustomNodeNodeExclusionTier[] +} + +export const CUSTOM_NODE_TIER_NODE_EXCLUSIONS: readonly TierNodeExclusion[] = [ + { + identity: 'comfyui-itools@0.6.8', + nodeType: 'iToolsCropImage', + pack: 'comfyui-itools', + reason: + "the banned 0.6.8 artifact's crop hook captures its preview widget once, then waits without re-reading it; identical runs terminate at either five widgets plus an error or six widgets without one", + restore: + 'publish and pin a non-banned iTools version whose crop hook waits on the live preview widget, then remove this entry', + ticket: + 'https://linear.app/comfyorg/issue/FE-1675/e2e-nodes-tests-fix-itools-crop-lifecycle-race-and-restore-s1-s8', + tiers: ['S1', 'S2', 'S3', 'S4', 'S5', 'S6', 'S7', 'S8'] + } +] + +function exclusionsForPack(pack: string): readonly TierNodeExclusion[] { + const folded = pack.toLowerCase() + return CUSTOM_NODE_TIER_NODE_EXCLUSIONS.filter( + (exclusion) => exclusion.pack.toLowerCase() === folded + ) +} + +export function eligibleNodeTypesForTier( + target: TierNodeExclusionTarget, + tier: CustomNodeNodeExclusionTier, + nodeTypes: readonly string[] +): string[] { + const packExclusions = exclusionsForPack(target.pack) + for (const exclusion of packExclusions) { + if (exclusion.identity !== target.identity) + throw new Error( + `${target.pack}/${exclusion.nodeType} ${exclusion.tiers.join('/')} exclusion is pinned to ${exclusion.identity}, but the manifest now uses ${target.identity}; remove or recalibrate the stale exclusion` + ) + if (!nodeTypes.includes(exclusion.nodeType)) + throw new Error( + `${target.pack}/${exclusion.nodeType} ${exclusion.tiers.join('/')} exclusion names a node that no longer registers; remove the stale exclusion` + ) + } + + const excluded = new Set( + packExclusions + .filter((exclusion) => exclusion.tiers.includes(tier)) + .map((exclusion) => exclusion.nodeType) + ) + return nodeTypes.filter((nodeType) => !excluded.has(nodeType)) +} + +export function tierNodeExclusionProblems( + targets: readonly TierNodeExclusionTarget[] +): string[] { + const targetsByPack = new Map( + targets.map((target) => [target.pack.toLowerCase(), target]) + ) + const seen = new Set() + const problems: string[] = [] + + for (const exclusion of CUSTOM_NODE_TIER_NODE_EXCLUSIONS) { + const target = targetsByPack.get(exclusion.pack.toLowerCase()) + if (!target) { + problems.push( + `${exclusion.pack}/${exclusion.nodeType} is not in the manifest` + ) + continue + } + if (target.identity !== exclusion.identity) + problems.push( + `${exclusion.pack}/${exclusion.nodeType} is pinned to ${exclusion.identity}, but the manifest now uses ${target.identity}` + ) + for (const tier of exclusion.tiers) { + const key = `${exclusion.pack.toLowerCase()}/${exclusion.nodeType}/${tier}` + if (seen.has(key)) problems.push(`${key} is duplicated`) + seen.add(key) + } + } + + return problems +} diff --git a/browser_tests/fixtures/customNode/typePairing.ts b/browser_tests/fixtures/customNode/typePairing.ts new file mode 100644 index 00000000000..d6eaeccd9b1 --- /dev/null +++ b/browser_tests/fixtures/customNode/typePairing.ts @@ -0,0 +1,378 @@ +// Type-driven pairing generator for the connectivity (contract) tier. +// Wildcard `*` slots are excluded from pairing: LiteGraph.isValidConnection +// short-circuits on `*` before the real type compare, so a wildcard link +// proves reachability, not type interop. + +export interface RawNodeDef { + input?: { + required?: Record + optional?: Record + } + output?: unknown[] + // object_info repeats RETURN_TYPES here when a node declares no + // RETURN_NAMES, so a combo output's entry is its option array. + output_name?: unknown[] + python_module?: string +} + +interface NormalizedSlot { + name: string + type: string + // COMBO slots: the literal option list, for same-vocabulary pairing. + comboOptions?: unknown[] +} + +export interface NormalizedNode { + type: string + pack: string + inputs: NormalizedSlot[] + outputs: NormalizedSlot[] + // Slots the corpus cannot address: no recognizable type (slotTypeOf null) + // or no string name on the instance (outputSlotName null). Recorded so a + // schema change can never silently shrink the corpus. + unknownSlots?: string[] +} + +interface SlotRef { + nodeType: string + pack: string + slotName: string + slotType: string +} + +export interface PlannedPair { + producer: SlotRef + consumer: SlotRef +} + +export interface PairingPlan { + pairs: PlannedPair[] + requiredPairIssues: string[] + // No compatible partner in the loaded corpus: a health signal, not a failure. + orphans: Array + // `*` / empty-typed slots, excluded by design (false confidence). + wildcards: Array + // COMBO slots with no same-vocabulary partner in the corpus, excluded: + // isValidConnection only compares the string COMBO while each slot carries + // its own option set, so pairing across different vocabularies proves + // nothing (a checkpoint dropdown would "connect" to a scheduler dropdown). + // Combos whose option lists match exactly ARE paired like any other type. + combos: Array + // Slots dropped at normalize time because their raw spec had no + // recognizable type or name. The sweep fails if this is non-empty. + unknownShapes: string[] +} + +// Extends the shared outcome taxonomy (runResult.ts); ORPHAN_TYPE is a +// plan-time skip so it never reaches the executor. +// WIDGET_ONLY_ON_INSTANCE: the pack's own frontend JS rebuilt a declared +// input as a widget-only control, so there is no socket to wire. The sweep +// fails unless applicability is made explicit at its call site. +export type ConnectivityOutcome = + | 'PASS' + | 'CONNECT_REJECTED' + | 'DYNAMIC_SLOT_CLEANUP_STALLED' + | 'ROUNDTRIP_LOST' + | 'SLOT_CONTRACT_MISMATCH' + | 'THREW' + | 'WIDGET_ONLY_ON_INSTANCE' + +export function packOf(pythonModule: string | undefined): string { + if (pythonModule?.startsWith('custom_nodes.')) + return pythonModule.slice('custom_nodes.'.length) + return 'core' +} + +function isWildcard(type: string): boolean { + return type === '' || type === '*' +} + +// COMBO list literals are arrays; their connectable socket type is COMBO. +function slotTypeOf(rawType: unknown): string | null { + if (Array.isArray(rawType)) return 'COMBO' + return typeof rawType === 'string' ? rawType : null +} + +// Faithful mirror of production naming (schemas/nodeDef/migration.ts): +// `output_name[index] || output_`, uncoerced - so a truthy non-string +// entry names the live slot something no string lookup can match (null here). +function outputSlotName(rawName: unknown, index: number): string | null { + if (!rawName) return `output_${index}` + return typeof rawName === 'string' ? rawName : null +} + +function inputSlots( + entries: Record | undefined, + unknown: string[] +): NormalizedSlot[] { + if (!entries) return [] + const slots: NormalizedSlot[] = [] + for (const [name, spec] of Object.entries(entries)) { + const specArray = Array.isArray(spec) ? spec : [spec] + const type = slotTypeOf(specArray[0]) + if (type === null) { + unknown.push(name) + continue + } + const opts = specArray[1] as + | { socketless?: boolean; options?: unknown } + | undefined + // socketless = widget only, no slot: not connectable, out of the matrix. + if (opts?.socketless) continue + if (type === 'COMBO') { + // Raw defs carry the option list as the type literal; the frontend's + // transformed defs use the string 'COMBO' with options in the opts. + const options = Array.isArray(specArray[0]) + ? (specArray[0] as unknown[]) + : Array.isArray(opts?.options) + ? opts.options + : undefined + slots.push({ name, type, comboOptions: options }) + continue + } + slots.push({ name, type }) + } + return slots +} + +export function normalizeNodeDefs( + defs: Record +): NormalizedNode[] { + return Object.entries(defs).map(([type, def]) => { + const unknown: string[] = [] + const node: NormalizedNode = { + type, + pack: packOf(def.python_module), + inputs: [ + ...inputSlots(def.input?.required, unknown), + ...inputSlots(def.input?.optional, unknown) + ], + outputs: (def.output ?? []).flatMap((rawType, index) => { + const slotType = slotTypeOf(rawType) + if (slotType === null) { + unknown.push(`output[${index}]`) + return [] + } + const name = outputSlotName(def.output_name?.[index], index) + if (name === null) { + unknown.push(`output[${index}].name`) + return [] + } + const slot: NormalizedSlot = { name, type: slotType } + if (slotType === 'COMBO') slot.comboOptions = rawType as unknown[] + return [slot] + }) + } + if (unknown.length > 0) node.unknownSlots = unknown + return node + }) +} + +// Faithful mirror of LiteGraph.isValidConnection (LiteGraphGlobal.ts): +// wildcard/empty always match, comparison is case-insensitive, comma-unions +// match if any member pair matches. The live sweep still connects through the +// REAL validator, so any drift here surfaces as CONNECT_REJECTED, not a +// silent false green. +export function isTypeCompatible(a: string, b: string): boolean { + if (isWildcard(a) || isWildcard(b)) return true + const typeA = a.toLowerCase() + const typeB = b.toLowerCase() + if (typeA === typeB) return true + if (!typeA.includes(',') && !typeB.includes(',')) return false + return typeA + .split(',') + .some((memberA) => + typeB.split(',').some((memberB) => isTypeCompatible(memberA, memberB)) + ) +} + +function slotRef(node: NormalizedNode, slot: NormalizedSlot): SlotRef { + return { + nodeType: node.type, + pack: node.pack, + slotName: slot.name, + slotType: slot.type + } +} + +// One representative compatible edge per slot, deterministically the first +// partner in (nodeType, slotName) order. This bounds cost to O(slots) but +// does NOT prove every pair; a full cross-product is an opt-in deep mode. +export function planPairs( + all: NormalizedNode[], + corpusTypes: string[], + requiredPairKeys: string[] = [], + knownNodeTypes: ReadonlySet = new Set(all.map((node) => node.type)) +): PairingPlan { + const sorted = [...all].sort((a, b) => a.type.localeCompare(b.type)) + const pairable = (slot: NormalizedSlot) => + !isWildcard(slot.type) && slot.type !== 'COMBO' + const producers: Array = sorted.flatMap((node) => + node.outputs.filter(pairable).map((slot) => slotRef(node, slot)) + ) + const consumers: Array = sorted.flatMap((node) => + node.inputs.filter(pairable).map((slot) => slotRef(node, slot)) + ) + // COMBO slots pair only on an identical option vocabulary; the string type + // alone would let a checkpoint dropdown "connect" to a scheduler dropdown. + // Vocabulary equality is a SET comparison: a wired input bypasses its own + // widget, so menu order and the options[0] default do not participate in + // the wire contract - only membership does (backend validation checks + // "value in options"). Values still compare as exact strings. + const vocabOf = (slot: NormalizedSlot) => + JSON.stringify( + (slot.comboOptions ?? []).map((option) => JSON.stringify(option)).sort() + ) + // A combo whose option list is unknown (transformed defs without an + // options array) must never pair - a blind match would wire dropdowns + // with no vocabulary evidence at all. + const comboProducers = sorted.flatMap((node) => + node.outputs + .filter( + (slot) => slot.type === 'COMBO' && Array.isArray(slot.comboOptions) + ) + .map((slot) => ({ ref: slotRef(node, slot), vocab: vocabOf(slot) })) + ) + const comboConsumers = sorted.flatMap((node) => + node.inputs + .filter( + (slot) => slot.type === 'COMBO' && Array.isArray(slot.comboOptions) + ) + .map((slot) => ({ ref: slotRef(node, slot), vocab: vocabOf(slot) })) + ) + + const corpus = all.filter((node) => corpusTypes.includes(node.type)) + const plan: PairingPlan = { + pairs: [], + requiredPairIssues: [], + orphans: [], + wildcards: [], + combos: [], + unknownShapes: corpus.flatMap((node) => + (node.unknownSlots ?? []).map((slot) => `${node.type}.${slot}`) + ) + } + const seen = new Set() + const addPair = (producer: SlotRef, consumer: SlotRef) => { + const key = `${producer.nodeType}.${producer.slotName}->${consumer.nodeType}.${consumer.slotName}` + if (seen.has(key)) return + seen.add(key) + plan.pairs.push({ producer, consumer }) + } + + for (const node of corpus) { + for (const slot of node.inputs) { + if (isWildcard(slot.type)) { + plan.wildcards.push({ ...slotRef(node, slot), dir: 'in' }) + continue + } + if (slot.type === 'COMBO') { + const producer = Array.isArray(slot.comboOptions) + ? comboProducers.find( + (candidate) => candidate.vocab === vocabOf(slot) + ) + : undefined + if (producer) addPair(producer.ref, slotRef(node, slot)) + else plan.combos.push({ ...slotRef(node, slot), dir: 'in' }) + continue + } + const producer = producers.find((candidate) => + isTypeCompatible(candidate.slotType, slot.type) + ) + if (producer) addPair(producer, slotRef(node, slot)) + else plan.orphans.push({ ...slotRef(node, slot), dir: 'in' }) + } + for (const slot of node.outputs) { + if (isWildcard(slot.type)) { + plan.wildcards.push({ ...slotRef(node, slot), dir: 'out' }) + continue + } + if (slot.type === 'COMBO') { + const consumer = Array.isArray(slot.comboOptions) + ? comboConsumers.find( + (candidate) => candidate.vocab === vocabOf(slot) + ) + : undefined + if (consumer) addPair(slotRef(node, slot), consumer.ref) + else plan.combos.push({ ...slotRef(node, slot), dir: 'out' }) + continue + } + const consumer = consumers.find((candidate) => + isTypeCompatible(slot.type, candidate.slotType) + ) + if (consumer) addPair(slotRef(node, slot), consumer) + else plan.orphans.push({ ...slotRef(node, slot), dir: 'out' }) + } + } + + const corpusTypeSet = new Set(corpusTypes) + for (const key of requiredPairKeys) { + const sides = key.split(' -> ') + if (sides.length !== 2) { + plan.requiredPairIssues.push(`${key}: invalid pair key`) + continue + } + const splitSide = (side: string) => { + const separator = side.lastIndexOf('.') + return separator > 0 && separator < side.length - 1 + ? [side.slice(0, separator), side.slice(separator + 1)] + : null + } + const producerSide = splitSide(sides[0]) + const consumerSide = splitSide(sides[1]) + if (!producerSide || !consumerSide) { + plan.requiredPairIssues.push(`${key}: invalid pair endpoint`) + continue + } + const producerNode = all.find((node) => node.type === producerSide[0]) + const consumerNode = all.find((node) => node.type === consumerSide[0]) + const unknownEndpoints = [ + !producerNode ? producerSide[0] : null, + !consumerNode ? consumerSide[0] : null + ].filter( + (nodeType): nodeType is string => + nodeType !== null && !knownNodeTypes.has(nodeType) + ) + if (unknownEndpoints.length > 0) { + plan.requiredPairIssues.push( + `${key}: unknown node type(s): ${unknownEndpoints.join(', ')}` + ) + continue + } + if (!producerNode || !consumerNode) continue + if ( + !corpusTypeSet.has(producerNode.type) && + !corpusTypeSet.has(consumerNode.type) + ) + continue + const producerSlot = producerNode.outputs.find( + (slot) => slot.name === producerSide[1] + ) + const consumerSlot = consumerNode.inputs.find( + (slot) => slot.name === consumerSide[1] + ) + if (!producerSlot || !consumerSlot) { + plan.requiredPairIssues.push(`${key}: declared slot is no longer present`) + continue + } + const compatible = + producerSlot.type === 'COMBO' || consumerSlot.type === 'COMBO' + ? producerSlot.type === 'COMBO' && + consumerSlot.type === 'COMBO' && + Array.isArray(producerSlot.comboOptions) && + Array.isArray(consumerSlot.comboOptions) && + vocabOf(producerSlot) === vocabOf(consumerSlot) + : isTypeCompatible(producerSlot.type, consumerSlot.type) + if (!compatible) { + plan.requiredPairIssues.push( + `${key}: declared slot types are incompatible` + ) + continue + } + addPair( + slotRef(producerNode, producerSlot), + slotRef(consumerNode, consumerSlot) + ) + } + return plan +} diff --git a/browser_tests/fixtures/customNode/valueDrift.ts b/browser_tests/fixtures/customNode/valueDrift.ts new file mode 100644 index 00000000000..410a0cefc5c --- /dev/null +++ b/browser_tests/fixtures/customNode/valueDrift.ts @@ -0,0 +1,284 @@ +// Exact serialized indices changed by the artifact-proven pack mechanisms. +// Unlisted indices and renderers without matching evidence remain strict. +export const ROUNDTRIP_VALUE_ALLOWED_INDICES_LITEGRAPH: Record< + string, + Record +> = { + 'ComfyUI_Fill-Nodes': { + FL_ColorPicker: '3,4,5,6', + FL_ReplaceColor: '5,6,7,8,9,10,11,12' + }, + 'ComfyUI-KJNodes': { + SplineEditor: '1' + }, + 'ComfyUI-LTXVideo': { + LTXVSparseTrackEditor: '1' + }, + 'WhatDreamsCost-ComfyUI': { + LoadAudioUI: '2,3,5', + LTXDirector: '3,4,5,7' + }, + 'comfyui-itools': { + iToolsRegexNode: '0' + } +} + +export const ROUNDTRIP_VALUE_ALLOWED_INDICES_VUE: Record< + string, + Record +> = { + 'ComfyUI_Fill-Nodes': { + FL_ColorPicker: '3,4,5,6', + FL_ReplaceColor: '5,6,7,8,9,10,11,12' + }, + 'ComfyUI-KJNodes': { + SplineEditor: '1' + }, + 'ComfyUI-LTXVideo': { + LTXVSparseTrackEditor: '1' + }, + 'WhatDreamsCost-ComfyUI': { + LoadAudioUI: '2,3,5', + LTXDirector: '3,4,5,7' + }, + 'comfyui-itools': { + iToolsRegexNode: '0' + } +} + +const VHS_ROUNDTRIP_VALUE_KEYS = { + VHS_LoadAudioUpload: 'choose audio to upload', + VHS_LoadImages: 'choose folder to upload', + VHS_LoadVideo: 'choose video to upload', + VHS_LoadVideoFFmpeg: 'choose video to upload', + VHS_VAEDecodeBatched: 'per_batch', + VHS_VAEEncodeBatched: 'per_batch' +} + +export const ROUNDTRIP_VALUE_ALLOWED_KEYS_LITEGRAPH = { + 'ComfyUI-VideoHelperSuite': VHS_ROUNDTRIP_VALUE_KEYS +} + +export const ROUNDTRIP_VALUE_ALLOWED_KEYS_VUE = { + 'ComfyUI-VideoHelperSuite': VHS_ROUNDTRIP_VALUE_KEYS +} + +export type RoundtripInitializationSignal = + | { property: string; predicate: 'defined' } + | { property: string; predicate: 'equals'; value: unknown } + | { predicate: 'widget-value'; value: unknown; widget: string } + | { predicate: 'widget-count'; value: number } + | { predicate: 'minimum-widget-count'; value: number } + +export const ROUNDTRIP_INITIALIZATION_SIGNALS: Record< + string, + Record +> = { + 'ComfyUI-KJNodes': { + ImageTransformKJ: { + predicate: 'widget-value', + value: '{"fillColor":"#000000"}', + widget: 'bboxes' + } + }, + 'WhatDreamsCost-ComfyUI': { + LoadAudioUI: { + property: '_initializing', + predicate: 'equals', + value: false + } + }, + 'comfyui-sam3': { + SAM3VideoSegmentation: { + property: '_hiddenInputs', + predicate: 'defined' + } + }, + 'comfyui-itools': { + iToolsPaintNode: { + predicate: 'widget-count', + value: 33 + } + } +} + +export function initializationSignalsForTypes( + signals: Record, + types: readonly string[] +): Record { + const typeSet = new Set(types) + return Object.fromEntries( + Object.entries(signals).filter(([type]) => typeSet.has(type)) + ) +} + +function expectedInitializationValue( + signal: RoundtripInitializationSignal +): string { + if (signal.predicate === 'defined') return 'defined' + if (signal.predicate === 'minimum-widget-count') + return `>= ${signal.value} widgets` + if (signal.predicate === 'widget-count') return `${signal.value} widgets` + if (signal.predicate === 'widget-value') + return `${signal.widget} = ${JSON.stringify(signal.value)}` + return JSON.stringify(signal.value) +} + +// Each pending entry carries what it wanted and what it saw. The poll that +// consumes this reports only the returned strings on timeout, so a bare node +// name costs a CI round trip to learn the one number that identifies whether +// the pack is slow or the contract is wrong. +export function pendingRoundtripInitializations( + signals: Record, + values: Record, + vueNodesEnabled: boolean +): string[] { + return Object.entries(signals) + .filter(([node, signal]) => { + if (signal.predicate === 'defined') return values[node] === undefined + if (signal.predicate === 'minimum-widget-count') + return typeof values[node] !== 'number' || values[node] < signal.value + if (signal.predicate === 'widget-count') + return !Object.is(values[node], signal.value) + return !Object.is(values[node], signal.value) + }) + .map( + ([node, signal]) => + `${node} (${vueNodesEnabled ? 'vue' : 'litegraph'}: expected ${expectedInitializationValue(signal)}, observed ${JSON.stringify(values[node])})` + ) +} + +export function pendingRestoredPreviewWidgets( + requiredByNode: Record, + observedByNode: Record +): string[] { + return Object.entries(requiredByNode).flatMap(([node, required]) => + required.flatMap((widget) => + observedByNode[node]?.includes(widget) + ? [] + : [ + `${node}: expected ${widget} after reload, observed [${(observedByNode[node] ?? []).join(',')}]` + ] + ) + ) +} + +export const CANVAS_PREVIEW_IMAGE_PATH_PATTERN = + /\.(?:jpe?g|png|webp)(?:\s*\[(?:input|output|temp)\])?\s*$/i + +export function isCanvasPreviewImagePath(value: unknown): value is string { + return ( + typeof value === 'string' && CANVAS_PREVIEW_IMAGE_PATH_PATTERN.test(value) + ) +} + +export interface RoundtripNodeLossExpectation { + reason: string + restore: string +} + +const FL_TIMELINE_NODE_LOSS: RoundtripNodeLossExpectation = { + reason: + 'the pack replaces node.serialize with a timeline-data serializer, so graph reload drops the node', + restore: + 'fix FL_TimeLine to preserve LiteGraph node serialization, then remove this entry when both renderer passes retain it' +} + +export const ROUNDTRIP_NODE_LOSS_EXPECTATIONS_LITEGRAPH: Record< + string, + Record +> = { + 'ComfyUI_Fill-Nodes': { FL_TimeLine: FL_TIMELINE_NODE_LOSS } +} + +export const ROUNDTRIP_NODE_LOSS_EXPECTATIONS_VUE: Record< + string, + Record +> = { + 'ComfyUI_Fill-Nodes': { FL_TimeLine: FL_TIMELINE_NODE_LOSS } +} + +export interface TopologyExpectation { + before: number + after: number + reason: string +} + +export const OUTPUT_TOPOLOGY_EXPECTATIONS_LITEGRAPH: Record< + string, + Record +> = { + 'ComfyUI_Fill-Nodes': { + FL_VideoBatchSplitter: { + before: 20, + after: 4, + reason: 'pack JS trims the declared outputs to output_count on creation' + } + }, + 'WhatDreamsCost-ComfyUI': { + MultiImageLoader: { + before: 51, + after: 1, + reason: 'pack JS trims unloaded image outputs on creation' + } + } +} + +export const OUTPUT_TOPOLOGY_EXPECTATIONS_VUE: Record< + string, + Record +> = { + 'ComfyUI_Fill-Nodes': { + FL_VideoBatchSplitter: { + before: 20, + after: 4, + reason: 'pack JS trims the declared outputs to output_count on creation' + } + }, + 'WhatDreamsCost-ComfyUI': { + MultiImageLoader: { + before: 51, + after: 1, + reason: 'pack JS trims unloaded image outputs on creation' + } + } +} + +export function matchesTopologyExpectation( + expectation: TopologyExpectation | undefined, + before: number, + after: number +): boolean { + if (expectation?.before !== before) return false + return expectation.after === after +} + +export function rendererLedgerFor( + vueNodesEnabled: boolean, + litegraph: T, + vue: T +): T { + return vueNodesEnabled ? vue : litegraph +} + +export function staleValueDriftIndices( + allowed: Record, + observed: Record +): string[] { + return Object.entries(allowed).flatMap(([node, indices]) => + indices + .filter((index) => !observed[node]?.includes(index)) + .map((index) => `${node}[${index}]`) + ) +} + +export function staleValueDriftKeys( + allowed: Record, + observed: Record +): string[] { + return Object.entries(allowed).flatMap(([node, keys]) => + keys + .filter((key) => !observed[node]?.includes(key)) + .map((key) => `${node}.${key}`) + ) +} diff --git a/browser_tests/fixtures/data/cloud/localExpectations.json b/browser_tests/fixtures/data/cloud/localExpectations.json new file mode 100644 index 00000000000..b424f854237 --- /dev/null +++ b/browser_tests/fixtures/data/cloud/localExpectations.json @@ -0,0 +1,159 @@ +{ + "ComfyUI-AdvancedLivePortrait": { + "expectedNodeCount": 6, + "reason": "stable-shard CI run 32082609026 registered 6 nodes vs Cloud's 4; the manifest's 2 disabled node IDs exactly match the delta" + }, + "ComfyUI-DepthAnythingV3": { + "expectedNodeCount": 15, + "reason": "stable-shard CI run 32082609026 registered 15 nodes vs Cloud's 9; the manifest's 6 disabled node IDs exactly match the delta" + }, + "ComfyUI-DynamiCrafterWrapper": { + "expectedNodeCount": 12, + "reason": "stable-shard CI run 32082609026 registered 12 nodes vs Cloud's 11; the manifest's 1 disabled node ID exactly matches the delta" + }, + "ComfyUI-Image-Filters": { + "expectedNodeCount": 61, + "reason": "stable-shard CI run 32082609026 registered 61 nodes vs Cloud's 60; the manifest's 1 disabled node ID exactly matches the delta" + }, + "ComfyUI-KJNodes": { + "expectedNodeCount": 241, + "reason": "the pinned source declares 228 base and 13 conditional NODE_CONFIG entries; stable-shard CI run 32082609026 registered all 241 vs Cloud's 227" + }, + "ComfyUI-LTXVideo": { + "expectedNodeCount": 76, + "reason": "the pinned source declares 76 unique static and decorated node IDs; stable-shard CI run 32082609026 registered all 76 vs Cloud's 74" + }, + "ComfyUI-Prompt-Combinator": { + "expectedNodeCount": 4, + "reason": "stable-shard CI run 32082609026 registered 4 nodes vs Cloud's 3; the manifest's 1 disabled node ID exactly matches the delta" + }, + "ComfyUI-Qwen-TTS": { + "expectedNodeCount": 8, + "reason": "stable-shard CI run 32082609026 registered 8 nodes vs Cloud's 6; the manifest's 2 disabled node IDs exactly match the delta" + }, + "ComfyUI-QwenVL": { + "expectedNodeCount": 5, + "reason": "stable-shard CI run 32082609026 registered 5 nodes vs Cloud's 3; the manifest's 2 disabled node IDs exactly match the delta" + }, + "ComfyUI-WanAnimatePreprocess": { + "expectedNodeCount": 4, + "reason": "the pinned registry artifact declares exactly 4 NODE_CLASS_MAPPINGS entries; stable-shard CI run 32082609026 registered all 4 vs Cloud's 5" + }, + "ComfyUI-WanVideoWrapper": { + "expectedNodeCount": 142, + "reason": "the pinned artifact declares 146 nodes across required and optional modules; stable-shard CI run 32082609026 registered an exact 142 vs Cloud's 141" + }, + "ComfyUI-test-framework": { + "expectedNodeCount": 9, + "reason": "the pinned commit's get_node_list returns exactly 9 classes; stable-shard CI run 32082609026 registered all 9 vs Cloud's 17" + }, + "ComfyUI_AudioTools": { + "expectedNodeCount": 31, + "reason": "the pinned commit declares exactly 31 NODE_CLASS_MAPPINGS entries; the fixed five-shard CPU environment registers all 31 vs Cloud's 29" + }, + "ComfyUI_Fill-Nodes": { + "expectedNodeCount": 188, + "reason": "stable-shard CI run 32082609026 registered 188 nodes vs Cloud's 110; the manifest's 78 disabled node IDs exactly match the delta" + }, + "ComfyUI_LayerStyle_Advance": { + "expectedNodeCount": 77, + "reason": "the pinned registry artifact declares 77 unique NODE_CLASS_MAPPINGS entries; stable-shard CI run 32082609026 registered all 77 vs Cloud's 46" + }, + "ControlAltAI-Nodes": { + "expectedNodeCount": 22, + "reason": "stable-shard CI run 32082609026 registered 22 nodes vs Cloud's 21; the manifest's 1 disabled node ID exactly matches the delta" + }, + "OneButtonPrompt": { + "expectedNodeCount": 8, + "reason": "stable-shard CI run 32082609026 registered 8 nodes vs Cloud's 7; the manifest's 1 disabled node ID exactly matches the delta" + }, + "RES4LYF": { + "expectedNodeCount": 293, + "reason": "stable-shard CI run 32082609026 registered 293 nodes vs Cloud's 285; the manifest's 8 disabled node IDs exactly match the delta" + }, + "WhatDreamsCost-ComfyUI": { + "expectedNodeCount": 8, + "reason": "stable-shard CI run 32082609026 registered 8 nodes vs Cloud's 6; the manifest's 2 disabled node IDs exactly match the delta" + }, + "basic_data_handling": { + "expectedNodeCount": 309, + "reason": "stable-shard CI run 32082609026 registered 309 nodes vs Cloud's 279; the manifest's 30 disabled node IDs exactly match the delta" + }, + "comfyui-PlyPreview": { + "expectedNodeCount": 4, + "reason": "stable-shard CI run 32082609026 registered 4 nodes vs Cloud's 2; the manifest's 2 disabled node IDs exactly match the delta" + }, + "comfyui-advanced-controlnet": { + "expectedNodeCount": 43, + "reason": "stable-shard CI run 32082609026 registered 43 nodes vs Cloud's 42; the manifest's 1 disabled node ID exactly matches the delta" + }, + "comfyui-animatediff-evolved": { + "expectedNodeCount": 145, + "reason": "stable-shard CI run 32082609026 registered 145 nodes vs Cloud's 144; the manifest's 1 disabled node ID exactly matches the delta" + }, + "comfyui-florence2": { + "expectedNodeCount": 4, + "reason": "stable-shard CI run 32082609026 registered 4 nodes vs Cloud's 3; the manifest's 1 disabled node ID exactly matches the delta" + }, + "comfyui-frame-interpolation": { + "expectedNodeCount": 14, + "reason": "the pinned registry artifact declares exactly 14 NODE_CLASS_MAPPINGS entries; stable-shard CI run 32082609026 registered all 14 vs Cloud's 11" + }, + "comfyui-impact-pack": { + "expectedExtensions": ["Comfy.Impack"], + "expectedNodeCount": 197, + "reason": "stable-shard CI run 32082609026 registered 197 nodes vs Cloud's 185, matching 12 disabled IDs, and registered the locally served Comfy.Impack extension" + }, + "comfyui-itools": { + "expectedNodeCount": 22, + "reason": "stable-shard CI run 32082609026 registered 22 nodes vs Cloud's 14; the manifest's 8 disabled node IDs exactly match the delta" + }, + "comfyui-layerdiffuse": { + "expectedNodeCount": 8, + "reason": "stable-shard CI run 32082609026 registered 8 nodes vs Cloud's 6; the manifest's 2 disabled node IDs exactly match the delta" + }, + "comfyui-logic": { + "expectedNodeCount": 7, + "reason": "stable-shard CI run 32082609026 registered 7 nodes vs Cloud's 6; the manifest's 1 disabled node ID exactly matches the delta" + }, + "comfyui-qwenmultiangle": { + "expectedNodeCount": 1, + "reason": "the pinned registry artifact's get_node_list returns exactly 1 class; stable-shard CI run 32082609026 registered it vs Cloud's 2" + }, + "comfyui-rmbg": { + "expectedNodeCount": 42, + "reason": "the pinned artifact declares 43 nodes, but its SAM3Segment module requires unavailable Triton on this CPU shard; the other 42 register exactly vs Cloud's 35" + }, + "comfyui-various": { + "expectedNodeCount": 68, + "reason": "stable-shard CI run 32082609026 registered 68 nodes vs Cloud's 55; the manifest's 13 disabled node IDs exactly match the delta" + }, + "comfyui-videohelpersuite": { + "expectedNodeCount": 40, + "reason": "stable-shard CI run 32082609026 registered 40 nodes vs Cloud's 32; the manifest's 8 disabled node IDs exactly match the delta" + }, + "comfyui_ipadapter_plus": { + "expectedNodeCount": 30, + "reason": "stable-shard CI run 32082609026 registered 30 nodes vs Cloud's 29; the manifest's 1 disabled node ID exactly matches the delta" + }, + "comfyui_layerstyle": { + "expectedNodeCount": 165, + "reason": "the pinned registry artifact declares 165 unique NODE_CLASS_MAPPINGS entries; stable-shard CI run 32082609026 registered all 165 vs Cloud's 162" + }, + "comfyui_steudio": { + "expectedNodeCount": 10, + "reason": "stable-shard CI run 32082609026 registered 10 nodes vs Cloud's 8; the manifest's 2 disabled node IDs exactly match the delta" + }, + "masquerade-nodes-comfyui": { + "expectedNodeCount": 22, + "reason": "stable-shard CI run 32082609026 registered 22 nodes vs Cloud's 20; the manifest's 2 disabled node IDs exactly match the delta" + }, + "wanvaceadvanced": { + "expectedNodeCount": 13, + "reason": "stable-shard CI run 32082609026 registered 13 nodes vs Cloud's 11; the manifest's 2 disabled node IDs exactly match the delta" + }, + "was-node-suite-comfyui": { + "expectedNodeCount": 220, + "reason": "stable-shard CI run 32082609026 registered 220 nodes vs Cloud's 183; the manifest's 37 disabled node IDs exactly match the delta" + } +} diff --git a/browser_tests/fixtures/data/cloud/packQuarantine.json b/browser_tests/fixtures/data/cloud/packQuarantine.json new file mode 100644 index 00000000000..8a0c4a1cd4c --- /dev/null +++ b/browser_tests/fixtures/data/cloud/packQuarantine.json @@ -0,0 +1,31 @@ +{ + "ComfyUI-LivePortraitKJ": { + "class": "unfetchable-ref", + "evidence": "deployRef @4d9dc6205b79e50f81c0e6d245a2e0e5e53d5ecd returns HTTP 422 'No commit found'; a full clone across all 6 branches and 134 commits does not contain it. The first 12 characters match the repository's real HEAD (4d9dc6205b793ffd0fb319816136d9b8c0dbfdff) and then diverge, which is a short sha expanded wrongly rather than a force-push.", + "reason": "supported_nodes.yaml pins a commit that does not exist upstream", + "since": "2026-08-14", + "upstreamFix": "correct the sha in supported_nodes.yaml to 4d9dc6205b793ffd0fb319816136d9b8c0dbfdff" + }, + "ComfyUI-SeedVR2_VideoUpscaler": { + "class": "requires-gpu-runner", + "evidence": "The suite reports \"not installed on this backend\": the files are on disk and pip succeeded, but ComfyUI never registers the pack because its import does not resolve without the hardware. A pack contributing zero nodes cannot be expressed as a node-count baseline.", + "reason": "installs cleanly but registers zero nodes on a CPU runner - the pack imports ensure_triton_compat, and Triton is GPU-only", + "since": "2026-08-15", + "upstreamFix": "add a GPU-backed shard and restore this pack to the tested population" + }, + "comfyui_nvidia_rtx_nodes": { + "class": "requires-gpu-runner", + "evidence": "The suite reports \"not installed on this backend\": the files are on disk and pip succeeded, but ComfyUI never registers the pack because its import does not resolve without the hardware. A pack contributing zero nodes cannot be expressed as a node-count baseline.", + "reason": "installs cleanly but registers zero nodes on a CPU runner - the pack targets NVIDIA RTX hardware", + "since": "2026-08-15", + "upstreamFix": "add a GPU-backed shard and restore this pack to the tested population" + }, + "radiance": { + "class": "unsatisfiable-requirement", + "evidence": "PyPI's Imath package tops out at 0.0.2, so pip cannot resolve it. The pin appears to name the C++ Imath library version. OpenEXR>=3.2.0, which this pack already requires on the line above, ships Imath.py inside its wheel - so the requirement is redundant, not merely unsatisfiable.", + "failurePattern": "Imath>=3.1.0", + "reason": "requirements.txt pins Imath>=3.1.0, which does not exist on PyPI", + "since": "2026-08-14", + "upstreamFix": "remove the Imath line from fxtdstudios/radiance requirements.txt" + } +} diff --git a/browser_tests/fixtures/data/cloud/supported_nodes.yaml b/browser_tests/fixtures/data/cloud/supported_nodes.yaml new file mode 100644 index 00000000000..0f35a5e697e --- /dev/null +++ b/browser_tests/fixtures/data/cloud/supported_nodes.yaml @@ -0,0 +1,1124 @@ +# Comfy Complete - Supported Custom Nodes +# Source: https://github.com/Comfy-Org/cloud/blob/299167cff712d50d659b8e44d1efa60f4d4102bb/comfy-complete/supported_nodes.yaml +# Imported: 2026-08-14 +# This file defines the custom node packs that are part of the Comfy Complete distribution. +# Each node pack is pinned to a specific version to ensure compatibility. + +# Available labels - all labels used in node_packs must be declared here +labels: + - ReadsArbitraryFile # Node can read files from arbitrary paths + - CreatesLargeOutputs # Node may create unusually large outputs + - DisabledOnCloud # Explicitly disabled for cloud deployments + - WritesToDisk # Node writes to the filesystem + - NetworkAccess # Node makes network requests + - Stateful # Node persists user-specific data between runs + - HasCustomEndpoints # Node registers custom HTTP server routes + - RequiresExternalAPI # Node requires external API keys (OpenAI, etc.) + - PathParsing # Node parses, manipulates, or exposes filesystem path information + - DuplicateOfCoreNode # Node duplicates functionality available in a core ComfyUI node + +node_packs: + - name: core + node_labels: + LoadImageDataSetFromFolder: + - ReadsArbitraryFile + LoadImageTextDataSetFromFolder: + - ReadsArbitraryFile + LoadTrainingDataset: + - ReadsArbitraryFile + SaveImageDataSetToFolder: + - WritesToDisk + SaveImageTextDataSetToFolder: + - WritesToDisk + SaveLoRA: + - CreatesLargeOutputs + SaveTrainingDataset: + - WritesToDisk + CheckpointSave: + - CreatesLargeOutputs + ImageOnlyCheckpointSave: + - CreatesLargeOutputs + ModelSave: + - CreatesLargeOutputs + ModelSaveKJ: + - CreatesLargeOutputs + LoraSave: + - WritesToDisk + CLIPSave: + - WritesToDisk + VAESave: + - WritesToDisk + + - name: https://github.com/kijai/ComfyUI-KJNodes@377ed49f540da1c5e419cd2fc494ccc79543d3c4 # v1.4.5 (nightly) + node_labels: + CameraPoseVisualizer: + - ReadsArbitraryFile + LoadVideosFromFolder: + - ReadsArbitraryFile + LoadImagesFromFolderKJ: + - ReadsArbitraryFile + SaveImageKJ: + - WritesToDisk + SaveStringKJ: + - WritesToDisk + ModelSaveKJ: + - WritesToDisk + StartRecordCUDAMemoryHistory: + - Stateful + EndRecordCUDAMemoryHistory: + - Stateful + - WritesToDisk + VisualizeCUDAMemoryHistory: + - WritesToDisk + WebcamCaptureCV2: + - DisabledOnCloud + ScreencapStream: + - DisabledOnCloud + PlaySoundKJ: + - ReadsArbitraryFile + GGUFLoaderKJ: + - DisabledOnCloud + + - name: https://github.com/kijai/ComfyUI-KwaiKolorsWrapper@6fc1cd9d20bb7537facf180e5494b486b9710e24 + + - name: comfyui_controlnet_aux + version: '1.1.5' + + - name: comfyui-animatediff-evolved + version: '1.5.6' + node_labels: + ADE_LoadCameraPosesFromPath: + - ReadsArbitraryFile + + - name: comfyui-videohelpersuite + version: '1.7.9' + node_labels: + VHS_LoadVideoPath: + - ReadsArbitraryFile + VHS_LoadVideoFFmpegPath: + - ReadsArbitraryFile + VHS_LoadImagesPath: + - ReadsArbitraryFile + VHS_LoadImagePath: + - ReadsArbitraryFile + VHS_SelectFilename: + - ReadsArbitraryFile + VHS_LoadImages: + - ReadsArbitraryFile + VHS_LoadAudio: + - ReadsArbitraryFile + VHS_BatchManager: + - DisabledOnCloud + + - name: comfyui_ultimatesdupscale + version: '1.6.2' + + - name: audio-separation-nodes-comfyui + version: '1.5.0' + + - name: ComfyUI-WanVideoWrapper + version: '1.4.7' + node_labels: + WanVideoVRAMManagement: + - DisabledOnCloud + WanVideoSetBlockSwap: + - DisabledOnCloud + WanVideoBlockSwap: + - DisabledOnCloud + WanVideoLoraSelectByName: + - DisabledOnCloud + DownloadAndLoadNLFModel: + - NetworkAccess + - DisabledOnCloud + + - name: 'https://github.com/kijai/ComfyUI-DynamiCrafterWrapper@d312c62982f1582b7955444d3a22ac948fb9c603' + version: '' + node_labels: + DownloadAndLoadCLIPModel: + - DisabledOnCloud + + - name: comfyui-enricos-nodes + version: '3.1.6' + + - name: comfyui-florence2 + version: '1.0.7' + node_labels: + Florence2ModelLoader: + - DisabledOnCloud + + - name: comfyui-inpaint-cropandstitch + version: '2.1.8' + web_directory: js + + - name: comfyui-inpaint-nodes + version: '1.4.0' + + - name: comfyui_essentials + version: '1.1.0' + + - name: comfyui-supir + version: '1.0.4' + + - name: comfyui-segment-anything-2 + version: '1.0.2' + + - name: 'https://github.com/Comfy-Org/CustomNodeComfyMath@c01177221c31b8e5fbc062778fc8254aeb541638' + version: '' + + - name: comfyui-impact-pack + version: '8.28.2' + node_labels: + PreviewBridge: + - DisabledOnCloud + PreviewBridgeLatent: + - DisabledOnCloud + ImageReceiver: + - DisabledOnCloud + ImageSender: + - DisabledOnCloud + MaskRectAreaAdvanced: + - DisabledOnCloud + MaskRectArea: + - DisabledOnCloud + ImpactSEGSPicker: + - DisabledOnCloud + LatentReceiver: + - DisabledOnCloud + ImpactValueReceiver: + - DisabledOnCloud + ImpactQueueTrigger: + - DisabledOnCloud + ImpactQueueTriggerCountdown: + - DisabledOnCloud + ImpactSetWidgetValue: + - DisabledOnCloud + + - name: comfyui_ipadapter_plus + version: '2.0.0' + node_labels: + IPAdapterSaveEmbeds: + - WritesToDisk + - DisabledOnCloud + + - name: ComfyUI-WanAnimatePreprocess + version: '1.0.1' + + - name: 'https://github.com/lum3on/ComfyUI_AudioTools@c816d0a1629c05a0f463ddb77ea96ada69875d32' + version: '' + node_labels: + AudioLoadBatch: + - ReadsArbitraryFile + AudioGetFromList: + - DisabledOnCloud + + - name: comfyui-fl-path-animator + version: '1.0.6' + + - name: 'https://github.com/kijai/ComfyUI-DepthAnythingV2@d505cbca99803fc63327b8305618a23e59a18b42' + version: '' + + - name: 'https://github.com/kijai/ComfyUI-Lotus@dcd5bea7a418717a6e0c27a5f607058cb9c24a5e' + version: '' + + - name: 'https://github.com/AIWarper/ComfyUI-NormalCrafterWrapper@10408381e60f42775a977966e111f1353d86e970' + version: '' + + - name: comfyui-advanced-controlnet + version: '1.5.6' + node_labels: + LoadImagesFromDirectory: + - ReadsArbitraryFile + + - name: comfyui-frame-interpolation + version: '1.0.8' + node_labels: + GMFSS Fortuna VFI: + - NetworkAccess + M2M VFI: + - NetworkAccess + Sepconv VFI: + - NetworkAccess + AMT VFI: + - NetworkAccess + STMFNet VFI: + - NetworkAccess + + - name: comfyui-impact-subpack + version: '1.3.5' + + - name: ComfyUI-SCAIL-Pose + version: '1.0.1' + + # v1.1.4 + - name: 'https://github.com/gseth/ControlAltAI-Nodes@721492b66c9cede8ae23ae10615462ad80cfd061' + version: '' + node_labels: + FluxAttentionControl: + - Stateful + - DisabledOnCloud + + - name: 'https://github.com/ClownsharkBatwing/RES4LYF@119679d8d8d26e6db52757e705488abb6399d7d4' + version: '' + node_labels: + # Important: labels key off object_info class_type (YAML key), not display_name. + # In object_info, class_type `ClownsharKSampler` is the deprecated legacy2 node + # (display_name "Legacy2_ClownsharKSampler"). The non-legacy sampler is + # class_type `ClownsharKSampler_Beta` and is intentionally not disabled here. + ConditioningToBase64: + - DisabledOnCloud + Base64ToConditioning: + - DisabledOnCloud + ClownsharKSamplerOptions: + - DisabledOnCloud + # Formula-string eval path is sourced from options/timestep nodes. + SamplerOptions_TimestepScaling: + - DisabledOnCloud + Sigmas Math1: + - DisabledOnCloud + Sigmas Math3: + - DisabledOnCloud + TextLoadFile: + - ReadsArbitraryFile + UNetSave: + - WritesToDisk + - CreatesLargeOutputs + + - name: comfyui_layerstyle + version: '2.0.32' + web_directory: js + node_labels: + 'LayerUtility: ImageTaggerSave': + - WritesToDisk + 'LayerUtility: ImageTaggerSaveV2': + - WritesToDisk + 'LayerUtility: LoadImagesFromPath': + - ReadsArbitraryFile + 'LayerUtility: PurgeVRAM': + - Stateful + - DisabledOnCloud + 'LayerUtility: PurgeVRAM V2': + - Stateful + - DisabledOnCloud + 'LayerUtility: QueueStop': + - DisabledOnCloud + 'LayerMask: RemBgUltra': + - NetworkAccess + - DisabledOnCloud + 'LayerMask: RmBgUltra V2': + - NetworkAccess + - DisabledOnCloud + 'LayerFilter: LightLeak': + - DisabledOnCloud + + - name: ComfyUI_LayerStyle_Advance + version: '2.0.37' + web_directory: js + node_labels: + 'LayerUtility: SaveImagePlus': + - WritesToDisk + 'LayerUtility: SaveImagePlusV2': + - WritesToDisk + 'LayerUtility: LoadPSD': + - ReadsArbitraryFile + 'LayerUtility: Gemini': + - RequiresExternalAPI + 'LayerUtility: GeminiV2': + - RequiresExternalAPI + 'LayerMask: ObjectDetectorGemini': + - RequiresExternalAPI + 'LayerMask: ObjectDetectorGeminiV2': + - RequiresExternalAPI + 'LayerUtility: GeminiImageEdit': + - RequiresExternalAPI + 'LayerUtility: DeepSeekAPI': + - RequiresExternalAPI + 'LayerUtility: DeepSeekAPIV2': + - RequiresExternalAPI + 'LayerUtility: ZhipuGLM4V': + - RequiresExternalAPI + 'LayerUtility: ZhipuGLM4': + - RequiresExternalAPI + 'LayerUtility: JimengI2IAPI': + - RequiresExternalAPI + 'LayerUtility: PromptEmbellish': + - RequiresExternalAPI + 'LayerUtility: PromptTagger': + - RequiresExternalAPI + 'LayerUtility: LoadJoyCaption2Model': + - DisabledOnCloud + 'LayerUtility: JoyCaption2': + - DisabledOnCloud + 'LayerUtility: JoyCaption2ExtraOptions': + - DisabledOnCloud + 'LayerUtility: JoyCaption2Split': + - DisabledOnCloud + 'LayerUtility: LoadJoyCaptionBeta1Model': + - DisabledOnCloud + 'LayerUtility: JoyCaptionBeta1': + - DisabledOnCloud + 'LayerUtility: JoyCaptionBeta1ExtraOptions': + - DisabledOnCloud + 'LayerUtility: LlamaVision': + - DisabledOnCloud + 'LayerUtility: PhiPrompt': + - DisabledOnCloud + 'LayerUtility: AddBlindWaterMark': + - DisabledOnCloud + 'LayerUtility: ShowBlindWaterMark': + - DisabledOnCloud + 'LayerUtility: ImageRewardFilter': + - DisabledOnCloud + 'LayerFilter: LightLeak': + - DisabledOnCloud + # TODO: Re-enable after merge to main triggers upload-models workflow + # which creates asset DB records. Models already in GCS, preset mappings + # and model_directories.go are correct. Just needs DB asset sync. + 'LayerMask: Florence2Ultra': + - DisabledOnCloud + 'LayerMask: LoadFlorence2Model': + - DisabledOnCloud + 'LayerUtility: Florence2Image2Prompt': + - DisabledOnCloud + 'LayerMask: HumanPartsUltra': + - NetworkAccess + - DisabledOnCloud + + # v1.0.0 + - name: 'https://github.com/filliptm/ComfyUI_Fill-ChatterBox@2f72aca98e4ceda18caceaef91ddf5683f5f093e' + version: '' + web_directory: web + + - name: comfyui-qwenmultiangle + version: '0.6.1' + web_directory: js + + # latest (2025-01-18) + - name: 'https://github.com/lquesada/ComfyUI-Prompt-Combinator@3576ace3f2776918c1046b6674fbe345a46dba2f' + version: '' + node_labels: + PromptCombinatorExportGallery: + - WritesToDisk + + # latest (2025-01-08) + - name: 'https://github.com/lihaoyun6/ComfyUI-FlashVSR_Ultra_Fast@4820b3f02347bddcbbb9a5a85ab7638fe976366e' + version: '' + + # v2.5.24 + - name: 'https://github.com/numz/ComfyUI-SeedVR2_VideoUpscaler@4490bd1f482e026674543386bb2a4d176da245b9' + version: '' + + - name: 'https://github.com/flybirdxx/ComfyUI-Qwen-TTS@5e97d3cce967c842aeee02854725c371c8a101d7' + version: '' + node_labels: + FB_Qwen3TTSSaveVoice: + - WritesToDisk + - DisabledOnCloud + FB_Qwen3TTSLoadSpeaker: + - ReadsArbitraryFile + - DisabledOnCloud + + - name: comfyui-sam3 + version: '0.1.2' + + - name: 'https://github.com/jtydhr88/ComfyUI-UltraShape1@2f8ea9b59bda7528ea2884b0868e205ece342e9e' + version: '' + node_labels: + UltraShapeSaveGLB: + - WritesToDisk + - DisabledOnCloud + UltraShapeMeshSelector: + - DisabledOnCloud + + - name: comfyui-rmbg + version: '3.0.0' + web_directory: web + node_labels: + AILab_LoadImage: + - ReadsArbitraryFile + AILab_LoadImageSimple: + - ReadsArbitraryFile + AILab_LoadImageAdvanced: + - ReadsArbitraryFile + AILab_LoadImageBatch: + - ReadsArbitraryFile + AILab_YoloV8: + - DisabledOnCloud + AILab_YoloV8Adv: + - DisabledOnCloud + Segment: + - DisabledOnCloud + SegmentV2: + - DisabledOnCloud + + - name: ComfyUI-QwenVL + version: '2.1.1' + node_labels: + AILab_QwenVL_GGUF: + - DisabledOnCloud + AILab_QwenVL_GGUF_Advanced: + - DisabledOnCloud + + - name: 'https://github.com/ltdrdata/was-node-suite-comfyui@afeee09ba44e713ec52a413ac6b105fd06b2d356' + version: '' + node_labels: + 'BLIP Analyze Image': + - DisabledOnCloud + 'BLIP Model Loader': + - DisabledOnCloud + 'CLIPSeg Batch Masking': + - DisabledOnCloud + 'CLIPSeg Masking': + - DisabledOnCloud + 'CLIPSeg Model Loader': + - DisabledOnCloud + CLIPSEG2: + - DisabledOnCloud + 'Cache Node': + - DisabledOnCloud + 'Checkpoint Loader': + - DuplicateOfCoreNode + - DisabledOnCloud + 'Checkpoint Loader (Simple)': + - DuplicateOfCoreNode + - DisabledOnCloud + 'Create Morph Image from Path': + - ReadsArbitraryFile + 'Create Grid Image': + - ReadsArbitraryFile + 'Create Video from Path': + - ReadsArbitraryFile + 'Debug Number to Console': + - DisabledOnCloud + 'Dictionary to Console': + - DisabledOnCloud + 'Diffusers Hub Model Down-Loader': + - NetworkAccess + 'Export API': + - DisabledOnCloud + 'Image Bounds to Console': + - DisabledOnCloud + 'Image History Loader': + - ReadsArbitraryFile + 'Image Load': + - ReadsArbitraryFile + 'Image Save': + - WritesToDisk + 'Image Send HTTP': + - NetworkAccess + 'Load Cache': + - DisabledOnCloud + 'Load Image Batch': + - ReadsArbitraryFile + 'Load Text File': + - ReadsArbitraryFile + 'MiDaS Depth Approximation': + - DisabledOnCloud + 'MiDaS Mask Image': + - DisabledOnCloud + 'MiDaS Model Loader': + - DisabledOnCloud + 'SAM Model Loader': + - DisabledOnCloud + 'Save Text File': + - WritesToDisk + 'Text File History Loader': + - ReadsArbitraryFile + 'Text Load Line From File': + - ReadsArbitraryFile + 'Text Random Prompt': + - DisabledOnCloud + 'True Random.org Number Generator': + - NetworkAccess + 'Upscale Model Loader': + - DuplicateOfCoreNode + - DisabledOnCloud + 'Video Dump Frames': + - WritesToDisk + 'Write to GIF': + - WritesToDisk + 'Write to Video': + - WritesToDisk + + - name: comfyui-sharp + version: '0.0.3' + + - name: comfyui-logic + version: '1.0.0' + node_labels: + DebugPrint: + - DisabledOnCloud + + - name: comfyui-PlyPreview + version: '1.1.0' + web_directory: web + node_labels: + PlyPreviewLoadGaussianPLYEnhance: + - ReadsArbitraryFile + PlyPreviewLoadGaussianPLYPathEnhance: + - ReadsArbitraryFile + + - name: https://github.com/kijai/ComfyUI-LivePortraitKJ@4d9dc6205b79e50f81c0e6d245a2e0e5e53d5ecd + + - name: 'https://github.com/fxtdstudios/radiance@f262f47ddfda01ece154bf80c22769b1e4cef795' + version: '' + web_directory: js + node_labels: + RadianceLUTApply: + - DisabledOnCloud + '◎ RadianceDigitalCinemaRead': + - ReadsArbitraryFile + '◎ RadianceNukeBridge': + - NetworkAccess + - WritesToDisk + '◎ RadianceQCExport': + - WritesToDisk + + - name: 'https://github.com/shiimizu/ComfyUI-TiledDiffusion@a155b1bac39147381aeaa52b9be42e545626a44f' + version: '' + + - name: 'https://github.com/spacepxl/ComfyUI-Image-Filters@bbb3fb0045461adf3602faeedaf40af57090d4e2' + version: '' + node_labels: + ModelTest: + - DisabledOnCloud + + - name: 'https://github.com/Comfy-Org/ComfyUI-test-framework@80caae462d3293a2d40c2938de33a5f50eea8333' + version: '' + + - name: ComfyUI-DepthAnythingV3 + version: '0.1.2' + web_directory: web + node_labels: + DA3_SavePointCloud: + - WritesToDisk + DA3_FilterGaussians: + - WritesToDisk + DA3_PreviewPointCloud: + - WritesToDisk + DepthAnythingV3_Streaming: + - WritesToDisk + DA3_DownloadModel: + - NetworkAccess + LoadSALADModel: + # Only consumed by DepthAnythingV3_Streaming (disabled on cloud) + - DisabledOnCloud + + # v1.0.0 + - name: 'https://github.com/PowerHouseMan/ComfyUI-AdvancedLivePortrait@3bba732915e22f18af0d221b9c5c282990181f1b' + version: '' + node_labels: + SaveExpData: + - WritesToDisk + - DisabledOnCloud + LoadExpData: + - ReadsArbitraryFile + - DisabledOnCloud + + # v0.9 + - name: 'https://github.com/BadCafeCode/masquerade-nodes-comfyui@432cb4d146a391b387a0cd25ace824328b5b61cf' + version: '' + node_labels: + 'Mask By Text': + - NetworkAccess + - DisabledOnCloud + 'Create QR Code': + - DisabledOnCloud + + # ComfyQR by coreyryanhanson - QR code generation for generative QR art + - name: 'https://github.com/coreyryanhanson/ComfyQR@e31449c99a6ba8c5c567646659e0a75e4056259f' + version: '' + + # comfyui-various by jamesWalker55 - image ops, primitives, RAFT optical flow, color ops + - name: 'https://github.com/jamesWalker55/comfyui-various@5bd85aaf7616878471469c4ec7e11bbd0cef3bf2' + version: '' + node_labels: + JWImageLoadRGB: + - ReadsArbitraryFile + JWImageLoadRGBA: + - ReadsArbitraryFile + JWLoadImagesFromString: + - ReadsArbitraryFile + JWImageSaveToPath: + - WritesToDisk + JWImageLoadRGBFromClipboard: + - DisabledOnCloud + 'JWImageLoadRGBA From Clipboard': + - DisabledOnCloud + RAFTLoadFlowFromEXRChannels: + - ReadsArbitraryFile + JWLoadImageSequence: + - ReadsArbitraryFile + JWLoadImageSequenceWithStopIndex: + - ReadsArbitraryFile + JWSaveImageSequence: + - WritesToDisk + JWImageLoadRGBIfExists: + - ReadsArbitraryFile + JWLoadAudio: + - ReadsArbitraryFile + JWAudioSaveToPath: + - WritesToDisk + + - name: comfyui-cogvideoxwrapper + version: '1.5.1' + + - name: 'https://github.com/glowcone/comfyui-string-converter@0e1e78f877e4d5fc2b46ace8fa21a0ef70e92f11' + version: '' + + - name: comfyui_face_parsing + version: '1.0.5' + + - name: nodesweet + version: '1.0.1' + + - name: 'https://github.com/AIrjen/OneButtonPrompt@2129c5dfe6a73d4a0cf971c3af6c3cddbd0e97f8' + version: '' + node_labels: + SavePromptToFile: + - WritesToDisk + + # v1.2.2 - SeC (Segment Concept) SOTA video object segmentation + - name: 'https://github.com/9nate-drake/Comfyui-SecNodes@af39e6260bc414873fbb2dc8df355904d1609754' + version: '' + + - name: 'https://github.com/kijai/ComfyUI-MimicMotionWrapper@89d0ed551dbb1e0a14e89a01d42a39e9ed755d9c' + version: '' + + - name: basic_data_handling + version: '1.4.0' + node_labels: + 'Basic data handling: PathLoadStringFile': + - ReadsArbitraryFile + 'Basic data handling: PathLoadImageRGB': + - ReadsArbitraryFile + 'Basic data handling: PathLoadImageRGBA': + - ReadsArbitraryFile + 'Basic data handling: PathLoadMaskFromAlpha': + - ReadsArbitraryFile + 'Basic data handling: PathLoadMaskFromGreyscale': + - ReadsArbitraryFile + 'Basic data handling: PathSaveStringFile': + - WritesToDisk + 'Basic data handling: PathSaveImageRGB': + - WritesToDisk + 'Basic data handling: PathSaveImageRGBA': + - WritesToDisk + 'Basic data handling: PathGlob': + - ReadsArbitraryFile + 'Basic data handling: PathListDir': + - ReadsArbitraryFile + 'Basic data handling: PathGetSize': + - ReadsArbitraryFile + 'Basic data handling: PathExists': + - ReadsArbitraryFile + 'Basic data handling: PathIsFile': + - ReadsArbitraryFile + 'Basic data handling: PathIsDir': + - ReadsArbitraryFile + 'Basic data handling: PathExpandVars': + - PathParsing + 'Basic data handling: PathGetCwd': + - PathParsing + 'Basic data handling: PathAbspath': + - PathParsing + 'Basic data handling: PathBasename': + - PathParsing + 'Basic data handling: PathCommonPrefix': + - PathParsing + 'Basic data handling: PathDirname': + - PathParsing + 'Basic data handling: PathGetExtension': + - PathParsing + 'Basic data handling: PathInputDir': + - PathParsing + 'Basic data handling: PathIsAbsolute': + - PathParsing + 'Basic data handling: PathJoin': + - PathParsing + 'Basic data handling: PathNormalize': + - PathParsing + 'Basic data handling: PathOutputDir': + - PathParsing + 'Basic data handling: PathRelative': + - PathParsing + 'Basic data handling: PathSetExtension': + - PathParsing + 'Basic data handling: PathSplit': + - PathParsing + 'Basic data handling: PathSplitExt': + - PathParsing + + - name: ComfyUI-MelBandRoFormer + version: '1.0.2' + + - name: 'https://github.com/Extraltodeus/Skimmed_CFG@b2cf6e99303f9f310054b888fa23c37da9a4d224' + version: '' + + - name: sd-dynamic-thresholding + version: '1.0.1' + + - name: 'https://github.com/filliptm/ComfyUI_Fill-Nodes@12c5bcded72a7f2c31c3df37516987c5d3691e28' + version: '' + web_directory: web + node_labels: + FL_API_Base64_ImageLoader: + - DisabledOnCloud + FL_CodeNode: + - DisabledOnCloud + - ReadsArbitraryFile + FL_JS: + - DisabledOnCloud + FL_WF_Agent: + - DisabledOnCloud + FL_SystemCheck: + - DisabledOnCloud + - HasCustomEndpoints + FL_Shadertoy: + - DisabledOnCloud + FL_InfiniteZoom: + - DisabledOnCloud + FL_PaperDrawn: + - DisabledOnCloud + FL_ProResVideo: + - DisabledOnCloud + - WritesToDisk + - ReadsArbitraryFile + FL_VideoCut: + - DisabledOnCloud + - WritesToDisk + FL_LoadImage: + - HasCustomEndpoints + - ReadsArbitraryFile + FL_ImagePicker: + - HasCustomEndpoints + - Stateful + FL_TimeLine: + - HasCustomEndpoints + FL_DirectoryCrawl: + - ReadsArbitraryFile + FL_ModelInspector: + - ReadsArbitraryFile + FL_ImageRandomizer: + - ReadsArbitraryFile + FL_NFTGenerator: + - ReadsArbitraryFile + FL_CaptionToCSV: + - ReadsArbitraryFile + FL_LoadCSV: + - ReadsArbitraryFile + FL_WordFrequencyGraph: + - ReadsArbitraryFile + FL_ImageCaptionLayout: + - ReadsArbitraryFile + FL_PDFLoader: + - ReadsArbitraryFile + FL_BulkPDFLoader: + - ReadsArbitraryFile + FL_ZipDirectory: + - ReadsArbitraryFile + FL_SaveImages: + - WritesToDisk + - ReadsArbitraryFile + FL_SaveWebM: + - WritesToDisk + - ReadsArbitraryFile + FL_CaptionSaver_V2: + - WritesToDisk + - ReadsArbitraryFile + FL_MirrorAndAppendCaptions: + - WritesToDisk + - ReadsArbitraryFile + FL_ImageCaptionLayoutPDF: + - WritesToDisk + - ReadsArbitraryFile + FL_ZipSave: + - WritesToDisk + - ReadsArbitraryFile + FL_ImageCaptionSaver: + - WritesToDisk + FL_VideoCaptionSaver: + - WritesToDisk + - CreatesLargeOutputs + FL_SaveCSV: + - WritesToDisk + FL_PDFSaver: + - WritesToDisk + FL_API_ImageSaver: + - WritesToDisk + FL_SaveAndDisplayImage: + - WritesToDisk + 'FL_SaveWebPImage(SaveImage)': + - WritesToDisk + FL_SaveRGBAAnimatedWebP: + - WritesToDisk + FL_FILM: + - NetworkAccess + - WritesToDisk + - Stateful + FL_RIFE: + - NetworkAccess + - WritesToDisk + - Stateful + FL_ClipScanner: + - Stateful + # External API nodes - disabled on cloud (require user API keys, cloud has partner nodes) + FL_OllamaCaptioner: + - DisabledOnCloud + - NetworkAccess + FL_HFHubModelUploader: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_HF_Character: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_HF_UploaderAbsolute: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_HFDatasetDownloader: + - DisabledOnCloud + - NetworkAccess + FL_Hedra_API: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_GoogleDriveDownloader: + - DisabledOnCloud + - NetworkAccess + FL_GoogleDriveImageDownloader: + - DisabledOnCloud + - NetworkAccess + FL_GoogleCloudStorage: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_SendToDiscordWebhook: + - DisabledOnCloud + - NetworkAccess + FL_KartelJobInput: + - DisabledOnCloud + - NetworkAccess + FL_KartelJobOutput: + - DisabledOnCloud + - NetworkAccess + FL_Fal_Gemini_ImageEdit: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_Fal_Kling_AIAvatar: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_Fal_Kontext: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_Fal_Pixverse: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_Fal_Pixverse_LipSync: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_Fal_Pixverse_Transition: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_Fal_SeedVR_Upscale: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_Fal_Seedance_i2v: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_Fal_Seedream_Edit: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_Fal_Sora: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_GeminiImageEditor: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_GeminiImageGenADV: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_GeminiTextAPI: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_GeminiVideoCaptioner: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_PixVerseAPI: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_RunwayAct2: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_RunwayImageAPI: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_VertexGemini25FlashImage: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_Veo3VideoGen: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_Dalle3: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_GPT_Image1: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_GPT_Image1_ADV: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_GPT_Text: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_GPT_Vision: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + FL_SimpleGPTVision: + - DisabledOnCloud + - NetworkAccess + - RequiresExternalAPI + + - name: comfyui_steudio + version: '2.0.5' + node_labels: + 'Display UI': + # Duplicate of core PreviewAny ("Preview as Text") node; does not work on cloud + - DuplicateOfCoreNode + - DisabledOnCloud + 'Load Images into List': + - ReadsArbitraryFile + + - name: 'https://github.com/logtd/ComfyUI-Fluxtapoz@17c71bea20e932945e9c1be0586cfd4b7e51cbf6' + version: '' + + # v1.0.5 + - name: 'https://github.com/lrzjason/ComfyUI-EditUtils@256a2340b161da3f814e5982ca4aa999f474df93' + version: '' + + - name: feedback-sampler + version: '2.0.0' + + - name: vewd + version: '1.7.2' + + - name: comfyui-workflow-prettier + version: '1.0.5' + web_directory: web + + - name: 'https://github.com/Lightricks/ComfyUI-LTXVideo@229437c6b65796d6a7a63ae34be2bd5ba31fa543' + version: '' + node_labels: + GemmaAPITextEncode: + - RequiresExternalAPI + - NetworkAccess + LTXVSaveConditioning: + - WritesToDisk + LTXVHDRDecodePostprocess: + - WritesToDisk + + - name: 'https://github.com/yuvraj108c/ComfyUI-Upscaler-Tensorrt@dfc266538368e4c9a03a296917716a214517ccb5' + version: '' + web_directory: js + + - name: comfyui-layerdiffuse + version: '1.0.4' + node_labels: + # SD1.5 attn_sharing nodes are broken in this custom node: AttentionSharingUnit.forward() does not accept + # transformer_options kwarg added in newer ComfyUI attention API + LayeredDiffusionJointApply: + - DisabledOnCloud + LayeredDiffusionCondJointApply: + - DisabledOnCloud + + - name: comfyui-itools + version: '0.6.8' + web_directory: web + node_labels: + iToolsPromptLoader: + - ReadsArbitraryFile + iToolsPromptSaver: + - WritesToDisk + iToolsLoadImages: + - ReadsArbitraryFile + iToolsLoadRandomImage: + - ReadsArbitraryFile + iToolsPaintNode: + - HasCustomEndpoints + - DisabledOnCloud + iToolsCropImage: + - HasCustomEndpoints + iToolsPromptStyler: + - HasCustomEndpoints + - DisabledOnCloud + iToolsPromptStylerExtra: + - HasCustomEndpoints + - DisabledOnCloud + # iToolsTestNode, iToolsDomNode, iToolsFreeChat, iToolsFreeSchnell are + # conditional nodes (disabled by default in __init__.py settings) and + # never register in object_info — no labels needed + + - name: https://github.com/Liquid-XO/ComfyUI_SchemaNodes@8e5676d89f046e8c28cbe7f2548f4abdaf86ca35 + version: '' # 0.1.1+8 (configurable image save format, remove SCHEMA_FIELD output) + web_directory: js + + - name: comfyui-ic-light-native + version: '1.0.1' + + - name: comfyui_nvidia_rtx_nodes + version: '0.1.2' + + - name: 'https://github.com/kijai/ComfyUI-GIMM-VFI@4c9a3123762af85e7c796e41737da0b70c75d72d' + version: '' + + - name: wanvaceadvanced + version: '1.4.0' + node_labels: + WanVacePhantomDual: + - DisabledOnCloud + WanVacePhantomExperimental: + - DisabledOnCloud + + - name: comfyui-systms-facecomposite + version: '1.0.0' + + - name: bfsnodes + version: '1.0.0' + + - name: https://github.com/WhatDreamsCost/WhatDreamsCost-ComfyUI@faa372880a4410af86f9a856f8617876cd55d17e + version: '' # v1.3.5 + web_directory: js + node_labels: + LoadVideoUI: + - HasCustomEndpoints # Registers /video_ui_custom_view and /video_ui_upload_chunk at module import + - ReadsArbitraryFile # video input accepts absolute paths + MultiImageLoader: + - ReadsArbitraryFile # image_paths multi-line STRING resolves absolute paths + + - name: https://github.com/liconstudio/ComfyUI-Licon-MSR@cc4f26f4eb60fc6f8abd5c4d58facdb0eb6d5470 + version: '' # v1.0.0 diff --git a/browser_tests/fixtures/data/curatedOutputHashes.core.json b/browser_tests/fixtures/data/curatedOutputHashes.core.json new file mode 100644 index 00000000000..77713382919 --- /dev/null +++ b/browser_tests/fixtures/data/curatedOutputHashes.core.json @@ -0,0 +1,30 @@ +{ + "recordedAt": { + "core": "b08e6cf35fac50d3ca8470dffb3f9a1fbb7187d2", + "run": "30329885634" + }, + "schema": 1, + "workflows": { + "ComfyUI-Impact-Pack/impact_primitives_run.json": { + "2": "sha256:20a57b5f7428f99444628a59c6645b468c2a95c407dd0ad03e8ffd94e26b932f", + "4": "sha256:446ddd0dd335f0aad310a8e77ce95e7f7cd2b4e143abbdea9490f3989b1a163a" + }, + "ComfyUI-VideoHelperSuite/vhs_video_pipeline_run.json": { + "3": "sha256:5a7e5136356d50b36f876e4969039119951d31ce9e7d3be451473d626611c546" + }, + "ComfyUI_essentials/essentials_math_display_run.json": { + "2": "sha256:ad67fc5ce0ded16ec82ef6bdedeb00b794864539be3e9604640861be9bc7b5bf" + }, + "ComfyUI-KJNodes/kjnodes_constants_run.json": { + "2": "sha256:20a57b5f7428f99444628a59c6645b468c2a95c407dd0ad03e8ffd94e26b932f", + "4": "sha256:446ddd0dd335f0aad310a8e77ce95e7f7cd2b4e143abbdea9490f3989b1a163a" + }, + "ComfyUI-Custom-Scripts/customscripts_string_show_run.json": { + "1": "sha256:eb9339336de09c1f7710ca1d02a4ee280406cdfec2bf92a82d89b7d28cd50bfb", + "2": "sha256:eb9339336de09c1f7710ca1d02a4ee280406cdfec2bf92a82d89b7d28cd50bfb" + }, + "was-node-suite-comfyui/was_number_text_run.json": { + "4": "sha256:3fe9864fb5ab14a9edc608955e5985d186875b2d7fbf76f27486872113e084d6" + } + } +} diff --git a/browser_tests/fixtures/data/customNodeManifest.cloud.json b/browser_tests/fixtures/data/customNodeManifest.cloud.json new file mode 100644 index 00000000000..16b35295531 --- /dev/null +++ b/browser_tests/fixtures/data/customNodeManifest.cloud.json @@ -0,0 +1,1523 @@ +{ + "source": { + "repository": "https://github.com/Comfy-Org/cloud", + "ref": "299167cff712d50d659b8e44d1efa60f4d4102bb", + "path": "comfy-complete/supported_nodes.yaml", + "importedAt": "2026-08-14" + }, + "coreDisabledNodes": { + "CLIPSave": ["WritesToDisk"], + "CheckpointSave": ["CreatesLargeOutputs"], + "ImageOnlyCheckpointSave": ["CreatesLargeOutputs"], + "LoadImageDataSetFromFolder": ["ReadsArbitraryFile"], + "LoadImageTextDataSetFromFolder": ["ReadsArbitraryFile"], + "LoadTrainingDataset": ["ReadsArbitraryFile"], + "LoraSave": ["WritesToDisk"], + "ModelSave": ["CreatesLargeOutputs"], + "ModelSaveKJ": ["CreatesLargeOutputs"], + "SaveImageDataSetToFolder": ["WritesToDisk"], + "SaveImageTextDataSetToFolder": ["WritesToDisk"], + "SaveLoRA": ["CreatesLargeOutputs"], + "SaveTrainingDataset": ["WritesToDisk"], + "VAESave": ["WritesToDisk"] + }, + "packs": [ + { + "pack": "ComfyQR", + "deployRef": "https://github.com/coreyryanhanson/ComfyQR@e31449c99a6ba8c5c567646659e0a75e4056259f", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["comfy-qr-by-image-size", "comfy-qr-by-module-size"], + "expectedNodeCount": 4, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-AdvancedLivePortrait", + "deployRef": "https://github.com/PowerHouseMan/ComfyUI-AdvancedLivePortrait@3bba732915e22f18af0d221b9c5c282990181f1b", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["AdvancedLivePortrait", "ExpData"], + "expectedNodeCount": 4, + "expectedExtensions": [], + "disabledNodes": { + "LoadExpData": ["DisabledOnCloud", "ReadsArbitraryFile"], + "SaveExpData": ["DisabledOnCloud", "WritesToDisk"] + }, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-DepthAnythingV2", + "deployRef": "https://github.com/kijai/ComfyUI-DepthAnythingV2@d505cbca99803fc63327b8305618a23e59a18b42", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": [ + "DepthAnything_V2", + "DownloadAndLoadDepthAnythingV2Model" + ], + "expectedNodeCount": 2, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-DepthAnythingV3", + "deployRef": "ComfyUI-DepthAnythingV3@0.1.2", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["DA3_CreateCameraParams", "DA3_EnableTiledProcessing"], + "expectedNodeCount": 9, + "expectedExtensions": [], + "disabledNodes": { + "DA3_DownloadModel": ["NetworkAccess"], + "DA3_FilterGaussians": ["WritesToDisk"], + "DA3_PreviewPointCloud": ["WritesToDisk"], + "DA3_SavePointCloud": ["WritesToDisk"], + "DepthAnythingV3_Streaming": ["WritesToDisk"], + "LoadSALADModel": ["DisabledOnCloud"] + }, + "timeoutMs": 30000, + "webDirectory": "web" + }, + { + "pack": "ComfyUI-DynamiCrafterWrapper", + "deployRef": "https://github.com/kijai/ComfyUI-DynamiCrafterWrapper@d312c62982f1582b7955444d3a22ac948fb9c603", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": [ + "DownloadAndLoadCLIPVisionModel", + "DownloadAndLoadDynamiCrafterCNModel" + ], + "expectedNodeCount": 11, + "expectedExtensions": [], + "disabledNodes": { + "DownloadAndLoadCLIPModel": ["DisabledOnCloud"] + }, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-EditUtils", + "deployRef": "https://github.com/lrzjason/ComfyUI-EditUtils@256a2340b161da3f814e5982ca4aa999f474df93", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["AdaptiveLongestEdge_EditUtils", "Any2Image_EditUtils"], + "expectedNodeCount": 17, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-FlashVSR_Ultra_Fast", + "deployRef": "https://github.com/lihaoyun6/ComfyUI-FlashVSR_Ultra_Fast@4820b3f02347bddcbbb9a5a85ab7638fe976366e", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["FlashVSRInitPipe", "FlashVSRNode"], + "expectedNodeCount": 3, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-Fluxtapoz", + "deployRef": "https://github.com/logtd/ComfyUI-Fluxtapoz@17c71bea20e932945e9c1be0586cfd4b7e51cbf6", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["ApplyRegionalConds", "ConfigureModifiedFlux"], + "expectedNodeCount": 22, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-GIMM-VFI", + "deployRef": "https://github.com/kijai/ComfyUI-GIMM-VFI@4c9a3123762af85e7c796e41737da0b70c75d72d", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["DownloadAndLoadGIMMVFIModel", "GIMMVFI_interpolate"], + "expectedNodeCount": 2, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-Image-Filters", + "deployRef": "https://github.com/spacepxl/ComfyUI-Image-Filters@bbb3fb0045461adf3602faeedaf40af57090d4e2", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["AdainFilterLatent", "AdainImage"], + "expectedNodeCount": 60, + "expectedExtensions": [], + "disabledNodes": { + "ModelTest": ["DisabledOnCloud"] + }, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-KJNodes", + "deployRef": "https://github.com/kijai/ComfyUI-KJNodes@377ed49f540da1c5e419cd2fc494ccc79543d3c4", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["AddLabel", "AddNoiseToTrackPath"], + "expectedNodeCount": 227, + "expectedExtensions": ["KJNodes.GetNode"], + "disabledNodes": { + "CameraPoseVisualizer": ["ReadsArbitraryFile"], + "EndRecordCUDAMemoryHistory": ["Stateful", "WritesToDisk"], + "GGUFLoaderKJ": ["DisabledOnCloud"], + "LoadImagesFromFolderKJ": ["ReadsArbitraryFile"], + "LoadVideosFromFolder": ["ReadsArbitraryFile"], + "ModelSaveKJ": ["WritesToDisk"], + "PlaySoundKJ": ["ReadsArbitraryFile"], + "SaveImageKJ": ["WritesToDisk"], + "SaveStringKJ": ["WritesToDisk"], + "ScreencapStream": ["DisabledOnCloud"], + "StartRecordCUDAMemoryHistory": ["Stateful"], + "VisualizeCUDAMemoryHistory": ["WritesToDisk"], + "WebcamCaptureCV2": ["DisabledOnCloud"] + }, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-KwaiKolorsWrapper", + "deployRef": "https://github.com/kijai/ComfyUI-KwaiKolorsWrapper@6fc1cd9d20bb7537facf180e5494b486b9710e24", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": [ + "DownloadAndLoadChatGLM3", + "DownloadAndLoadKolorsModel" + ], + "expectedNodeCount": 5, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-LTXVideo", + "deployRef": "https://github.com/Lightricks/ComfyUI-LTXVideo@229437c6b65796d6a7a63ae34be2bd5ba31fa543", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["APGGuider", "AddLatentGuide"], + "expectedNodeCount": 74, + "expectedExtensions": ["LTXVideo.SparseTrackEditor"], + "disabledNodes": { + "GemmaAPITextEncode": ["NetworkAccess", "RequiresExternalAPI"], + "LTXVHDRDecodePostprocess": ["WritesToDisk"], + "LTXVSaveConditioning": ["WritesToDisk"] + }, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-Licon-MSR", + "deployRef": "https://github.com/liconstudio/ComfyUI-Licon-MSR@cc4f26f4eb60fc6f8abd5c4d58facdb0eb6d5470", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["LiconMSR"], + "expectedNodeCount": 1, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-LivePortraitKJ", + "deployRef": "https://github.com/kijai/ComfyUI-LivePortraitKJ@4d9dc6205b79e50f81c0e6d245a2e0e5e53d5ecd", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": [ + "DownloadAndLoadLivePortraitModels", + "KeypointsToImage" + ], + "expectedNodeCount": 9, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-Lotus", + "deployRef": "https://github.com/kijai/ComfyUI-Lotus@dcd5bea7a418717a6e0c27a5f607058cb9c24a5e", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["LoadLotusModel", "LotusSampler"], + "expectedNodeCount": 2, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-MelBandRoFormer", + "deployRef": "ComfyUI-MelBandRoFormer@1.0.2", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["MelBandRoFormerModelLoader", "MelBandRoFormerSampler"], + "expectedNodeCount": 2, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-MimicMotionWrapper", + "deployRef": "https://github.com/kijai/ComfyUI-MimicMotionWrapper@89d0ed551dbb1e0a14e89a01d42a39e9ed755d9c", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": [ + "DiffusersScheduler", + "DownloadAndLoadMimicMotionModel" + ], + "expectedNodeCount": 5, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-NormalCrafterWrapper", + "deployRef": "https://github.com/AIWarper/ComfyUI-NormalCrafterWrapper@10408381e60f42775a977966e111f1353d86e970", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["DetailTransfer", "NormalCrafterNode"], + "expectedNodeCount": 2, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-Prompt-Combinator", + "deployRef": "https://github.com/lquesada/ComfyUI-Prompt-Combinator@3576ace3f2776918c1046b6674fbe345a46dba2f", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["PromptCombinator", "PromptCombinatorMerger"], + "expectedNodeCount": 3, + "expectedExtensions": [], + "disabledNodes": { + "PromptCombinatorExportGallery": ["WritesToDisk"] + }, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-Qwen-TTS", + "deployRef": "https://github.com/flybirdxx/ComfyUI-Qwen-TTS@5e97d3cce967c842aeee02854725c371c8a101d7", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": [ + "FB_Qwen3TTSCustomVoice", + "FB_Qwen3TTSDialogueInference" + ], + "expectedNodeCount": 6, + "expectedExtensions": [], + "disabledNodes": { + "FB_Qwen3TTSLoadSpeaker": ["DisabledOnCloud", "ReadsArbitraryFile"], + "FB_Qwen3TTSSaveVoice": ["DisabledOnCloud", "WritesToDisk"] + }, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-QwenVL", + "deployRef": "ComfyUI-QwenVL@2.1.1", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["AILab_QwenVL", "AILab_QwenVL_Advanced"], + "expectedNodeCount": 3, + "expectedExtensions": [], + "disabledNodes": { + "AILab_QwenVL_GGUF": ["DisabledOnCloud"], + "AILab_QwenVL_GGUF_Advanced": ["DisabledOnCloud"] + }, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-SCAIL-Pose", + "deployRef": "ComfyUI-SCAIL-Pose@1.0.1", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": [ + "ConvertOpenPoseKeypointsToDWPose", + "PoseDetectionVitPoseToDWPose" + ], + "expectedNodeCount": 3, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-SeedVR2_VideoUpscaler", + "deployRef": "https://github.com/numz/ComfyUI-SeedVR2_VideoUpscaler@4490bd1f482e026674543386bb2a4d176da245b9", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["SeedVR2LoadDiTModel", "SeedVR2LoadVAEModel"], + "expectedNodeCount": 4, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-TiledDiffusion", + "deployRef": "https://github.com/shiimizu/ComfyUI-TiledDiffusion@a155b1bac39147381aeaa52b9be42e545626a44f", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["SpotDiffusionParams_TiledDiffusion", "TiledDiffusion"], + "expectedNodeCount": 4, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-UltraShape1", + "deployRef": "https://github.com/jtydhr88/ComfyUI-UltraShape1@2f8ea9b59bda7528ea2884b0868e205ece342e9e", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["UltraShapeConvertToGLB", "UltraShapeLoadCoarseMesh"], + "expectedNodeCount": 6, + "expectedExtensions": [], + "disabledNodes": { + "UltraShapeMeshSelector": ["DisabledOnCloud"], + "UltraShapeSaveGLB": ["DisabledOnCloud", "WritesToDisk"] + }, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-Upscaler-Tensorrt", + "deployRef": "https://github.com/yuvraj108c/ComfyUI-Upscaler-Tensorrt@dfc266538368e4c9a03a296917716a214517ccb5", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["LoadUpscalerTensorrtModel", "UpscalerTensorrt"], + "expectedNodeCount": 2, + "expectedExtensions": ["UpscalerTensorrt.CustomResize"], + "disabledNodes": {}, + "timeoutMs": 30000, + "webDirectory": "js" + }, + { + "pack": "ComfyUI-WanAnimatePreprocess", + "deployRef": "ComfyUI-WanAnimatePreprocess@1.0.1", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["DrawViTPose", "OnnxDetectionModelLoader"], + "expectedNodeCount": 5, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-WanVideoWrapper", + "deployRef": "ComfyUI-WanVideoWrapper@1.4.7", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": [ + "CreateCFGScheduleFloatList", + "CreateScheduleFloatList" + ], + "expectedNodeCount": 141, + "expectedExtensions": [], + "disabledNodes": { + "DownloadAndLoadNLFModel": ["DisabledOnCloud", "NetworkAccess"], + "WanVideoBlockSwap": ["DisabledOnCloud"], + "WanVideoLoraSelectByName": ["DisabledOnCloud"], + "WanVideoSetBlockSwap": ["DisabledOnCloud"], + "WanVideoVRAMManagement": ["DisabledOnCloud"] + }, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI-test-framework", + "deployRef": "https://github.com/Comfy-Org/ComfyUI-test-framework@80caae462d3293a2d40c2938de33a5f50eea8333", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["AssertContainsColor", "AssertEqual"], + "expectedNodeCount": 17, + "expectedExtensions": ["comfyui.test.framework.auto.save.api"], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI_AudioTools", + "deployRef": "https://github.com/lum3on/ComfyUI_AudioTools@c816d0a1629c05a0f463ddb77ea96ada69875d32", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["AudioAmplify", "AudioBPMDetector"], + "expectedNodeCount": 29, + "expectedExtensions": [], + "disabledNodes": { + "AudioGetFromList": ["DisabledOnCloud"], + "AudioLoadBatch": ["ReadsArbitraryFile"] + }, + "timeoutMs": 30000 + }, + { + "pack": "ComfyUI_Fill-ChatterBox", + "deployRef": "https://github.com/filliptm/ComfyUI_Fill-ChatterBox@2f72aca98e4ceda18caceaef91ddf5683f5f093e", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": [ + "FL_ChatterboxDialogTTS", + "FL_ChatterboxMultilingualTTS" + ], + "expectedNodeCount": 5, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000, + "webDirectory": "web" + }, + { + "pack": "ComfyUI_Fill-Nodes", + "deployRef": "https://github.com/filliptm/ComfyUI_Fill-Nodes@12c5bcded72a7f2c31c3df37516987c5d3691e28", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["FL_AnimatedShapePatterns", "FL_AnimeLineExtractor"], + "expectedNodeCount": 110, + "expectedExtensions": ["Fill-Nodes.TetrisGame"], + "disabledNodes": { + "FL_API_Base64_ImageLoader": ["DisabledOnCloud"], + "FL_API_ImageSaver": ["WritesToDisk"], + "FL_BulkPDFLoader": ["ReadsArbitraryFile"], + "FL_CaptionSaver_V2": ["ReadsArbitraryFile", "WritesToDisk"], + "FL_CaptionToCSV": ["ReadsArbitraryFile"], + "FL_ClipScanner": ["Stateful"], + "FL_CodeNode": ["DisabledOnCloud", "ReadsArbitraryFile"], + "FL_Dalle3": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_DirectoryCrawl": ["ReadsArbitraryFile"], + "FL_FILM": ["NetworkAccess", "Stateful", "WritesToDisk"], + "FL_Fal_Gemini_ImageEdit": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_Fal_Kling_AIAvatar": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_Fal_Kontext": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_Fal_Pixverse": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_Fal_Pixverse_LipSync": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_Fal_Pixverse_Transition": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_Fal_SeedVR_Upscale": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_Fal_Seedance_i2v": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_Fal_Seedream_Edit": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_Fal_Sora": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_GPT_Image1": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_GPT_Image1_ADV": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_GPT_Text": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_GPT_Vision": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_GeminiImageEditor": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_GeminiImageGenADV": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_GeminiTextAPI": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_GeminiVideoCaptioner": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_GoogleCloudStorage": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_GoogleDriveDownloader": ["DisabledOnCloud", "NetworkAccess"], + "FL_GoogleDriveImageDownloader": ["DisabledOnCloud", "NetworkAccess"], + "FL_HFDatasetDownloader": ["DisabledOnCloud", "NetworkAccess"], + "FL_HFHubModelUploader": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_HF_Character": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_HF_UploaderAbsolute": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_Hedra_API": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_ImageCaptionLayout": ["ReadsArbitraryFile"], + "FL_ImageCaptionLayoutPDF": ["ReadsArbitraryFile", "WritesToDisk"], + "FL_ImageCaptionSaver": ["WritesToDisk"], + "FL_ImagePicker": ["HasCustomEndpoints", "Stateful"], + "FL_ImageRandomizer": ["ReadsArbitraryFile"], + "FL_InfiniteZoom": ["DisabledOnCloud"], + "FL_JS": ["DisabledOnCloud"], + "FL_KartelJobInput": ["DisabledOnCloud", "NetworkAccess"], + "FL_KartelJobOutput": ["DisabledOnCloud", "NetworkAccess"], + "FL_LoadCSV": ["ReadsArbitraryFile"], + "FL_LoadImage": ["HasCustomEndpoints", "ReadsArbitraryFile"], + "FL_MirrorAndAppendCaptions": ["ReadsArbitraryFile", "WritesToDisk"], + "FL_ModelInspector": ["ReadsArbitraryFile"], + "FL_NFTGenerator": ["ReadsArbitraryFile"], + "FL_OllamaCaptioner": ["DisabledOnCloud", "NetworkAccess"], + "FL_PDFLoader": ["ReadsArbitraryFile"], + "FL_PDFSaver": ["WritesToDisk"], + "FL_PaperDrawn": ["DisabledOnCloud"], + "FL_PixVerseAPI": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_ProResVideo": [ + "DisabledOnCloud", + "ReadsArbitraryFile", + "WritesToDisk" + ], + "FL_RIFE": ["NetworkAccess", "Stateful", "WritesToDisk"], + "FL_RunwayAct2": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_RunwayImageAPI": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_SaveAndDisplayImage": ["WritesToDisk"], + "FL_SaveCSV": ["WritesToDisk"], + "FL_SaveImages": ["ReadsArbitraryFile", "WritesToDisk"], + "FL_SaveRGBAAnimatedWebP": ["WritesToDisk"], + "FL_SaveWebM": ["ReadsArbitraryFile", "WritesToDisk"], + "FL_SaveWebPImage(SaveImage)": ["WritesToDisk"], + "FL_SendToDiscordWebhook": ["DisabledOnCloud", "NetworkAccess"], + "FL_Shadertoy": ["DisabledOnCloud"], + "FL_SimpleGPTVision": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_SystemCheck": ["DisabledOnCloud", "HasCustomEndpoints"], + "FL_TimeLine": ["HasCustomEndpoints"], + "FL_Veo3VideoGen": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_VertexGemini25FlashImage": [ + "DisabledOnCloud", + "NetworkAccess", + "RequiresExternalAPI" + ], + "FL_VideoCaptionSaver": ["CreatesLargeOutputs", "WritesToDisk"], + "FL_VideoCut": ["DisabledOnCloud", "WritesToDisk"], + "FL_WF_Agent": ["DisabledOnCloud"], + "FL_WordFrequencyGraph": ["ReadsArbitraryFile"], + "FL_ZipDirectory": ["ReadsArbitraryFile"], + "FL_ZipSave": ["ReadsArbitraryFile", "WritesToDisk"] + }, + "timeoutMs": 30000, + "webDirectory": "web" + }, + { + "pack": "ComfyUI_LayerStyle_Advance", + "deployRef": "ComfyUI_LayerStyle_Advance@2.0.37", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["LayerMask: BBoxJoin", "LayerMask: BenUltra"], + "expectedNodeCount": 46, + "expectedExtensions": [], + "disabledNodes": { + "LayerFilter: LightLeak": ["DisabledOnCloud"], + "LayerMask: Florence2Ultra": ["DisabledOnCloud"], + "LayerMask: HumanPartsUltra": ["DisabledOnCloud", "NetworkAccess"], + "LayerMask: LoadFlorence2Model": ["DisabledOnCloud"], + "LayerMask: ObjectDetectorGemini": ["RequiresExternalAPI"], + "LayerMask: ObjectDetectorGeminiV2": ["RequiresExternalAPI"], + "LayerUtility: AddBlindWaterMark": ["DisabledOnCloud"], + "LayerUtility: DeepSeekAPI": ["RequiresExternalAPI"], + "LayerUtility: DeepSeekAPIV2": ["RequiresExternalAPI"], + "LayerUtility: Florence2Image2Prompt": ["DisabledOnCloud"], + "LayerUtility: Gemini": ["RequiresExternalAPI"], + "LayerUtility: GeminiImageEdit": ["RequiresExternalAPI"], + "LayerUtility: GeminiV2": ["RequiresExternalAPI"], + "LayerUtility: ImageRewardFilter": ["DisabledOnCloud"], + "LayerUtility: JimengI2IAPI": ["RequiresExternalAPI"], + "LayerUtility: JoyCaption2": ["DisabledOnCloud"], + "LayerUtility: JoyCaption2ExtraOptions": ["DisabledOnCloud"], + "LayerUtility: JoyCaption2Split": ["DisabledOnCloud"], + "LayerUtility: JoyCaptionBeta1": ["DisabledOnCloud"], + "LayerUtility: JoyCaptionBeta1ExtraOptions": ["DisabledOnCloud"], + "LayerUtility: LlamaVision": ["DisabledOnCloud"], + "LayerUtility: LoadJoyCaption2Model": ["DisabledOnCloud"], + "LayerUtility: LoadJoyCaptionBeta1Model": ["DisabledOnCloud"], + "LayerUtility: LoadPSD": ["ReadsArbitraryFile"], + "LayerUtility: PhiPrompt": ["DisabledOnCloud"], + "LayerUtility: PromptEmbellish": ["RequiresExternalAPI"], + "LayerUtility: PromptTagger": ["RequiresExternalAPI"], + "LayerUtility: SaveImagePlus": ["WritesToDisk"], + "LayerUtility: SaveImagePlusV2": ["WritesToDisk"], + "LayerUtility: ShowBlindWaterMark": ["DisabledOnCloud"], + "LayerUtility: ZhipuGLM4": ["RequiresExternalAPI"], + "LayerUtility: ZhipuGLM4V": ["RequiresExternalAPI"] + }, + "timeoutMs": 30000, + "webDirectory": "js" + }, + { + "pack": "ComfyUI_SchemaNodes", + "deployRef": "https://github.com/Liquid-XO/ComfyUI_SchemaNodes@8e5676d89f046e8c28cbe7f2548f4abdaf86ca35", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["SchemaAudioParameter", "SchemaBooleanParameter"], + "expectedNodeCount": 8, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000, + "webDirectory": "js" + }, + { + "pack": "Comfyui-SecNodes", + "deployRef": "https://github.com/9nate-drake/Comfyui-SecNodes@af39e6260bc414873fbb2dc8df355904d1609754", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["CoordinatePlotter", "SeCModelLoader"], + "expectedNodeCount": 3, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "ControlAltAI-Nodes", + "deployRef": "https://github.com/gseth/ControlAltAI-Nodes@721492b66c9cede8ae23ae10615462ad80cfd061", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["BooleanBasic", "BooleanReverse"], + "expectedNodeCount": 21, + "expectedExtensions": ["ControlAltAI.IntegerSettingsAdvanced"], + "disabledNodes": { + "FluxAttentionControl": ["DisabledOnCloud", "Stateful"] + }, + "timeoutMs": 30000 + }, + { + "pack": "CustomNodeComfyMath", + "deployRef": "https://github.com/Comfy-Org/CustomNodeComfyMath@c01177221c31b8e5fbc062778fc8254aeb541638", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["CM_BoolBinaryOperation", "CM_BoolToInt"], + "expectedNodeCount": 53, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "OneButtonPrompt", + "deployRef": "https://github.com/AIrjen/OneButtonPrompt@2129c5dfe6a73d4a0cf971c3af6c3cddbd0e97f8", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["AutoNegativePrompt", "CreatePromptVariant"], + "expectedNodeCount": 7, + "expectedExtensions": [], + "disabledNodes": { + "SavePromptToFile": ["WritesToDisk"] + }, + "timeoutMs": 30000 + }, + { + "pack": "RES4LYF", + "deployRef": "https://github.com/ClownsharkBatwing/RES4LYF@119679d8d8d26e6db52757e705488abb6399d7d4", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["AdvancedNoise", "BongSampler"], + "expectedNodeCount": 285, + "expectedExtensions": ["Comfy.RES4LYF.DisplayInfo"], + "disabledNodes": { + "Base64ToConditioning": ["DisabledOnCloud"], + "ClownsharKSamplerOptions": ["DisabledOnCloud"], + "ConditioningToBase64": ["DisabledOnCloud"], + "SamplerOptions_TimestepScaling": ["DisabledOnCloud"], + "Sigmas Math1": ["DisabledOnCloud"], + "Sigmas Math3": ["DisabledOnCloud"], + "TextLoadFile": ["ReadsArbitraryFile"], + "UNetSave": ["CreatesLargeOutputs", "WritesToDisk"] + }, + "timeoutMs": 30000 + }, + { + "pack": "Skimmed_CFG", + "deployRef": "https://github.com/Extraltodeus/Skimmed_CFG@b2cf6e99303f9f310054b888fa23c37da9a4d224", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": [ + "CFG_Skimming_Single_Scale_Pre_CFG", + "ConstantSkimPreCFG" + ], + "expectedNodeCount": 7, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "WhatDreamsCost-ComfyUI", + "deployRef": "https://github.com/WhatDreamsCost/WhatDreamsCost-ComfyUI@faa372880a4410af86f9a856f8617876cd55d17e", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["LTXDirector", "LTXDirectorGuide"], + "expectedNodeCount": 6, + "expectedExtensions": [], + "disabledNodes": { + "LoadVideoUI": ["HasCustomEndpoints", "ReadsArbitraryFile"], + "MultiImageLoader": ["ReadsArbitraryFile"] + }, + "timeoutMs": 30000, + "webDirectory": "js" + }, + { + "pack": "audio-separation-nodes-comfyui", + "deployRef": "audio-separation-nodes-comfyui@1.5.0", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["AudioCombine", "AudioCrop"], + "expectedNodeCount": 7, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "basic_data_handling", + "deployRef": "basic_data_handling@1.4.0", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": [ + "Basic data handling: Boolean And", + "Basic data handling: Boolean Nand" + ], + "expectedNodeCount": 279, + "expectedExtensions": ["Basic data handling: dynamic input"], + "disabledNodes": { + "Basic data handling: PathAbspath": ["PathParsing"], + "Basic data handling: PathBasename": ["PathParsing"], + "Basic data handling: PathCommonPrefix": ["PathParsing"], + "Basic data handling: PathDirname": ["PathParsing"], + "Basic data handling: PathExists": ["ReadsArbitraryFile"], + "Basic data handling: PathExpandVars": ["PathParsing"], + "Basic data handling: PathGetCwd": ["PathParsing"], + "Basic data handling: PathGetExtension": ["PathParsing"], + "Basic data handling: PathGetSize": ["ReadsArbitraryFile"], + "Basic data handling: PathGlob": ["ReadsArbitraryFile"], + "Basic data handling: PathInputDir": ["PathParsing"], + "Basic data handling: PathIsAbsolute": ["PathParsing"], + "Basic data handling: PathIsDir": ["ReadsArbitraryFile"], + "Basic data handling: PathIsFile": ["ReadsArbitraryFile"], + "Basic data handling: PathJoin": ["PathParsing"], + "Basic data handling: PathListDir": ["ReadsArbitraryFile"], + "Basic data handling: PathLoadImageRGB": ["ReadsArbitraryFile"], + "Basic data handling: PathLoadImageRGBA": ["ReadsArbitraryFile"], + "Basic data handling: PathLoadMaskFromAlpha": ["ReadsArbitraryFile"], + "Basic data handling: PathLoadMaskFromGreyscale": [ + "ReadsArbitraryFile" + ], + "Basic data handling: PathLoadStringFile": ["ReadsArbitraryFile"], + "Basic data handling: PathNormalize": ["PathParsing"], + "Basic data handling: PathOutputDir": ["PathParsing"], + "Basic data handling: PathRelative": ["PathParsing"], + "Basic data handling: PathSaveImageRGB": ["WritesToDisk"], + "Basic data handling: PathSaveImageRGBA": ["WritesToDisk"], + "Basic data handling: PathSaveStringFile": ["WritesToDisk"], + "Basic data handling: PathSetExtension": ["PathParsing"], + "Basic data handling: PathSplit": ["PathParsing"], + "Basic data handling: PathSplitExt": ["PathParsing"] + }, + "timeoutMs": 30000 + }, + { + "pack": "bfsnodes", + "deployRef": "bfsnodes@1.0.0", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["ReservedRegionFrameComposer"], + "expectedNodeCount": 1, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-PlyPreview", + "deployRef": "comfyui-PlyPreview@1.1.0", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": [ + "PlyPreviewPreviewGaussianEnhance", + "PlyPreviewProcessGaussianPLYEnhance" + ], + "expectedNodeCount": 2, + "expectedExtensions": ["plypreview.gaussianpreview"], + "disabledNodes": { + "PlyPreviewLoadGaussianPLYEnhance": ["ReadsArbitraryFile"], + "PlyPreviewLoadGaussianPLYPathEnhance": ["ReadsArbitraryFile"] + }, + "timeoutMs": 30000, + "webDirectory": "web" + }, + { + "pack": "comfyui-advanced-controlnet", + "deployRef": "comfyui-advanced-controlnet@1.5.6", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": [ + "ACN_AdvancedControlNetApply", + "ACN_AdvancedControlNetApplySingle" + ], + "expectedNodeCount": 42, + "expectedExtensions": [], + "disabledNodes": { + "LoadImagesFromDirectory": ["ReadsArbitraryFile"] + }, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-animatediff-evolved", + "deployRef": "comfyui-animatediff-evolved@1.5.6", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["ADE_ADBlockCombo", "ADE_ADBlockIndiv"], + "expectedNodeCount": 144, + "expectedExtensions": [], + "disabledNodes": { + "ADE_LoadCameraPosesFromPath": ["ReadsArbitraryFile"] + }, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-cogvideoxwrapper", + "deployRef": "comfyui-cogvideoxwrapper@1.5.1", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["CogVideoContextOptions", "CogVideoControlNet"], + "expectedNodeCount": 24, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-enricos-nodes", + "deployRef": "comfyui-enricos-nodes@3.1.6", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["Compositor3", "CompositorColorPicker"], + "expectedNodeCount": 7, + "expectedExtensions": ["ComfyUI.Enrico.ColorPicker"], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-florence2", + "deployRef": "comfyui-florence2@1.0.7", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": [ + "DownloadAndLoadFlorence2Lora", + "DownloadAndLoadFlorence2Model" + ], + "expectedNodeCount": 3, + "expectedExtensions": [], + "disabledNodes": { + "Florence2ModelLoader": ["DisabledOnCloud"] + }, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-frame-interpolation", + "deployRef": "comfyui-frame-interpolation@1.0.8", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["CAIN VFI", "FILM VFI"], + "expectedNodeCount": 11, + "expectedExtensions": [], + "disabledNodes": { + "AMT VFI": ["NetworkAccess"], + "GMFSS Fortuna VFI": ["NetworkAccess"], + "M2M VFI": ["NetworkAccess"], + "STMFNet VFI": ["NetworkAccess"], + "Sepconv VFI": ["NetworkAccess"] + }, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-ic-light-native", + "deployRef": "comfyui-ic-light-native@1.0.1", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["ICLightApplyMaskGrey", "ICLightAppply"], + "expectedNodeCount": 3, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-impact-pack", + "deployRef": "comfyui-impact-pack@8.28.2", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["AddMask", "AnyPipeToBasic"], + "expectedNodeCount": 185, + "expectedExtensions": [], + "disabledNodes": { + "ImageReceiver": ["DisabledOnCloud"], + "ImageSender": ["DisabledOnCloud"], + "ImpactQueueTrigger": ["DisabledOnCloud"], + "ImpactQueueTriggerCountdown": ["DisabledOnCloud"], + "ImpactSEGSPicker": ["DisabledOnCloud"], + "ImpactSetWidgetValue": ["DisabledOnCloud"], + "ImpactValueReceiver": ["DisabledOnCloud"], + "LatentReceiver": ["DisabledOnCloud"], + "MaskRectArea": ["DisabledOnCloud"], + "MaskRectAreaAdvanced": ["DisabledOnCloud"], + "PreviewBridge": ["DisabledOnCloud"], + "PreviewBridgeLatent": ["DisabledOnCloud"] + }, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-impact-subpack", + "deployRef": "comfyui-impact-subpack@1.3.5", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["UltralyticsDetectorProvider"], + "expectedNodeCount": 1, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-inpaint-cropandstitch", + "deployRef": "comfyui-inpaint-cropandstitch@2.1.8", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["InpaintCrop", "InpaintCropImproved"], + "expectedNodeCount": 6, + "expectedExtensions": ["inpaint-cropandstitch.showcontrol"], + "disabledNodes": {}, + "timeoutMs": 30000, + "webDirectory": "js" + }, + { + "pack": "comfyui-inpaint-nodes", + "deployRef": "comfyui-inpaint-nodes@1.4.0", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["INPAINT_ApplyFooocusInpaint", "INPAINT_ColorMatch"], + "expectedNodeCount": 12, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-itools", + "deployRef": "comfyui-itools@0.6.8", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["iToolsAddOverlay", "iToolsCheckerBoard"], + "expectedNodeCount": 14, + "expectedExtensions": ["iTools.compareNode"], + "disabledNodes": { + "iToolsCropImage": ["HasCustomEndpoints"], + "iToolsLoadImages": ["ReadsArbitraryFile"], + "iToolsLoadRandomImage": ["ReadsArbitraryFile"], + "iToolsPaintNode": ["DisabledOnCloud", "HasCustomEndpoints"], + "iToolsPromptLoader": ["ReadsArbitraryFile"], + "iToolsPromptSaver": ["WritesToDisk"], + "iToolsPromptStyler": ["DisabledOnCloud", "HasCustomEndpoints"], + "iToolsPromptStylerExtra": ["DisabledOnCloud", "HasCustomEndpoints"] + }, + "timeoutMs": 30000, + "webDirectory": "web" + }, + { + "pack": "comfyui-layerdiffuse", + "deployRef": "comfyui-layerdiffuse@1.0.4", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["LayeredDiffusionApply", "LayeredDiffusionCondApply"], + "expectedNodeCount": 6, + "expectedExtensions": [], + "disabledNodes": { + "LayeredDiffusionCondJointApply": ["DisabledOnCloud"], + "LayeredDiffusionJointApply": ["DisabledOnCloud"] + }, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-logic", + "deployRef": "comfyui-logic@1.0.0", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["Bool", "Compare"], + "expectedNodeCount": 6, + "expectedExtensions": [], + "disabledNodes": { + "DebugPrint": ["DisabledOnCloud"] + }, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-qwenmultiangle", + "deployRef": "comfyui-qwenmultiangle@0.6.1", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["QwenMultiangleCameraNode"], + "expectedNodeCount": 2, + "expectedExtensions": ["ComfyUI.QwenMultiangle"], + "disabledNodes": {}, + "timeoutMs": 30000, + "webDirectory": "js" + }, + { + "pack": "comfyui-rmbg", + "deployRef": "comfyui-rmbg@3.0.0", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["AILab_ColorInput", "AILab_ColorToMask"], + "expectedNodeCount": 35, + "expectedExtensions": ["AILab.colorWidget"], + "disabledNodes": { + "AILab_LoadImage": ["ReadsArbitraryFile"], + "AILab_LoadImageAdvanced": ["ReadsArbitraryFile"], + "AILab_LoadImageBatch": ["ReadsArbitraryFile"], + "AILab_LoadImageSimple": ["ReadsArbitraryFile"], + "AILab_YoloV8": ["DisabledOnCloud"], + "AILab_YoloV8Adv": ["DisabledOnCloud"], + "Segment": ["DisabledOnCloud"], + "SegmentV2": ["DisabledOnCloud"] + }, + "timeoutMs": 30000, + "webDirectory": "web" + }, + { + "pack": "comfyui-sam3", + "deployRef": "comfyui-sam3@0.1.2", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["LoadSAM3Model", "SAM3BBoxCollector"], + "expectedNodeCount": 14, + "expectedExtensions": ["comfyui.sam3.video_dynamic"], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-segment-anything-2", + "deployRef": "comfyui-segment-anything-2@1.0.2", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["DownloadAndLoadSAM2Model", "Florence2toCoordinates"], + "expectedNodeCount": 6, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-sharp", + "deployRef": "comfyui-sharp@0.0.3", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["LoadImageWithExif", "LoadSharpModel"], + "expectedNodeCount": 3, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-string-converter", + "deployRef": "https://github.com/glowcone/comfyui-string-converter@0e1e78f877e4d5fc2b46ace8fa21a0ef70e92f11", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["StringToFloat", "StringToInt"], + "expectedNodeCount": 2, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-supir", + "deployRef": "comfyui-supir@1.0.4", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["SUPIR_Upscale", "SUPIR_conditioner"], + "expectedNodeCount": 10, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-systms-facecomposite", + "deployRef": "comfyui-systms-facecomposite@1.0.0", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["FaceComposite"], + "expectedNodeCount": 1, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-various", + "deployRef": "https://github.com/jamesWalker55/comfyui-various@5bd85aaf7616878471469c4ec7e11bbd0cef3bf2", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["JWAudioBlend", "JWDatetimeString"], + "expectedNodeCount": 55, + "expectedExtensions": [], + "disabledNodes": { + "JWAudioSaveToPath": ["WritesToDisk"], + "JWImageLoadRGB": ["ReadsArbitraryFile"], + "JWImageLoadRGBA": ["ReadsArbitraryFile"], + "JWImageLoadRGBA From Clipboard": ["DisabledOnCloud"], + "JWImageLoadRGBFromClipboard": ["DisabledOnCloud"], + "JWImageLoadRGBIfExists": ["ReadsArbitraryFile"], + "JWImageSaveToPath": ["WritesToDisk"], + "JWLoadAudio": ["ReadsArbitraryFile"], + "JWLoadImageSequence": ["ReadsArbitraryFile"], + "JWLoadImageSequenceWithStopIndex": ["ReadsArbitraryFile"], + "JWLoadImagesFromString": ["ReadsArbitraryFile"], + "JWSaveImageSequence": ["WritesToDisk"], + "RAFTLoadFlowFromEXRChannels": ["ReadsArbitraryFile"] + }, + "timeoutMs": 30000 + }, + { + "pack": "comfyui-videohelpersuite", + "deployRef": "comfyui-videohelpersuite@1.7.9", + "tiers": ["load", "connectivity", "run"], + "workflow": "assets/customNodes/vhs_video_pipeline_cloud_run.json", + "expectedNodes": ["VHS_LoadVideo", "VHS_VideoInfo"], + "expectedNodeCount": 32, + "expectedRunnableCount": 31, + "expectedExtensions": ["VideoHelperSuite.Core"], + "disabledNodes": { + "VHS_BatchManager": ["DisabledOnCloud"], + "VHS_LoadAudio": ["ReadsArbitraryFile"], + "VHS_LoadImagePath": ["ReadsArbitraryFile"], + "VHS_LoadImages": ["ReadsArbitraryFile"], + "VHS_LoadImagesPath": ["ReadsArbitraryFile"], + "VHS_LoadVideoFFmpegPath": ["ReadsArbitraryFile"], + "VHS_LoadVideoPath": ["ReadsArbitraryFile"], + "VHS_SelectFilename": ["ReadsArbitraryFile"] + }, + "timeoutMs": 90000 + }, + { + "pack": "comfyui-workflow-prettier", + "deployRef": "comfyui-workflow-prettier@1.0.5", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["WorkflowPrettifier"], + "expectedNodeCount": 1, + "expectedExtensions": ["comfyui.workflow.prettier"], + "disabledNodes": {}, + "timeoutMs": 30000, + "webDirectory": "web" + }, + { + "pack": "comfyui_controlnet_aux", + "deployRef": "comfyui_controlnet_aux@1.1.5", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["AIO_Preprocessor", "AnimalPosePreprocessor"], + "expectedNodeCount": 64, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "comfyui_essentials", + "deployRef": "comfyui_essentials@1.1.0", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["ApplyCLIPSeg+", "BatchCount+"], + "expectedNodeCount": 62, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "comfyui_face_parsing", + "deployRef": "comfyui_face_parsing@1.0.5", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": [ + "BBoxDecompose(FaceParsing)", + "BBoxDetect(FaceParsing)" + ], + "expectedNodeCount": 27, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "comfyui_ipadapter_plus", + "deployRef": "comfyui_ipadapter_plus@2.0.0", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["IPAAdapterFaceIDBatch", "IPAdapter"], + "expectedNodeCount": 29, + "expectedExtensions": [], + "disabledNodes": { + "IPAdapterSaveEmbeds": ["DisabledOnCloud", "WritesToDisk"] + }, + "timeoutMs": 30000 + }, + { + "pack": "comfyui_layerstyle", + "deployRef": "comfyui_layerstyle@2.0.32", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["LayerColor: AutoAdjust", "LayerColor: AutoAdjustV2"], + "expectedNodeCount": 162, + "expectedExtensions": [], + "disabledNodes": { + "LayerFilter: LightLeak": ["DisabledOnCloud"], + "LayerMask: RemBgUltra": ["DisabledOnCloud", "NetworkAccess"], + "LayerMask: RmBgUltra V2": ["DisabledOnCloud", "NetworkAccess"], + "LayerUtility: ImageTaggerSave": ["WritesToDisk"], + "LayerUtility: ImageTaggerSaveV2": ["WritesToDisk"], + "LayerUtility: LoadImagesFromPath": ["ReadsArbitraryFile"], + "LayerUtility: PurgeVRAM": ["DisabledOnCloud", "Stateful"], + "LayerUtility: PurgeVRAM V2": ["DisabledOnCloud", "Stateful"], + "LayerUtility: QueueStop": ["DisabledOnCloud"] + }, + "timeoutMs": 30000, + "webDirectory": "js" + }, + { + "pack": "comfyui_nvidia_rtx_nodes", + "deployRef": "comfyui_nvidia_rtx_nodes@0.1.2", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["RTXVideoSuperResolution"], + "expectedNodeCount": 1, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "comfyui_steudio", + "deployRef": "comfyui_steudio@2.0.5", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["Combine Tiles", "Divide Image and Select Tile"], + "expectedNodeCount": 8, + "expectedExtensions": [], + "disabledNodes": { + "Display UI": ["DisabledOnCloud", "DuplicateOfCoreNode"], + "Load Images into List": ["ReadsArbitraryFile"] + }, + "timeoutMs": 30000 + }, + { + "pack": "comfyui_ultimatesdupscale", + "deployRef": "comfyui_ultimatesdupscale@1.6.2", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["UltimateSDUpscale", "UltimateSDUpscaleCustomSample"], + "expectedNodeCount": 3, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "feedback-sampler", + "deployRef": "feedback-sampler@2.0.0", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["FeedbackSampler"], + "expectedNodeCount": 1, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "masquerade-nodes-comfyui", + "deployRef": "https://github.com/BadCafeCode/masquerade-nodes-comfyui@432cb4d146a391b387a0cd25ace824328b5b61cf", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["Blur", "Change Channel Count"], + "expectedNodeCount": 20, + "expectedExtensions": [], + "disabledNodes": { + "Create QR Code": ["DisabledOnCloud"], + "Mask By Text": ["DisabledOnCloud", "NetworkAccess"] + }, + "timeoutMs": 30000 + }, + { + "pack": "nodesweet", + "deployRef": "nodesweet@1.0.1", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["AudioReactiveTransform", "AudioWeightsRemap"], + "expectedNodeCount": 6, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "radiance", + "deployRef": "https://github.com/fxtdstudios/radiance@f262f47ddfda01ece154bf80c22769b1e4cef795", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["ACES2OutputTransform", "ARRIWideGamut4"], + "expectedNodeCount": 70, + "expectedExtensions": ["Radiance.IO"], + "disabledNodes": { + "RadianceLUTApply": ["DisabledOnCloud"], + "◎ RadianceDigitalCinemaRead": ["ReadsArbitraryFile"], + "◎ RadianceNukeBridge": ["NetworkAccess", "WritesToDisk"], + "◎ RadianceQCExport": ["WritesToDisk"] + }, + "timeoutMs": 30000, + "webDirectory": "js" + }, + { + "pack": "sd-dynamic-thresholding", + "deployRef": "sd-dynamic-thresholding@1.0.1", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["DynamicThresholdingFull", "DynamicThresholdingSimple"], + "expectedNodeCount": 2, + "expectedExtensions": [], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "vewd", + "deployRef": "vewd@1.7.2", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["Vewd"], + "expectedNodeCount": 1, + "expectedExtensions": ["vewd"], + "disabledNodes": {}, + "timeoutMs": 30000 + }, + { + "pack": "wanvaceadvanced", + "deployRef": "wanvaceadvanced@1.4.0", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["HuMoI2VPatch", "StringToFloatListRanged"], + "expectedNodeCount": 11, + "expectedExtensions": [], + "disabledNodes": { + "WanVacePhantomDual": ["DisabledOnCloud"], + "WanVacePhantomExperimental": ["DisabledOnCloud"] + }, + "timeoutMs": 30000 + }, + { + "pack": "was-node-suite-comfyui", + "deployRef": "https://github.com/ltdrdata/was-node-suite-comfyui@afeee09ba44e713ec52a413ac6b105fd06b2d356", + "tiers": ["load", "connectivity"], + "workflow": "", + "expectedNodes": ["Blend Latents", "Boolean To Text"], + "expectedNodeCount": 183, + "expectedExtensions": [], + "disabledNodes": { + "BLIP Analyze Image": ["DisabledOnCloud"], + "BLIP Model Loader": ["DisabledOnCloud"], + "CLIPSEG2": ["DisabledOnCloud"], + "CLIPSeg Batch Masking": ["DisabledOnCloud"], + "CLIPSeg Masking": ["DisabledOnCloud"], + "CLIPSeg Model Loader": ["DisabledOnCloud"], + "Cache Node": ["DisabledOnCloud"], + "Checkpoint Loader": ["DisabledOnCloud", "DuplicateOfCoreNode"], + "Checkpoint Loader (Simple)": [ + "DisabledOnCloud", + "DuplicateOfCoreNode" + ], + "Create Grid Image": ["ReadsArbitraryFile"], + "Create Morph Image from Path": ["ReadsArbitraryFile"], + "Create Video from Path": ["ReadsArbitraryFile"], + "Debug Number to Console": ["DisabledOnCloud"], + "Dictionary to Console": ["DisabledOnCloud"], + "Diffusers Hub Model Down-Loader": ["NetworkAccess"], + "Export API": ["DisabledOnCloud"], + "Image Bounds to Console": ["DisabledOnCloud"], + "Image History Loader": ["ReadsArbitraryFile"], + "Image Load": ["ReadsArbitraryFile"], + "Image Save": ["WritesToDisk"], + "Image Send HTTP": ["NetworkAccess"], + "Load Cache": ["DisabledOnCloud"], + "Load Image Batch": ["ReadsArbitraryFile"], + "Load Text File": ["ReadsArbitraryFile"], + "MiDaS Depth Approximation": ["DisabledOnCloud"], + "MiDaS Mask Image": ["DisabledOnCloud"], + "MiDaS Model Loader": ["DisabledOnCloud"], + "SAM Model Loader": ["DisabledOnCloud"], + "Save Text File": ["WritesToDisk"], + "Text File History Loader": ["ReadsArbitraryFile"], + "Text Load Line From File": ["ReadsArbitraryFile"], + "Text Random Prompt": ["DisabledOnCloud"], + "True Random.org Number Generator": ["NetworkAccess"], + "Upscale Model Loader": ["DisabledOnCloud", "DuplicateOfCoreNode"], + "Video Dump Frames": ["WritesToDisk"], + "Write to GIF": ["WritesToDisk"], + "Write to Video": ["WritesToDisk"] + }, + "timeoutMs": 30000 + } + ], + "unjoinedYamlPacks": ["comfyui-fl-path-animator"] +} diff --git a/browser_tests/fixtures/data/customNodeManifest.core.json b/browser_tests/fixtures/data/customNodeManifest.core.json new file mode 100644 index 00000000000..680e1de0f1c --- /dev/null +++ b/browser_tests/fixtures/data/customNodeManifest.core.json @@ -0,0 +1,155 @@ +[ + { + "pack": "ComfyUI-Impact-Pack", + "repo": "https://github.com/ltdrdata/ComfyUI-Impact-Pack", + "pin": "429d0159ad429e64d2b3916e6e7be9c22d025c3c", + "pinnedAt": "2026-08-14", + "tiers": ["load", "connectivity", "run"], + "workflow": "assets/customNodes/impact_primitives_run.json", + "expectedNodes": ["ImpactInt", "ImpactFloat"], + "expectedNodeCount": 197, + "expectedRunnableCount": 80, + "expectedExtensions": ["Comfy.Impack"], + "requiresGpu": false, + "requiresModels": [], + "timeoutMs": 30000, + "cannotRunAlone": [ + "AnyPipeToBasic", + "CLIPSegDetectorProvider", + "ImpactMakeImageBatch", + "ImpactMakeMaskBatch", + "LatentSender", + "MasksToMaskList", + "PreviewBridgeLatent" + ] + }, + { + "pack": "ComfyUI-VideoHelperSuite", + "repo": "https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite", + "pin": "4ee72c065db22c9d96c2427954dc69e7b908444b", + "pinnedAt": "2026-08-14", + "tiers": ["load", "connectivity", "run"], + "workflow": "assets/customNodes/vhs_video_pipeline_run.json", + "expectedNodes": ["VHS_LoadVideoPath", "VHS_VideoInfo"], + "expectedNodeCount": 40, + "expectedRunnableCount": 31, + "expectedExtensions": ["VideoHelperSuite.Core"], + "requiresGpu": false, + "requiresModels": [], + "timeoutMs": 90000 + }, + { + "pack": "ComfyUI_essentials", + "repo": "https://github.com/cubiq/ComfyUI_essentials", + "pin": "9d9f4bedfc9f0321c19faf71855e228c93bd0dc9", + "pinnedAt": "2026-08-14", + "tiers": ["load", "connectivity", "run"], + "workflow": "assets/customNodes/essentials_math_display_run.json", + "expectedNodes": ["SimpleMathInt+", "DisplayAny"], + "expectedNodeCount": 85, + "expectedRunnableCount": 62, + "expectedExtensions": ["essentials.DisplayAny"], + "requiresGpu": false, + "requiresModels": [], + "timeoutMs": 30000, + "cannotRunAlone": [ + "ImageApplyLUT+", + "ImageUntile+", + "MaskFromList+", + "PixelOEPixelize+", + "SimpleCondition+", + "SimpleMath+", + "SimpleMathCondition+", + "SimpleMathDual+" + ] + }, + { + "pack": "ComfyUI-KJNodes", + "repo": "https://github.com/kijai/ComfyUI-KJNodes", + "pin": "e27a505b3ba6ce42687fe00500deda103d9d6071", + "pinnedAt": "2026-08-14", + "tiers": ["load", "connectivity", "run"], + "workflow": "assets/customNodes/kjnodes_constants_run.json", + "expectedNodes": ["INTConstant", "FloatConstant"], + "expectedNodeCount": 244, + "expectedRunnableCount": 141, + "expectedExtensions": ["KJNodes.appearance"], + "requiresGpu": false, + "requiresModels": [], + "timeoutMs": 30000, + "cannotRunAlone": [ + "CameraPoseVisualizer", + "CreateAudioMask", + "CreateGradientFromCoords", + "CreateInstanceDiffusionTracking", + "CreateShapeImageOnPath", + "CreateShapeMaskOnPath", + "CreateTextOnPath", + "CrossFadeImages", + "CrossFadeImagesMulti", + "CustomControlNetWeightsFluxFromList", + "CutAndDragOnPath", + "EndRecordCUDAMemoryHistory", + "FloatToMask", + "GetImagesFromBatchIndexed", + "GetLatentsFromBatchIndexed", + "ImageAndMaskPreview", + "ImageGridtoBatch", + "ImagePadForOutpaintTargetSize", + "InterpolateCoords", + "LoadImagesFromFolderKJ", + "LoadVideosFromFolder", + "PlotCoordinates", + "StartRecordCUDAMemoryHistory", + "Superprompt", + "VisualizeCUDAMemoryHistory", + "WebcamCaptureCV2", + "WeightScheduleConvert", + "WeightScheduleExtend", + "WidgetToString" + ] + }, + { + "pack": "ComfyUI-Custom-Scripts", + "repo": "https://github.com/pythongosssss/ComfyUI-Custom-Scripts", + "pin": "609f3afaa74b2f88ef9ce8d939626065e3247469", + "pinnedAt": "2026-08-14", + "tiers": ["load", "connectivity", "run"], + "workflow": "assets/customNodes/customscripts_string_show_run.json", + "expectedNodes": ["StringFunction|pysssss", "ShowText|pysssss"], + "expectedNodeCount": 13, + "expectedRunnableCount": 10, + "expectedExtensions": ["pysssss.ShowText"], + "requiresGpu": false, + "requiresModels": [], + "timeoutMs": 30000, + "cannotRunAlone": ["MathExpression|pysssss"] + }, + { + "pack": "was-node-suite-comfyui", + "repo": "https://github.com/WASasquatch/was-node-suite-comfyui", + "pin": "ea935d1044ae5a26efa54ebeb18fe9020af49a45", + "pinnedAt": "2026-08-14", + "tiers": ["load", "connectivity", "run"], + "workflow": "assets/customNodes/was_number_text_run.json", + "expectedNodes": ["Constant Number", "Number to Text", "Text to Console"], + "expectedNodeCount": 220, + "expectedRunnableCount": 162, + "expectedExtensions": [], + "requiresGpu": false, + "requiresModels": [], + "timeoutMs": 30000, + "cannotRunAlone": [ + "Bus Node", + "Diffusers Hub Model Down-Loader", + "Image Aspect Ratio", + "Image Batch", + "Image Send HTTP", + "Latent Batch", + "Mask Batch", + "Samples Passthrough (Stat System)", + "Text Dictionary Convert", + "Text to Number" + ] + } +] diff --git a/browser_tests/fixtures/utils/consoleErrorCollector.ts b/browser_tests/fixtures/utils/consoleErrorCollector.ts new file mode 100644 index 00000000000..b402d679d25 --- /dev/null +++ b/browser_tests/fixtures/utils/consoleErrorCollector.ts @@ -0,0 +1,54 @@ +import type { ConsoleMessage, Page, TestInfo } from '@playwright/test' + +const startupErrors = new WeakMap() + +export function recordStartupConsoleErrors( + page: Page, + errors: readonly string[] +): void { + startupErrors.set(page, [...errors]) +} + +export function startupConsoleErrors(page: Page): readonly string[] { + return startupErrors.get(page) ?? [] +} + +export async function attachPageDiagnosticEvidence( + testInfo: Pick, + name: string, + values: readonly string[] +): Promise { + await testInfo.attach(name, { + body: JSON.stringify(values, null, 2), + contentType: 'application/json' + }) +} + +export function collectConsoleErrors(page: Page): { + errors: string[] + stop: () => void +} { + const errors: string[] = [] + const listener = (message: ConsoleMessage) => { + if (message.type() !== 'error') return + const url = message.location().url + errors.push(url ? `${message.text()} [${url}]` : message.text()) + } + // Uncaught page exceptions and unhandled promise rejections never reach + // console.error; Chromium surfaces both through pageerror. Without this + // listener a pack script crashing outside a console call passes silently. + const pageErrorListener = (error: Error) => { + errors.push(`Uncaught page error: ${error.stack ?? error.message}`) + } + page.on('console', listener) + page.on('pageerror', pageErrorListener) + return { + get errors() { + return errors + }, + stop: () => { + page.off('console', listener) + page.off('pageerror', pageErrorListener) + } + } +} diff --git a/browser_tests/fixtures/utils/customNodeSuite.ts b/browser_tests/fixtures/utils/customNodeSuite.ts new file mode 100644 index 00000000000..32c6b7e0fb0 --- /dev/null +++ b/browser_tests/fixtures/utils/customNodeSuite.ts @@ -0,0 +1,271 @@ +import type { Page, Response } from '@playwright/test' + +import type { + ActivePathPointer, + DraftIndexV2, + DraftPayloadV2, + OpenPathsPointer +} from '@/platform/workflow/persistence/base/draftTypes' +import { StorageKeys } from '@/platform/workflow/persistence/base/storageKeys' +import type { ComfyWorkflowJSON } from '@/platform/workflow/validation/schemas/workflowSchema' + +const CUSTOM_NODE_BLANK_WORKFLOW_PATH = + 'workflows/Custom Nodes E2E Blank Workflow.json' +// Importing blankGraph evaluates import.meta.env in Playwright's Node process. +const CUSTOM_NODE_BLANK_GRAPH: ComfyWorkflowJSON = { + last_node_id: 0, + last_link_id: 0, + nodes: [], + links: [], + groups: [], + config: {}, + extra: {}, + version: 0.4 +} + +export async function installCustomNodeBlankStartup(page: Page): Promise { + const workspaceId = 'personal' + const path = CUSTOM_NODE_BLANK_WORKFLOW_PATH + const draftKey = StorageKeys.draftKey(path) + const updatedAt = Date.now() + const index: DraftIndexV2 = { + v: 2, + updatedAt, + order: [draftKey], + entries: { + [draftKey]: { + path, + name: 'Custom Nodes E2E Blank Workflow.json', + isTemporary: true, + updatedAt + } + } + } + const payload: DraftPayloadV2 = { + data: JSON.stringify(CUSTOM_NODE_BLANK_GRAPH), + updatedAt + } + const active: ActivePathPointer = { workspaceId, path } + const open: OpenPathsPointer = { + workspaceId, + paths: [path], + activeIndex: 0 + } + const entries = [ + [StorageKeys.draftIndex(workspaceId), JSON.stringify(index)], + [StorageKeys.draftPayload(path, workspaceId), JSON.stringify(payload)], + [StorageKeys.lastActivePath(workspaceId), JSON.stringify(active)], + [StorageKeys.lastOpenPaths(workspaceId), JSON.stringify(open)] + ] + await page.addInitScript((entries) => { + for (const [key, value] of entries) localStorage.setItem(key, value) + }, entries) +} + +function throwCollectedErrors(errors: readonly unknown[]): void { + if (errors.length === 1) throw errors[0] + if (errors.length > 1) + throw new AggregateError(errors, 'test and fixture teardown failed') +} + +export async function runWithCollectedCleanup( + run: () => Promise, + cleanups: readonly (() => Promise)[] +): Promise { + const errors: unknown[] = [] + try { + await run() + } catch (error) { + errors.push(error) + } + for (const cleanup of cleanups) { + try { + await cleanup() + } catch (error) { + errors.push(error) + } + } + throwCollectedErrors(errors) +} + +// The suite restores a pre-seeded blank draft instead of the bundled +// default graph, whose model references error on the model-less backend. +// The restored startup outcome also prevents onboarding without a reload. +// The shared fixture disables the errors tab to hide missing-model +// indicators in unrelated suites; this suite exists to SEE errors, so every +// error surface stays live. +export const customNodeSuiteSettings = { + 'Comfy.TutorialCompleted': true, + 'Comfy.Workflow.Persist': true, + 'Comfy.RightSidePanel.ShowErrorsTab': true +} + +/** + * Watches this page's `/prompt` POSTs, handing each response's `prompt_id` + * (undefined when the body carries none) plus the raw body and status to + * `handle`. `settled` resolves once every response seen so far has been + * parsed, which is what lets a caller read a complete id ledger. + */ +export function onPromptIdResponse( + page: Page, + handle: ( + promptId: string | undefined, + body: unknown, + status: number, + sequence: number + ) => void +): { detach: () => void; settled: () => Promise } { + const parsing = new Set>() + let sequence = 0 + const listener = (response: Response) => { + if (response.request().method() !== 'POST') return + if (!new URL(response.url()).pathname.endsWith('/prompt')) return + const responseSequence = ++sequence + parsing.add( + response + .json() + .then((body: unknown) => { + const id = (body as { prompt_id?: unknown } | null)?.prompt_id + handle( + typeof id === 'string' ? id : undefined, + body, + response.status(), + responseSequence + ) + }) + .catch(() => { + if (response.status() >= 400) + handle(undefined, undefined, response.status(), responseSequence) + }) + ) + } + page.on('response', listener) + return { + detach: () => page.off('response', listener), + settled: async () => { + await Promise.allSettled([...parsing]) + } + } +} + +interface PromptLedger { + ids: Set + settled: () => Promise +} + +// Backends are shared - two workers, two CI runs - so an unscoped +// interrupt(null) + clearItems('queue') cancels another client's in-flight +// prompt. This ledger is what lets the drain below touch only what this +// page submitted. +const promptLedgers = new WeakMap() + +/** + * Starts recording this page's `/prompt` submissions. Install in `beforeEach`, + * before anything queues: {@link drainBackendToIdle} refuses to run without the + * ledger rather than silently degrading to cleaning nothing up. + */ +export function trackSubmittedPrompts(page: Page): void { + if (promptLedgers.has(page)) return + const ids = new Set() + const { settled } = onPromptIdResponse(page, (promptId) => { + if (promptId !== undefined) ids.add(promptId) + }) + promptLedgers.set(page, { ids, settled }) +} + +interface QueueIds { + running: string[] + pending: string[] +} + +function ownedQueueEntries( + queue: QueueIds, + ownedIds: ReadonlySet +): QueueIds { + return { + running: queue.running.filter((id) => ownedIds.has(id)), + pending: queue.pending.filter((id) => ownedIds.has(id)) + } +} + +function readQueueIds(page: Page): Promise { + return page.evaluate(async () => { + const { Running, Pending } = await window.app!.api.getQueue({ + throwOnError: true + }) + return { + running: Running.map((job) => job.id), + pending: Pending.map((job) => job.id) + } + }) +} + +function sleep(page: Page): Promise { + return page.evaluate(() => new Promise((resolve) => setTimeout(resolve, 500))) +} + +// Every test gets a fresh page, but they share ONE backend. An execution +// tier that ends while a prompt is still draining leaves that work running +// on the shared backend; the next test's fresh page connects mid-execution +// and catches its async error events (console noise, a popped error dialog) +// or its still-running prompt (queue-busy). Draining to idle in an afterEach +// - while the finishing test's own page is still open, so any late events +// land there - is what makes each test unable to affect the next. +// Scope is this page's own prompts, for both the cancelling and the result: a +// co-tenant's busy queue is neither cleared nor charged to us, and leftovers +// from an earlier test on the shared backend need waitForQueueQuiet instead. +// Cancellation repeats each poll because a pending entry can be promoted to +// running between the read and the delete. +// Returns 0 when this page's work reached idle, 1 when it was still busy after +// the budget (a genuinely wedged, non-interruptible execution). +export async function drainBackendToIdle( + page: Page, + budgetMs = 150_000 +): Promise { + const ledger = promptLedgers.get(page) + if (!ledger) + throw new Error( + 'drainBackendToIdle: no prompt ledger for this page - call trackSubmittedPrompts(page) in beforeEach' + ) + // A submission whose response body is still parsing is not in ids yet, and + // reading past it would report this page as having nothing to clean up. + await ledger.settled() + if (ledger.ids.size === 0) return 0 + + const deadline = Date.now() + budgetMs + let owned = ownedQueueEntries(await readQueueIds(page), ledger.ids) + while (owned.running.length + owned.pending.length > 0) { + await page.evaluate(async ({ running, pending }) => { + for (const id of running) await window.app!.api.interrupt(id) + for (const id of pending) await window.app!.api.deleteItem('queue', id) + }, owned) + if (Date.now() >= deadline) return 1 + await sleep(page) + owned = ownedQueueEntries(await readQueueIds(page), ledger.ids) + } + return 0 +} + +/** + * Polls until the WHOLE backend queue is empty, cancelling nothing. Use this + * where the work being waited on is not this page's own - a previous test's + * still-draining execution on the shared backend, which + * {@link drainBackendToIdle} cannot see and must not cancel. Returns 0 when + * the queue emptied, 1 when it was still busy after the budget. + */ +export async function waitForQueueQuiet( + page: Page, + budgetMs: number +): Promise { + const depth = async () => { + const { running, pending } = await readQueueIds(page) + return running.length + pending.length + } + const deadline = Date.now() + budgetMs + let remaining = await depth() + while (remaining !== 0 && Date.now() < deadline) { + await sleep(page) + remaining = await depth() + } + return remaining === 0 ? 0 : 1 +} diff --git a/browser_tests/fixtures/utils/errorSurfaces.ts b/browser_tests/fixtures/utils/errorSurfaces.ts new file mode 100644 index 00000000000..271032a8e24 --- /dev/null +++ b/browser_tests/fixtures/utils/errorSurfaces.ts @@ -0,0 +1,137 @@ +import type { Locator, Page } from '@playwright/test' +import { expect } from '@playwright/test' + +import { TestIds } from '@e2e/fixtures/selectors' + +interface VisibleError { + surface: string + text: string +} + +type VisibleErrorWindow = Window & + typeof globalThis & { + __cnVisibleErrors?: VisibleError[] + } + +const trackedPages = new WeakSet() + +const visibleErrorSurfaceSelectors = [ + { + surface: 'errorOverlay', + selector: `[data-testid="${TestIds.dialogs.errorOverlay}"]` + }, + { + surface: 'errorDialog', + selector: `[data-testid="${TestIds.dialogs.errorDialog}"]` + }, + { surface: 'nodeRenderErrors', selector: '.node-error' }, + { surface: 'errorToasts', selector: '.p-toast-message-error' } +] + +const visibleErrorSampleIntervalMs = 100 + +function installVisibleErrorRecorder(config: { + selectors: typeof visibleErrorSurfaceSelectors + sampleIntervalMs: number +}): void { + const { selectors, sampleIntervalMs } = config + const target = window as VisibleErrorWindow + if (target.__cnVisibleErrors !== undefined) return + const seen = new Set() + const errors: VisibleError[] = [] + const combinedSelector = selectors.map(({ selector }) => selector).join(',') + target.__cnVisibleErrors = errors + const sample = () => { + for (const element of document.querySelectorAll(combinedSelector)) { + if ( + !(element instanceof HTMLElement) || + !element.checkVisibility({ + checkOpacity: true, + checkVisibilityCSS: true + }) + ) + continue + const text = (element.innerText || element.textContent || '') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 300) + for (const { surface, selector } of selectors) { + if (!element.matches(selector)) continue + const key = `${surface}\u0000${text}` + if (seen.has(key)) continue + seen.add(key) + errors.push({ surface, text }) + } + } + } + const sampleNext = () => { + sample() + window.setTimeout(sampleNext, sampleIntervalMs) + } + sample() + window.setTimeout(sampleNext, sampleIntervalMs) +} + +export async function trackVisibleErrors(page: Page): Promise { + if (trackedPages.has(page)) return + const config = { + selectors: visibleErrorSurfaceSelectors, + sampleIntervalMs: visibleErrorSampleIntervalMs + } + await page.addInitScript(installVisibleErrorRecorder, config) + await page.evaluate(installVisibleErrorRecorder, config) + trackedPages.add(page) +} + +// The app's user-visible error surfaces. A regression run is green only if a +// human looking at the screen would see zero errors - not merely a clean +// console. The harness self-check asserts the overlay IS visible after a +// forced execution error, so these selectors are permanently proven live. +export function errorSurfaces(page: Page): Record { + return { + errorOverlay: page.getByTestId(TestIds.dialogs.errorOverlay), + errorDialog: page.getByTestId(TestIds.dialogs.errorDialog), + nodeRenderErrors: page.locator('.node-error'), + errorToasts: page.locator('.p-toast-message-error') + } +} + +async function surfaceTexts( + page: Page, + surface: string, + locator: Locator +): Promise { + try { + return (await locator.allInnerTexts()).map((text) => + text.replace(/\s+/g, ' ').trim().slice(0, 300) + ) + } catch (error) { + if (page.isClosed() || /has been closed/.test(String(error))) throw error + return [`${surface} present but unreadable: ${String(error)}`] + } +} + +// The suite's central invariant: a regression run is green only if every +// user-visible error surface is empty at the caller's readiness boundary. +export async function expectNoVisibleErrors( + page: Page, + context: string +): Promise { + expect( + trackedPages.has(page), + `${context}: visible-error recorder was not installed before navigation` + ).toBe(true) + const history = await page.evaluate( + () => (window as VisibleErrorWindow).__cnVisibleErrors + ) + expect( + history, + `${context}: visible-error recorder sentinel is missing` + ).toBeDefined() + expect(history, `${context}: transient visible errors`).toEqual([]) + for (const [surface, locator] of Object.entries(errorSurfaces(page))) + expect( + await surfaceTexts(page, surface, locator), + `${context}: ${surface}` + ).toEqual([]) +} diff --git a/browser_tests/fixtures/utils/litegraphUtils.ts b/browser_tests/fixtures/utils/litegraphUtils.ts index c33d7d36dcc..3b9cde016dd 100644 --- a/browser_tests/fixtures/utils/litegraphUtils.ts +++ b/browser_tests/fixtures/utils/litegraphUtils.ts @@ -51,12 +51,7 @@ export class SubgraphSlotReference { throw new Error(`${type} slot '${slotName}' has no position`) } - // Convert from offset to canvas coordinates - const canvasPos = window.app!.canvas.ds.convertOffsetToCanvas([ - slot.pos[0], - slot.pos[1] - ]) - return canvasPos + return window.app!.canvasPosToClientPos([slot.pos[0], slot.pos[1]]) }, [this.type, this.slotName] as const ) @@ -86,12 +81,10 @@ export class SubgraphSlotReference { throw new Error(`No ${type} node found in subgraph`) } - // Convert from offset to canvas coordinates - const canvasPos = window.app!.canvas.ds.convertOffsetToCanvas([ + return window.app!.canvasPosToClientPos([ node.emptySlot.pos[0], node.emptySlot.pos[1] ]) - return canvasPos }, [this.type] as const ) @@ -117,23 +110,11 @@ class NodeSlotReference { if (!node) throw new Error(`Node ${id} not found.`) const rawPos = node.getConnectionPos(type === 'input', index) - const convertedPos = - window.app!.canvas.ds!.convertOffsetToCanvas(rawPos) - - // Debug logging - convert Float64Arrays to regular arrays for visibility - console.warn( - `NodeSlotReference debug for ${type} slot ${index} on node ${id}:`, - { - nodePos: [node.pos[0], node.pos[1]], - nodeSize: [node.size[0], node.size[1]], - rawConnectionPos: [rawPos[0], rawPos[1]], - convertedPos: [convertedPos[0], convertedPos[1]], - currentGraphType: - 'inputNode' in window.app!.canvas.graph! ? 'Subgraph' : 'LGraph' - } - ) - - return convertedPos + // page.mouse needs page coords. canvasPosToClientPos applies the + // canvas transform AND the canvas element's client offset, so it + // survives pack JS injecting chrome above the canvas (e.g. rgthree's + // progress bar shifting it off (0,0)). + return window.app!.canvasPosToClientPos([rawPos[0], rawPos[1]]) }, [this.type, this.node.id, this.index] as const ) @@ -263,25 +244,17 @@ class NodeWidgetReference { } async click() { - await this.node.comfyPage.canvas.click({ - position: await this.getPosition() - }) + const pos = await this.getPosition() + await this.node.comfyPage.page.mouse.click(pos.x, pos.y) + await this.node.comfyPage.nextFrame() } async dragHorizontal(delta: number) { const pos = await this.getPosition() - const canvas = this.node.comfyPage.canvas - const canvasPos = (await canvas.boundingBox())! - await this.node.comfyPage.canvasOps.dragAndDrop( - { - x: canvasPos.x + pos.x, - y: canvasPos.y + pos.y - }, - { - x: canvasPos.x + pos.x + delta, - y: canvasPos.y + pos.y - } - ) + await this.node.comfyPage.canvasOps.dragAndDrop(pos, { + x: pos.x + delta, + y: pos.y + }) } async getValue() { @@ -362,7 +335,9 @@ export class NodeReference { modifiers?: ('Shift' | 'Control' | 'Alt' | 'Meta')[] } ): Promise { - const titlePos = await this.getTitlePosition() + const titlePos = await this.comfyPage.canvasOps.toAbsolute( + await this.getTitlePosition() + ) const target = { x: titlePos.x + delta.x, y: titlePos.y + delta.y } const modifiers = options?.modifiers ?? [] const keyboard = this.comfyPage.page.keyboard diff --git a/browser_tests/tests/customNodes/README.md b/browser_tests/tests/customNodes/README.md new file mode 100644 index 00000000000..60e2fdb369d --- /dev/null +++ b/browser_tests/tests/customNodes/README.md @@ -0,0 +1,34 @@ +# Custom-node regression suite - spec directory + +This directory holds the suite's Playwright specs, fixtures live in +`browser_tests/fixtures/customNode/`, and the break patches for the +detection-proof branch live in `detection-proof/`. + +All documentation - architecture, scope and milestones, running locally, +pack onboarding, and the detection proof - lives in the Technical Design +Doc: [docs/custom-node-regression-suite.md](../../../docs/custom-node-regression-suite.md). + +Quick start: `pnpm test:custom-nodes:local`. + +## Pinned inputs and refresh + +The gating workflow loads +[`customNodeManifest.cloud.json`](../../fixtures/data/customNodeManifest.cloud.json). +It records the exact private Cloud repository commit and YAML path used to +build the population. Git packs use full commit SHAs; registry packs use exact +published versions; ComfyUI core is pinned in `ci-tests-custom-nodes.yaml`. + +Refreshing the Cloud population is deliberate because the generated node +counts and sentinels must come from the same deployment as the source YAML: + +1. Replace `data/cloud/supported_nodes.yaml` from the reviewed Cloud commit and + update its `# Source` and `# Imported` headers. +2. Capture `/object_info` from that Cloud deployment. +3. Run `pnpm gen:cloud-manifest `. +4. Run the local CPU gate and review every count, extension, applicability, and + exact-failure delta before merging. + +The generator validates source provenance and deploy refs, joins YAML packs to +their runtime directory identities, and preserves the reviewed run-tier +workflow, runnable count, extension sentinels, and cannot-run calibration. It +does not claim to monitor the private Cloud repository automatically. diff --git a/browser_tests/tests/customNodes/allNodes.spec.ts b/browser_tests/tests/customNodes/allNodes.spec.ts new file mode 100644 index 00000000000..34246545734 --- /dev/null +++ b/browser_tests/tests/customNodes/allNodes.spec.ts @@ -0,0 +1,1863 @@ +// Every-node coverage: the suite's core contract (mounts, survives +// save/reload, calibrated model-free execution) applied to ALL eligible nodes a pack +// registers - not just the curated expectedNodes sentinels. Node lists come +// from the live backend, so a pack update is covered the moment it installs. +import type { Page } from '@playwright/test' + +import type { ComfyPage } from '@e2e/fixtures/ComfyPage' +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import type { RequiredSocket } from '@e2e/fixtures/customNode/autoRun' +import { + AUTO_RUN_WIDGET_INPUTS, + batchAutoRunnable, + CLOUD_RUN_EXCLUSIONS, + planAutoRuns, + SYNTH_PRODUCERS +} from '@e2e/fixtures/customNode/autoRun' +import { + LocalDesktopTarget, + isServerSideFault +} from '@e2e/fixtures/customNode/ComfyTarget' +import { + isForeignExecutionNoise, + unallowlistedErrors, + unallowlistedGlobalExtensionErrorsForPacks +} from '@e2e/fixtures/customNode/consoleErrorLedger' +import { failureSummary } from '@e2e/fixtures/customNode/failureReport' +import { + expectedNodeCountFor, + loadAllManifestPackNames, + loadManifest, + packIdentity +} from '@e2e/fixtures/customNode/manifest' +import { describeRunOutcome } from '@e2e/fixtures/customNode/runResult' +import { + assertPackLedgerKeys, + packLedgerFor +} from '@e2e/fixtures/customNode/packLedger' +import type { CustomNodeTier } from '@e2e/fixtures/customNode/manifest' +import type { RawNodeDef } from '@e2e/fixtures/customNode/typePairing' +import { normalizeNodeDefs } from '@e2e/fixtures/customNode/typePairing' +import { eligibleNodeTypesForTier } from '@e2e/fixtures/customNode/tierNodeExclusions' +import { + CANVAS_PREVIEW_IMAGE_PATH_PATTERN, + initializationSignalsForTypes, + matchesTopologyExpectation, + OUTPUT_TOPOLOGY_EXPECTATIONS_LITEGRAPH, + OUTPUT_TOPOLOGY_EXPECTATIONS_VUE, + pendingRestoredPreviewWidgets, + pendingRoundtripInitializations, + rendererLedgerFor, + ROUNDTRIP_INITIALIZATION_SIGNALS, + ROUNDTRIP_NODE_LOSS_EXPECTATIONS_LITEGRAPH, + ROUNDTRIP_NODE_LOSS_EXPECTATIONS_VUE, + ROUNDTRIP_VALUE_ALLOWED_INDICES_LITEGRAPH, + ROUNDTRIP_VALUE_ALLOWED_INDICES_VUE, + ROUNDTRIP_VALUE_ALLOWED_KEYS_LITEGRAPH, + ROUNDTRIP_VALUE_ALLOWED_KEYS_VUE, + staleValueDriftIndices, + staleValueDriftKeys +} from '@e2e/fixtures/customNode/valueDrift' +import { + attachPageDiagnosticEvidence, + collectConsoleErrors +} from '@e2e/fixtures/utils/consoleErrorCollector' +import { + customNodeSuiteSettings, + drainBackendToIdle, + runWithCollectedCleanup, + trackSubmittedPrompts, + waitForQueueQuiet +} from '@e2e/fixtures/utils/customNodeSuite' +import { expectNoVisibleErrors } from '@e2e/fixtures/utils/errorSurfaces' + +const target = new LocalDesktopTarget() + +// Measured optimum (deterministic across repeats, best ms/node); see PR. +const BATCH_SIZE = 24 +const AUTO_RUN_BATCH = 10 +const ROUNDTRIP_INITIALIZATION_TIMEOUT_MS = 30_000 +// The auto-run disambiguation pass (re-run one node alone) waits longer than +// the batch does: by then we are timing a single node, so we can afford to +// distinguish "slow under load" from "genuinely hung" instead of misreading a +// slow CPU run as a regression. +const SINGLE_RERUN_TIMEOUT = 60_000 +const GRID_SPACING = { x: 420, y: 360 } + +type AllNodesTier = 'S1' | 'S2' | 'S3' | 'S9' + +// Nodes unsafe to execute on a bare backend; every entry names the mechanism. +const AUTO_RUN_EXCLUDE: Record> = { + 'ComfyUI-Impact-Pack': { + ImpactRemoteBoolean: + 'remote-control widget node; its executing signal flip-flops between PASS and PARTIAL run-to-run (same class as essentials TransitionMask+)', + ImpactRemoteInt: + 'remote-control widget node; its executing signal flip-flops between PASS and PARTIAL run-to-run (same class as ImpactRemoteBoolean)', + ImpactSchedulerAdapter: + 'executing signal flip-flops between PASS and PARTIAL run-to-run (same class as essentials TransitionMask+)', + ImpactQueueTrigger: + 'queue-control node: backend execution emits impact-add-queue whenever its mode widget is on (the default), and the pack JS answers with a background app.queuePrompt whose re-entrancy refusal (app.processingQueue) then pins a bare VALIDATION_FAIL on whichever node the harness submits next', + ImpactQueueTriggerCountdown: + 'queue-control node: backend execution emits impact-add-queue while counting, and the pack JS answers with a background app.queuePrompt whose re-entrancy refusal poisons the next harness submission (same emitter chain as ImpactQueueTrigger; the pack installs no submit-time queue hook)', + ImageReceiver: + 'environment-variable execution: av.error.InvalidDataError decoding its default image on macOS, clean on Linux CI' + }, + 'ComfyUI-KJNodes': { + CreateMagicMask: + 'environment-variable execution: RuntimeError on the macOS CPU stack, clean on Linux CI', + CreateVoronoiMask: + 'environment-variable execution: RuntimeError on the macOS CPU stack, clean on Linux CI', + GenerateNoise: + 'environment-variable execution: rejected at validation locally, clean on Linux CI', + Screencap_mss: + 'captures the screen; no X display on CI runners, real display locally', + ImageGrabPIL: 'grabs the screen via PIL; OSError on headless CI runners', + LoadAndResizeImage: + 'image-combo default follows input dir contents; a non-image media file (our staged video) makes PIL error - content-variable', + PointsEditor: + 'requires its pack JS to inject the points JSON at queue time; raw defaults JSONDecodeError. Excluded unconditionally - curated-workflow candidate', + SplineEditor: + 'requires its pack JS to inject the spline JSON at queue time; raw defaults JSONDecodeError. Excluded unconditionally - curated-workflow candidate', + StringToFloatList: + 'requires its pack JS to normalize the list string at queue time; raw defaults ValueError. Excluded unconditionally - curated-workflow candidate' + }, + 'was-node-suite-comfyui': { + 'BLIP Model Loader': + 'downloads BLIP weights at execution; hangs non-interruptibly without them and would pull large models on a networked runner', + 'SAM Model Loader': + 'downloads Segment Anything weights at execution; same non-interruptible download class as BLIP', + 'MiDaS Model Loader': + 'downloads MiDaS weights via torch hub at execution; same non-interruptible download class as BLIP', + 'CLIPSeg Model Loader': + 'downloads a CLIPSeg segmentation model at execution; same non-interruptible download class as BLIP', + 'CLIPSeg Batch Masking': + 'runs CLIPSeg inference, which downloads its model on first use; same network/model-dependent class as CLIPSeg Model Loader', + 'True Random.org Number Generator': + 'fetches entropy from random.org at validation/execution; network-dependent', + 'Text Random Prompt': + 'calls the Lexica API with requests.get and no timeout at execution; network-dependent', + 'Create Video from Path': + 'invokes ffmpeg on a filesystem path; FileNotFoundError on CI runners, environment-variable', + 'Create Grid Image': + 'scans the input dir for images; ValueError when only non-image media is present - content-variable', + 'Random Number': + 'environment-variable execution: TypeError locally, clean on Linux CI', + 'Image History Loader': + 'reads WAS run history; state-dependent (KeyError on a fresh CI backend)', + 'Image Nova Filter': + 'pure-Python per-pixel loop takes minutes on a 512x512 input and does not respond to interrupt; observed jamming the queue for 20+ minutes', + 'Image Rembg (Remove Background)': + 'runs `pip install rembg` inside execute when the package is missing (WAS lazy-install); the silent network install blocks the executor non-interruptibly - observed deadlocking the queue', + 'Text Find and Replace Input': + 'infinite `while find in text` loop when find is an empty string (every empty-default run); non-interruptible pure-Python spin - upstream-report candidate', + 'Text File History Loader': + 'combo follows WAS text-file history; state-dependent (validation-fails on a fresh backend, may pass on a used one)', + 'MiDaS Depth Approximation': + 'loads MiDaS via torch.hub inside execute when no model is wired; downloads on a networked runner - same non-interruptible download class as the MiDaS loader', + 'MiDaS Mask Image': + 'loads MiDaS via torch.hub inside execute when no model is wired; downloads non-interruptibly on a networked runner (hung CI), runs clean only where the hub cache is warm', + CLIPSEG2: + 'calls transformers from_pretrained(CIDAS/clipseg-rd64-refined) inside execute when no model is wired; downloads from HuggingFace on a networked runner - same class as BLIP/SAM', + 'Image Analyze': + 'environment-variable execution: RuntimeError on the macOS CPU stack, clean on Linux CI', + 'Image Crop Face': + 'environment-variable execution: clean on macOS, AttributeError on Linux CI (OpenCV cascade lookup differs)', + 'Text Parse A1111 Embeddings': + 'environment-variable execution: FileNotFoundError on macOS, clean on Linux CI (embeddings dir handling differs)' + }, + 'ComfyUI-Custom-Scripts': { + 'LoadText|pysssss': + 'reads a text file chosen by a state-dependent combo; flips between clean and ValueError with backend content state (same class as WAS Text File History Loader)' + }, + ComfyUI_essentials: { + 'RemBGSession+': + 'initializes a rembg session that downloads its ONNX model at execution; hangs (non-interruptibly) on a backend without network/model access', + 'TransitionMask+': + 'list-expanded execution emits no per-node executing event on some runs, so the executed-set signal flip-flops between PASS and PARTIAL; mount/save-reload/connectivity tiers still cover it', + 'TransparentBGSession+': + 'ML-session initializer like RemBGSession+; sets up/downloads a background-removal model at execution, unstable on a bare backend', + 'LoadCLIPSegModels+': + 'downloads a CLIPSeg segmentation model at execution; network/model-dependent (same class as the excluded RemBG/TransparentBG loaders)' + } +} + +// Plain-typed widgets whose value is owned by pack JS: a programmatic write +// is legitimately rewritten, so set-and-stick does not apply. Keyed +// `NodeType.widgetName`; every entry names the mechanism. +const WIDGET_SET_ALLOWLIST: Record> = { + 'ComfyUI-Impact-Pack': { + 'BasicPipeToDetailerPipe.Select to add Wildcard': + 'menu-action combo: pack JS applies the chosen wildcard and resets the combo to its label', + 'BasicPipeToDetailerPipeSDXL.Select to add Wildcard': + 'menu-action combo: pack JS applies the chosen wildcard and resets the combo to its label', + 'ImpactWildcardEncode.Select to add Wildcard': + 'menu-action combo: pack JS applies the chosen wildcard to the text widget and resets the combo to its label', + 'ImpactWildcardProcessor.Select to add Wildcard': + 'menu-action combo: pack JS applies the chosen wildcard and resets the combo to its label', + 'ToDetailerPipe.Select to add Wildcard': + 'menu-action combo: pack JS applies the chosen wildcard and resets the combo to its label', + 'ToDetailerPipeSDXL.Select to add Wildcard': + 'menu-action combo: pack JS applies the chosen wildcard and resets the combo to its label', + 'EditDetailerPipe.Select to add Wildcard': + 'menu-action combo: pack JS applies the chosen wildcard and resets the combo to its label', + 'EditDetailerPipeSDXL.Select to add Wildcard': + 'menu-action combo: pack JS applies the chosen wildcard and resets the combo to its label', + 'PreviewBridge.image': + 'pack JS canonicalizes the value to its internal $nodeId-slot reference on every write', + 'PreviewBridgeLatent.image': + 'pack JS canonicalizes the value to its internal $nodeId-slot reference on every write' + }, + 'ComfyUI-KJNodes': { + 'Ideogram4PromptBuilderKJ.style_palette_data': + 'hidden serialized palette owned by the Ideogram editor', + 'Ideogram4PromptBuilderKJ.elements_data': + 'hidden serialized regions owned by the Ideogram editor', + 'Ideogram4PromptBuilderKJ.bg_brightness': + 'hidden value owned by the Ideogram editor slider', + 'Ideogram4PromptBuilderKJ.output_format': + 'hidden value owned by the Ideogram editor output menu', + 'Ideogram4PromptBuilderKJ.coord_mode': + 'hidden value owned by the Ideogram editor output menu', + 'Ideogram4PromptBuilderKJ.bbox_order': + 'hidden value owned by the Ideogram editor output menu', + 'ImageTransformKJ.bboxes': + 'serialized crop state owned by the image-transform editor', + 'PointsEditor.points_store': + 'serialized point state owned by the points editor', + 'PointsEditor.coordinates': + 'derived coordinates owned by the points editor', + 'PointsEditor.neg_coordinates': + 'derived negative coordinates owned by the points editor', + 'PointsEditor.bbox_store': + 'serialized bounding boxes owned by the points editor', + 'PointsEditor.bboxes': 'derived bounding boxes owned by the points editor', + 'SplineEditor.points_store': + 'serialized path state owned by the spline editor', + 'SplineEditor.coordinates': + 'derived sampled coordinates owned by the spline editor' + }, + 'ComfyUI-LTXVideo': { + 'LTXVSparseTrackEditor.points_store': + 'serialized track state owned by the sparse-track editor', + 'LTXVSparseTrackEditor.coordinates': + 'derived coordinates owned by the sparse-track editor' + } +} + +// Exact pack-owned serialization changes. Every node must also appear in a +// renderer-specific index or key ledger. +const ROUNDTRIP_VALUE_ALLOWLIST: Record> = { + 'ComfyUI-VideoHelperSuite': { + VHS_LoadVideo: + 'upload-button widget serializes a placeholder value fresh but is rebuilt valueless by pack JS on configure', + VHS_LoadVideoFFmpeg: + 'upload-button widget serializes a placeholder value fresh but is rebuilt valueless by pack JS on configure', + VHS_LoadImages: + 'upload-button widget serializes a placeholder value fresh but is rebuilt valueless by pack JS on configure', + VHS_LoadAudioUpload: + 'upload-button widget serializes a placeholder value fresh but is rebuilt valueless by pack JS on configure', + VHS_VAEDecodeBatched: + 'per_batch serializes null after configure (VHS ANNOTATED widget deserialization gap) - upstream-report candidate', + VHS_VAEEncodeBatched: + 'per_batch serializes null after configure (VHS ANNOTATED widget deserialization gap) - upstream-report candidate' + }, + 'ComfyUI-KJNodes': { + SplineEditor: + 'the spline editor regenerates its derived sampled coordinates from the preserved source points on configure' + }, + 'ComfyUI-LTXVideo': { + LTXVSparseTrackEditor: + 'the sparse-track editor regenerates its derived integer coordinates from the preserved source points on configure' + }, + 'ComfyUI_Fill-Nodes': { + FL_ColorPicker: + 'color widget canonicalizes the probe color and derived RGB channels on configure', + FL_ReplaceColor: + 'color widgets canonicalize the probe colors and derived RGB channels on configure' + }, + 'WhatDreamsCost-ComfyUI': { + LoadAudioUI: + 'custom audio UI clamps end and duration to the loaded file duration and normalizes absent player state on configure', + LTXDirector: + 'director UI canonicalizes timeline JSON and its derived prompt fields on configure' + }, + 'comfyui-itools': { + iToolsRegexNode: + 'pattern-picker reload replaces the regex field with the selected built-in pattern' + } +} + +interface DuplicateWidgetExpectation { + counts: Record + reason: string + restore: string +} + +const MOUNT_WIDGET_DUPLICATE_EXPECTATIONS: Record< + string, + Record +> = { + 'ComfyUI_Fill-Nodes': { + FL_TimeLine: { + counts: { + frames_per_second: 2, + interpolation_mode: 2, + ipadapter_preset: 2, + number_animation_frames: 2, + video_height: 2, + video_width: 2 + }, + reason: + 'the pinned pack JS adds six widgets already supplied by the Python node definition; Vue renders one row per unique widget identity', + restore: + 'remove this expectation when the pack stops creating the duplicate widgets' + } + } +} + +// The pack-attributed console-noise ledger moved to a shared fixture module +// (consoleErrorLedger.ts) so the curated run tier applies the same +// exceptions; this spec reads it through the shared filtering helpers above. + +const PACK_LEDGERS: Record>> = { + AUTO_RUN_WIDGET_INPUTS, + AUTO_RUN_EXCLUDE, + CLOUD_RUN_EXCLUSIONS, + MOUNT_WIDGET_DUPLICATE_EXPECTATIONS, + OUTPUT_TOPOLOGY_EXPECTATIONS_LITEGRAPH, + OUTPUT_TOPOLOGY_EXPECTATIONS_VUE, + ROUNDTRIP_NODE_LOSS_EXPECTATIONS_LITEGRAPH, + ROUNDTRIP_NODE_LOSS_EXPECTATIONS_VUE, + ROUNDTRIP_INITIALIZATION_SIGNALS, + ROUNDTRIP_VALUE_ALLOWLIST, + ROUNDTRIP_VALUE_ALLOWED_INDICES_LITEGRAPH, + ROUNDTRIP_VALUE_ALLOWED_INDICES_VUE, + ROUNDTRIP_VALUE_ALLOWED_KEYS_LITEGRAPH, + ROUNDTRIP_VALUE_ALLOWED_KEYS_VUE, + WIDGET_SET_ALLOWLIST +} + +for (const [name, ledger] of Object.entries(PACK_LEDGERS)) + assertPackLedgerKeys(name, ledger, loadAllManifestPackNames()) + +test.use({ initialSettings: customNodeSuiteSettings }) + +test.beforeEach(async ({ comfyPage }) => { + trackSubmittedPrompts(comfyPage.page) +}) + +// Leave the shared backend idle after every test so the next test's fresh +// page never connects to a still-running execution and inherits its async +// errors (see drainBackendToIdle). This is what makes the tiers order- and +// run-independent on one shared backend. +test.afterEach(async ({ comfyPage }) => { + expect( + await drainBackendToIdle(comfyPage.page, 10_000), + 'test-owned backend work did not reach idle during cleanup' + ).toBe(0) +}) + +// The widgetValueStore keys state by node id and survives graph.clear(), so +// a recreated node that reuses an id inherits the previous node's same-named +// widget values (core bug, upstream-report candidate). Every in-page builder +// below restores a monotonic id base after clear() so ids are never reused +// within a page. Inlined per evaluate - page callbacks cannot share helpers. +declare global { + interface Window { + __cnIdBase?: number + // Save/reload comparison rig; parked on the window so its stages can + // run as separate evaluates with frame yields between them. + __cnRt?: { + problems: string[] + captureInitialWidgetCounts: () => void + previewWidgetState: () => { + requiredByNode: Record + observedByNode: Record + } + snapshotAndConfigure: () => void + compare: (label: string, strict: boolean) => void + setAndStick: () => void + finish: () => { + problems: string[] + nodeLosses: string[] + topologyDrifts: Array<{ + node: string + before: number + after: number + beforeNames: string + afterNames: string + label: string + }> + valueDrifts: Record + keyDrifts: Record + } + } + } +} + +interface MountedShape { + id: string + widgetNames: string[] + inputNames: string[] + outputCount: number +} + +// null entry = createNode failed for that type. The shape feeds the +// def-vs-instance fidelity check, which is renderer-independent. +function addChunk( + page: Page, + types: string[] +): Promise> { + return page.evaluate( + ([chunk, spacingX, spacingY]) => { + window.app!.graph.clear() + window.app!.graph.last_node_id = window.__cnIdBase ?? 0 + const cols = Math.ceil(Math.sqrt(chunk.length)) + const shapes: Array<{ + id: string + widgetNames: string[] + inputNames: string[] + outputCount: number + } | null> = [] + for (const [index, type] of chunk.entries()) { + const node = window.LiteGraph!.createNode(type) + if (!node) { + shapes.push(null) + continue + } + node.pos = [ + (index % cols) * (spacingX as number), + Math.floor(index / cols) * (spacingY as number) + ] + window.app!.graph.add(node) + shapes.push({ + id: String(node.id), + widgetNames: (node.widgets ?? []).map((widget) => widget.name), + inputNames: (node.inputs ?? []).map((input) => input.name), + outputCount: (node.outputs ?? []).length + }) + } + window.__cnIdBase = window.app!.graph.last_node_id + const canvas = window.app!.canvas + const rect = canvas.canvas.getBoundingClientRect() + const width = cols * (spacingX as number) + const height = Math.ceil(chunk.length / cols) * (spacingY as number) + const scale = Math.min( + (rect.width / Math.max(width, 1)) * 0.9, + (rect.height / Math.max(height, 1)) * 0.9, + 1 + ) + canvas.ds.scale = scale + canvas.ds.offset = [60 / scale, 60 / scale] + canvas.setDirty(true, true) + return shapes + }, + [types, GRID_SPACING.x, GRID_SPACING.y] as const + ) +} + +// What the def promises the instance must materialize, in any renderer: +// every non-socketless declared input (as a widget or a socket - pack JS may +// legally convert between the two) and every declared output. Autogrow +// template inputs (COMFY_AUTOGROW_*) materialize either as the container or +// as their first `min` expansion slots, and renderers differ on which. +function declaredShape(def: RawNodeDef): { + inputNames: string[] + autogrow: Array<{ container: string; expansion: string[] }> + outputCount: number +} { + const inputNames: string[] = [] + const autogrow: Array<{ container: string; expansion: string[] }> = [] + for (const section of [def.input?.required, def.input?.optional]) + for (const [name, spec] of Object.entries(section ?? {})) { + const opts = (Array.isArray(spec) ? spec[1] : undefined) as + | { + socketless?: boolean + template?: { names?: string[]; min?: number } + } + | undefined + if (opts?.socketless) continue + if (opts?.template) { + // Expansion slots materialize dot-qualified: `container.slotName`. + autogrow.push({ + container: name, + expansion: (opts.template.names ?? []) + .slice(0, opts.template.min ?? 0) + .map((slotName) => `${name}.${slotName}`) + }) + continue + } + inputNames.push(name) + } + return { inputNames, autogrow, outputCount: (def.output ?? []).length } +} + +function packNodeKeysFromDefs( + defs: Record, + pack: string +): string[] { + return normalizeNodeDefs(defs) + .filter((node) => node.pack === pack) + .map((node) => node.type) + .sort() +} + +// Packs that live under custom_nodes/ but are not community packs under test, +// keyed by the mechanism that puts them there. Same discipline as every other +// ledger: an entry carries its reason, and anything NOT listed must have a row. +const INFRASTRUCTURE_PACKS: Record = { + ComfyUI_devtools: + 'harness pack the suite installs to observe the backend; never under test', + websocket_image_save: + 'ships inside ComfyUI core custom_nodes/ (comfyanonymous/ComfyUI), so it is core infrastructure the yaml does not list' +} + +// The per-row count assert below catches drift INSIDE a known pack. It cannot +// catch a pack the backend registers that no manifest row covers, because the +// suite only iterates rows - a newly deployed pack would land with zero +// coverage and every existing assertion would stay green. Close the set both +// ways: the live custom-node pack set must equal the manifest's. +const manifestEntries = loadManifest() +const installedManifestPacks = manifestEntries.map((entry) => entry.pack) + +test.describe('manifest covers every registered pack @custom-nodes', () => { + test('no pack registers on the backend without a manifest row', async ({ + comfyPage + }) => { + const defs = (await comfyPage.page.evaluate(() => + window.app!.api.getNodeDefs() + )) as unknown as Record + const live = new Set( + normalizeNodeDefs(defs) + .map((node) => node.pack) + .filter((pack) => pack !== 'core' && !(pack in INFRASTRUCTURE_PACKS)) + ) + const covered = new Set(manifestEntries.map((entry) => entry.pack)) + const uncovered = [...live].filter((pack) => !covered.has(pack)).sort() + expect( + uncovered, + `backend registers pack(s) with no manifest row: ${uncovered.join(', ')} - they have ZERO coverage; add the manifest row` + ).toEqual([]) + }) +}) + +type AllNodesTierRunner = { + entry: ReturnType[number] + run: ( + comfyPage: ComfyPage, + tier: AllNodesTier, + defs: Record + ) => Promise +} + +const allNodesTierRunners: AllNodesTierRunner[] = [] + +for (const entry of manifestEntries) { + const registerAllNodesTests = () => { + const expectAllNodesTier = async ( + comfyPage: ComfyPage, + selectedTier: AllNodesTier, + defs: Record + ) => { + const registeredKeys = packNodeKeysFromDefs(defs, entry.pack) + expect( + registeredKeys.length, + `${entry.pack} not installed on this backend` + ).toBeGreaterThan(0) + async function runTier( + selectedBy: AllNodesTier[], + tier: () => Promise + ) { + if (!selectedBy.includes(selectedTier)) return + await runWithCollectedCleanup(tier, [ + () => comfyPage.nodeOps.clearGraph(), + async () => { + expect( + await drainBackendToIdle(comfyPage.page, 10_000), + `${entry.pack} left test-owned backend work running` + ).toBe(0) + } + ]) + } + + await runTier(['S1', 'S2'], async () => { + // Exact-count guard: the corpus below comes from the live backend, so + // a pack silently registering fewer nodes (broken sub-import, a core + // change breaking registration) would shrink coverage while every + // remaining assertion stays green. At a fixed pin the count is + // deterministic; a delta in either direction fails until the manifest + // is deliberately recalibrated with the pin/core change that moved it. + console.log( + `custom-nodes count: ${entry.pack} = ${registeredKeys.length}` + ) + expect( + registeredKeys, + `${entry.pack} registers ${registeredKeys.length} nodes but ${expectedNodeCountFor(entry)} are expected on this backend - a pack node failed to register (or the pack changed); recalibrate only with the change that moved it` + ).toHaveLength(expectedNodeCountFor(entry)) + const keys = eligibleNodeTypesForTier( + { identity: packIdentity(entry), pack: entry.pack }, + selectedTier === 'S1' ? 'S1' : 'S2', + registeredKeys + ) + const declaredByKey = new Map( + keys.map((key) => [key, declaredShape(defs[key])]) + ) + const duplicateWidgetExpectations = packLedgerFor( + MOUNT_WIDGET_DUPLICATE_EXPECTATIONS, + entry.pack + ) + for (const ledgered of Object.keys(duplicateWidgetExpectations)) + expect( + keys, + `stale MOUNT_WIDGET_DUPLICATE_EXPECTATIONS entry: ${ledgered} is not registered by ${entry.pack}` + ).toContain(ledgered) + + const rendererPasses = + selectedTier === 'S1' + ? [false] + : selectedTier === 'S2' + ? [true] + : [false, true] + for (const vueNodesEnabled of rendererPasses) { + const outputTopologyExpectations = packLedgerFor( + rendererLedgerFor( + vueNodesEnabled, + OUTPUT_TOPOLOGY_EXPECTATIONS_LITEGRAPH, + OUTPUT_TOPOLOGY_EXPECTATIONS_VUE + ), + entry.pack + ) + const observedOutputTopologies = new Set() + for (const ledgered of Object.keys(outputTopologyExpectations)) + expect( + keys, + `stale OUTPUT_TOPOLOGY_EXPECTATIONS entry: ${ledgered} is not registered by ${entry.pack}` + ).toContain(ledgered) + const consoleErrors = collectConsoleErrors(comfyPage.page) + await comfyPage.settings.setSetting( + 'Comfy.VueNodes.Enabled', + vueNodesEnabled + ) + const failures: string[] = [] + const renderer = vueNodesEnabled ? 'vue' : 'litegraph' + for (let offset = 0; offset < keys.length; offset += BATCH_SIZE) { + const chunk = keys.slice(offset, offset + BATCH_SIZE) + const shapes = await addChunk(comfyPage.page, chunk) + await comfyPage.nextFrame() + const count = await comfyPage.nodeOps.getGraphNodesCount() + if (count !== chunk.length) + failures.push( + `chunk@${offset}: graph has ${count} of ${chunk.length} nodes` + ) + for (const [index, shape] of shapes.entries()) { + const key = chunk[index] + if (shape === null) { + failures.push(`${key}: createNode returned null`) + continue + } + // Renderer-independent fidelity: the instance must materialize + // everything the def declares. Pack JS converting a widget to a + // socket (or back) is legal; dropping the input entirely is not. + const declared = declaredByKey.get(key)! + const present = new Set([ + ...shape.widgetNames, + ...shape.inputNames + ]) + for (const name of declared.inputNames) + if (!present.has(name)) + failures.push( + `${key}: instance is missing declared input "${name}" (${renderer})` + ) + for (const { container, expansion } of declared.autogrow) + if ( + !present.has(container) && + !expansion.every((name) => present.has(name)) + ) + failures.push( + `${key}: autogrow input "${container}" materialized neither its container nor its first ${expansion.length} slot(s) (${renderer})` + ) + if (shape.outputCount < declared.outputCount) { + if ( + matchesTopologyExpectation( + outputTopologyExpectations[key], + declared.outputCount, + shape.outputCount + ) + ) + observedOutputTopologies.add(key) + else + failures.push( + `${key}: instance has ${shape.outputCount} of ${declared.outputCount} declared outputs (${renderer})` + ) + } + if (!vueNodesEnabled) continue + const visible = await comfyPage.page + .locator(`[data-node-id="${shape.id}"]`) + .waitFor({ state: 'visible', timeout: 2_000 }) + .then(() => true) + .catch(() => false) + if (!visible) failures.push(`${key}: no Vue mount`) + } + // A mount with missing widgets or slots is a broken node, not a + // pass. Extra DOM elements are tolerated; missing ones fail. + if (vueNodesEnabled) + failures.push( + ...(await comfyPage.page.evaluate( + ([chunkIds, expectedDuplicatesByNode]) => { + const problems: string[] = [] + for (const id of chunkIds) { + if (id === null) continue + const node = window.app!.graph.nodes.find( + (candidate) => String(candidate.id) === id + ) + const root = document.querySelector( + `[data-node-id="${id}"]` + ) + if (!node || !root) continue + const widgets = ( + (node.widgets ?? []) as Array<{ + advanced?: boolean + name?: string + type?: string + options?: { + advanced?: boolean + canvasOnly?: boolean + hidden?: boolean + } + }> + ).filter( + (widget) => + !!widget.type && + widget.type !== 'converted-widget' && + !widget.options?.canvasOnly && + !(widget.options?.advanced ?? widget.advanced) && + !widget.options?.hidden && + // Vue renders the seed-control combo inside its + // parent widget row, not as its own row. + widget.name !== 'control_after_generate' + ) + const domWidgets = root.querySelectorAll( + '[data-testid="node-widget"]' + ).length + const expectedDuplicates = + expectedDuplicatesByNode[node.type!] + if (expectedDuplicates) { + const widgetNameCounts: Record = {} + for (const widget of widgets) + widgetNameCounts[widget.name ?? ''] = + (widgetNameCounts[widget.name ?? ''] ?? 0) + 1 + const observedDuplicates = Object.fromEntries( + Object.entries(widgetNameCounts) + .filter(([, count]) => count > 1) + .sort(([left], [right]) => + left.localeCompare(right) + ) + ) + const expectedDuplicateCounts = Object.fromEntries( + Object.entries(expectedDuplicates.counts).sort( + ([left], [right]) => left.localeCompare(right) + ) + ) + if ( + JSON.stringify(observedDuplicates) !== + JSON.stringify(expectedDuplicateCounts) + ) + problems.push( + `${node.type}: duplicate widget identities ${JSON.stringify(observedDuplicates)} do not match ${JSON.stringify(expectedDuplicateCounts)}; ${expectedDuplicates.reason}; ${expectedDuplicates.restore}` + ) + const uniqueWidgetCount = new Set( + widgets.map((widget) => widget.name) + ).size + if (domWidgets !== uniqueWidgetCount) + problems.push( + `${node.type}: Vue mounts ${domWidgets} rows for ${uniqueWidgetCount} unique widget identities; ${expectedDuplicates.reason}; ${expectedDuplicates.restore}` + ) + } else if (domWidgets < widgets.length) + problems.push( + `${node.type}: Vue mounts ${domWidgets} of ${widgets.length} widgets` + ) + const slots = + (node.inputs ?? []).filter( + (input) => !(input as { widget?: unknown }).widget + ).length + (node.outputs ?? []).length + const domSlots = root.querySelectorAll( + '[data-testid="slot-connection-dot"]' + ).length + if (domSlots < slots) + problems.push( + `${node.type}: Vue mounts ${domSlots} of ${slots} slots` + ) + } + return problems + }, + [ + shapes.map((shape) => shape?.id ?? null), + duplicateWidgetExpectations + ] as const + )) + ) + } + consoleErrors.stop() + expect( + failures, + `VueNodes=${vueNodesEnabled}: ${JSON.stringify(failures, null, 1)}` + ).toEqual([]) + for (const ledgered of Object.keys(outputTopologyExpectations)) + expect( + [...observedOutputTopologies], + `stale OUTPUT_TOPOLOGY_EXPECTATIONS entry: ${ledgered} no longer has its exact declared-to-instance output topology with VueNodes=${vueNodesEnabled}` + ).toContain(ledgered) + const unallowlisted = unallowlistedGlobalExtensionErrorsForPacks( + installedManifestPacks, + unallowlistedErrors(entry.pack, consoleErrors.errors) + ) + const allowed = consoleErrors.errors.filter( + (error) => !unallowlisted.includes(error) + ) + if (allowed.length > 0) + console.log( + `${entry.pack}: ${allowed.length} console error(s) matched a pack-scoped extension allowlist` + ) + expect( + unallowlisted.filter((error) => !isForeignExecutionNoise(error)), + `console errors with VueNodes=${vueNodesEnabled}` + ).toEqual([]) + await expectNoVisibleErrors( + comfyPage.page, + `after all-nodes VueNodes=${vueNodesEnabled} pass` + ) + } + }) + + await runTier(['S3'], async () => { + const keys = eligibleNodeTypesForTier( + { identity: packIdentity(entry), pack: entry.pack }, + 'S3', + registeredKeys + ) + const allowedWidgets = packLedgerFor(WIDGET_SET_ALLOWLIST, entry.pack) + for (const ledgered of Object.keys(allowedWidgets)) + expect( + keys, + `stale WIDGET_SET_ALLOWLIST entry: ${ledgered} names a node not registered by ${entry.pack}` + ).toContain(ledgered.slice(0, ledgered.indexOf('.'))) + const allowedValueDrift = packLedgerFor( + ROUNDTRIP_VALUE_ALLOWLIST, + entry.pack + ) + for (const ledgered of Object.keys(allowedValueDrift)) + expect( + keys, + `stale ROUNDTRIP_VALUE_ALLOWLIST entry: ${ledgered} is not registered by ${entry.pack}` + ).toContain(ledgered) + const exactValueDriftNodes = new Set( + [ + packLedgerFor( + ROUNDTRIP_VALUE_ALLOWED_INDICES_LITEGRAPH, + entry.pack + ), + packLedgerFor(ROUNDTRIP_VALUE_ALLOWED_INDICES_VUE, entry.pack), + packLedgerFor(ROUNDTRIP_VALUE_ALLOWED_KEYS_LITEGRAPH, entry.pack), + packLedgerFor(ROUNDTRIP_VALUE_ALLOWED_KEYS_VUE, entry.pack) + ].flatMap((ledger) => Object.keys(ledger)) + ) + const initializationSignals = packLedgerFor( + ROUNDTRIP_INITIALIZATION_SIGNALS, + entry.pack + ) + for (const node of Object.keys(initializationSignals)) + expect( + keys, + `stale ROUNDTRIP_INITIALIZATION_SIGNALS entry: ${node} is not registered by ${entry.pack}` + ).toContain(node) + for (const ledgered of exactValueDriftNodes) + expect( + allowedValueDrift, + `exact roundtrip value entry ${ledgered} has no matching mechanism in ROUNDTRIP_VALUE_ALLOWLIST` + ).toHaveProperty(ledgered) + for (const ledgered of Object.keys(allowedValueDrift)) + expect( + [...exactValueDriftNodes], + `ROUNDTRIP_VALUE_ALLOWLIST entry ${ledgered} has no exact renderer contract` + ).toContain(ledgered) + // Widget values flow through the same store in both renderers, but + // only the Vue pass runs component mount/configure effects that can + // write back into that store - so the round-trip must hold under + // both. Each stage yields a frame so those effects actually flush + // before the next serialize (a single evaluate would serialize before + // any Vue component reacted). + const roundtripRenderers = [false, true] + const rendererMismatches: string[] = [] + for (const vueNodesEnabled of roundtripRenderers) { + const rawValueIndices = packLedgerFor( + rendererLedgerFor( + vueNodesEnabled, + ROUNDTRIP_VALUE_ALLOWED_INDICES_LITEGRAPH, + ROUNDTRIP_VALUE_ALLOWED_INDICES_VUE + ), + entry.pack + ) + const allowedValueIndices = Object.fromEntries( + Object.keys(rawValueIndices).map((node) => [ + node, + rawValueIndices[node].split(',').map(Number) + ]) + ) + const rawValueKeys = packLedgerFor( + rendererLedgerFor( + vueNodesEnabled, + ROUNDTRIP_VALUE_ALLOWED_KEYS_LITEGRAPH, + ROUNDTRIP_VALUE_ALLOWED_KEYS_VUE + ), + entry.pack + ) + const allowedValueKeys = Object.fromEntries( + Object.keys(rawValueKeys).map((node) => [ + node, + rawValueKeys[node].split(',') + ]) + ) + const observedValueDrift = new Map>() + const observedKeyDrift = new Map>() + const expectedNodeLosses = packLedgerFor( + rendererLedgerFor( + vueNodesEnabled, + ROUNDTRIP_NODE_LOSS_EXPECTATIONS_LITEGRAPH, + ROUNDTRIP_NODE_LOSS_EXPECTATIONS_VUE + ), + entry.pack + ) + const observedNodeLosses = new Set() + for (const ledgered of Object.keys(expectedNodeLosses)) + expect( + keys, + `stale ROUNDTRIP_NODE_LOSS_EXPECTATIONS entry: ${ledgered} is not registered by ${entry.pack}` + ).toContain(ledgered) + await comfyPage.settings.setSetting( + 'Comfy.VueNodes.Enabled', + vueNodesEnabled + ) + const consoleErrors = collectConsoleErrors(comfyPage.page) + const mismatches: string[] = [] + for (let offset = 0; offset < keys.length; offset += BATCH_SIZE) { + const chunk = keys.slice(offset, offset + BATCH_SIZE) + const chunkInitializationSignals = initializationSignalsForTypes( + initializationSignals, + chunk + ) + // Stage 1 - create the chunk and park the comparison rig on the + // window; its closures carry state across the staged evaluates. + await comfyPage.page.evaluate( + ([ + types, + packManaged, + exactValueDriftIndices, + exactValueDriftKeys, + allowedNodeLosses, + vueNodesEnabled, + canvasPreviewImagePathPattern + ]) => { + window.app!.graph.clear() + window.app!.graph.last_node_id = window.__cnIdBase ?? 0 + const created = new Map< + string, + { + type: string + widgetCount: number | null + previewWidgetNames: string[] + } + >() + const acceptedImagePathPattern = new RegExp( + canvasPreviewImagePathPattern.source, + canvasPreviewImagePathPattern.flags + ) + for (const type of types) { + const node = window.LiteGraph!.createNode(type) + if (!node) continue + window.app!.graph.add(node) + const widgets = node.widgets ?? [] + const uploadWidget = widgets.find( + (widget) => + widget.type === 'button' && widget.name === 'upload' + ) + const imageWidget = widgets.find( + (widget) => widget.name === uploadWidget?.value + ) + const expectsCanvasPreview = + !vueNodesEnabled && + node.previewMediaType === 'image' && + acceptedImagePathPattern.test(String(imageWidget?.value)) + created.set(String(node.id), { + type, + widgetCount: null, + previewWidgetNames: expectsCanvasPreview + ? ['$$canvas-image-preview'] + : [] + }) + } + window.__cnIdBase = window.app!.graph.last_node_id + const problems: string[] = [] + const topologyDrifts: Array<{ + node: string + before: number + after: number + beforeNames: string + afterNames: string + label: string + }> = [] + const valueDrifts = new Map>() + const keyDrifts = new Map>() + const nodeLosses = new Set() + // Serialized widgets_values can be an array or a named object; + // reload may legitimately APPEND entries (control_after_generate + // materializes, packs add value-driven dynamic widgets) but must + // never lose or change what was saved. + const preserves = ( + before: unknown, + after: unknown + ): boolean => { + const beforeNormalized = + Array.isArray(before) && before.length === 0 ? null : before + const afterNormalized = + Array.isArray(after) && after.length === 0 ? null : after + if (beforeNormalized === null) return true + if (Array.isArray(beforeNormalized)) + return ( + Array.isArray(afterNormalized) && + afterNormalized.length >= beforeNormalized.length && + beforeNormalized.every( + (value, index) => + JSON.stringify(value) === + JSON.stringify(afterNormalized[index]) + ) + ) + if (typeof beforeNormalized === 'object') + return ( + typeof afterNormalized === 'object' && + afterNormalized !== null && + Object.entries(beforeNormalized).every( + ([key, value]) => + JSON.stringify(value) === + JSON.stringify( + (afterNormalized as Record)[key] + ) + ) + ) + return ( + JSON.stringify(beforeNormalized) === + JSON.stringify(afterNormalized) + ) + } + const widgetNamesById = () => + new Map( + window.app!.graph.nodes.map((node) => [ + String(node.id), + (node.widgets ?? []) + .map((widget) => widget.name) + .join(',') + ]) + ) + let namesBefore = new Map() + let firstPass: ReturnType< + NonNullable['graph']['serialize'] + > | null = null + window.__cnRt = { + problems, + captureInitialWidgetCounts() { + for (const node of window.app!.graph.nodes) { + const expected = created.get(String(node.id)) + if (expected) + expected.widgetCount = (node.widgets ?? []).length + } + }, + previewWidgetState() { + const requiredByNode: Record = {} + const observedByNode: Record = {} + for (const [id, expected] of created) { + if (expected.previewWidgetNames.length === 0) continue + requiredByNode[expected.type] = + expected.previewWidgetNames + const node = window.app!.graph.nodes.find( + (candidate) => String(candidate.id) === id + ) + observedByNode[expected.type] = (node?.widgets ?? []).map( + (widget) => widget.name + ) + } + return { requiredByNode, observedByNode } + }, + snapshotAndConfigure() { + namesBefore = widgetNamesById() + firstPass = window.app!.graph.serialize() + window.app!.graph.configure(firstPass) + }, + // strict = pristine pass: reload may add dynamic widgets but + // must never shrink a node (the "widgets disappear after + // save/reload" bug class) and must preserve every value. The + // set-values pass is not strict: a changed combo/toggle + // legitimately rebuilds a dynamic node's widget set, so + // values are only compared where the topology stayed put. + compare(label: string, strict: boolean) { + const secondPass = window.app!.graph.serialize() + const namesAfter = widgetNamesById() + const byId = (pass: NonNullable) => + new Map( + (pass.nodes ?? []).map((node) => [ + String(node.id), + node + ]) + ) + const beforeNodes = byId(firstPass!) + const afterNodes = byId(secondPass) + for (const [id, expected] of created) { + const before = beforeNodes.get(id) + const after = afterNodes.get(id) + const restored = window.app!.graph.nodes.find( + (node) => String(node.id) === id + ) + if (!before || !after || !restored) { + if (allowedNodeLosses.includes(expected.type)) + nodeLosses.add(expected.type) + else + problems.push( + `${expected.type}: lost on ${label} reload` + ) + continue + } + if (after.type !== before.type) + problems.push( + `${expected.type}: type became ${String(after.type)} on ${label} reload` + ) + const widgets = (restored.widgets ?? []).length + if (expected.widgetCount === null) { + problems.push( + `${expected.type}: initial widget topology was not captured` + ) + continue + } + const topologyShrank = widgets < expected.widgetCount + if (strict && topologyShrank) + topologyDrifts.push({ + node: expected.type, + before: expected.widgetCount, + after: widgets, + beforeNames: namesBefore.get(id) ?? '', + afterNames: namesAfter.get(id) ?? '', + label + }) + if (!strict && namesBefore.get(id) !== namesAfter.get(id)) + continue + const allowedIndices = + exactValueDriftIndices[expected.type] + if ( + allowedIndices && + Array.isArray(before.widgets_values) && + Array.isArray(after.widgets_values) && + after.widgets_values.length >= + before.widgets_values.length + ) { + const changed = before.widgets_values.flatMap( + (value, index) => + JSON.stringify(value) === + JSON.stringify(after.widgets_values?.[index]) + ? [] + : [index] + ) + for (const index of changed) + if (allowedIndices.includes(index)) { + const observed = + valueDrifts.get(expected.type) ?? + new Set() + observed.add(index) + valueDrifts.set(expected.type, observed) + } + if ( + changed.every((index) => + allowedIndices.includes(index) + ) + ) + continue + } + const allowedKeys = exactValueDriftKeys[expected.type] + const beforeWidgetValues: unknown = before.widgets_values + const afterWidgetValues: unknown = after.widgets_values + if ( + allowedKeys && + typeof beforeWidgetValues === 'object' && + beforeWidgetValues !== null && + !Array.isArray(beforeWidgetValues) && + typeof afterWidgetValues === 'object' && + afterWidgetValues !== null && + !Array.isArray(afterWidgetValues) + ) { + const changed = Object.entries( + beforeWidgetValues + ).flatMap(([key, value]) => + JSON.stringify(value) === + JSON.stringify( + (afterWidgetValues as Record)[key] + ) + ? [] + : [key] + ) + for (const key of changed) + if (allowedKeys.includes(key)) { + const observed = + keyDrifts.get(expected.type) ?? new Set() + observed.add(key) + keyDrifts.set(expected.type, observed) + } + if (changed.every((key) => allowedKeys.includes(key))) + continue + } + if ( + !preserves(before.widgets_values, after.widgets_values) + ) + problems.push( + `${expected.type}: widgets_values ${JSON.stringify(before.widgets_values ?? null)} -> ${JSON.stringify(after.widgets_values ?? null)} on ${label} reload` + ) + } + }, + // Set-and-stick: every plain widget must hold a programmatic + // non-default write. Widget types owned by pack JS (editors, + // previews, annotated controls) are skipped: their values are + // not a user-writable contract. + setAndStick() { + const SETTABLE = new Set([ + 'number', + 'slider', + 'toggle', + 'text', + 'string', + 'customtext', + 'combo' + ]) + for (const node of window.app!.graph.nodes) { + const nodeType = created.get(String(node.id))?.type + if (!nodeType) continue + for (const widget of node.widgets ?? []) { + if (!SETTABLE.has(String(widget.type))) continue + if (`${nodeType}.${widget.name}` in packManaged) + continue + const target = ((): unknown => { + const options = ( + widget as { + options?: { + values?: unknown + min?: number + max?: number + step2?: number + } + } + ).options + if (typeof widget.value === 'boolean') + return !widget.value + if (typeof widget.value === 'number') { + const step = options?.step2 || 1 + const up = widget.value + step + if (options?.max === undefined || up <= options.max) + return up + const down = widget.value - step + if ( + options?.min === undefined || + down >= options.min + ) + return down + return undefined + } + if (typeof widget.value === 'string') { + if (widget.type === 'combo') + return Array.isArray(options?.values) + ? options.values.find( + (option: unknown) => option !== widget.value + ) + : undefined + return `${widget.value}_cn` + } + return undefined + })() + if (target === undefined || target === null) continue + widget.value = target as typeof widget.value + if (widget.value !== target) + problems.push( + `${nodeType}.${widget.name}: set ${JSON.stringify(target)} but widget kept ${JSON.stringify(widget.value)}` + ) + } + } + }, + finish() { + const out = { + problems: [...problems], + nodeLosses: [...nodeLosses], + topologyDrifts, + valueDrifts: Object.fromEntries( + [...valueDrifts].map(([node, indices]) => [ + node, + [...indices] + ]) + ), + keyDrifts: Object.fromEntries( + [...keyDrifts].map(([node, keys]) => [node, [...keys]]) + ) + } + window.app!.graph.clear() + return out + } + } + }, + [ + chunk, + allowedWidgets, + allowedValueIndices, + allowedValueKeys, + Object.keys(expectedNodeLosses), + vueNodesEnabled, + { + source: CANVAS_PREVIEW_IMAGE_PATH_PATTERN.source, + flags: CANVAS_PREVIEW_IMAGE_PATH_PATTERN.flags + } + ] as const + ) + await comfyPage.nextFrame() + const waitForInitialization = async () => { + await expect + .poll( + async () => { + const values = await comfyPage.page.evaluate((signals) => { + const values: Record = {} + for (const [type, signal] of Object.entries(signals)) { + const node = window.app!.graph.nodes.find( + (candidate) => candidate.type === type + ) as unknown as Record | undefined + if ( + signal.predicate === 'widget-count' || + signal.predicate === 'minimum-widget-count' + ) { + const widgets = ( + node as unknown as + | { + widgets?: unknown[] + } + | undefined + )?.widgets + values[type] = (widgets ?? []).length + } else if (signal.predicate === 'widget-value') { + const widgets = ( + node as unknown as + | { + widgets?: Array<{ + name: string + value: unknown + }> + } + | undefined + )?.widgets + values[type] = widgets?.find( + (widget) => widget.name === signal.widget + )?.value + } else { + values[type] = node?.[signal.property] + } + } + return values + }, chunkInitializationSignals) + const pendingInitialization = + pendingRoundtripInitializations( + chunkInitializationSignals, + values, + vueNodesEnabled + ) + const previewWidgetState = await comfyPage.page.evaluate( + () => window.__cnRt!.previewWidgetState() + ) + return [ + ...pendingInitialization, + ...pendingRestoredPreviewWidgets( + previewWidgetState.requiredByNode, + previewWidgetState.observedByNode + ) + ] + }, + { timeout: ROUNDTRIP_INITIALIZATION_TIMEOUT_MS } + ) + .toEqual([]) + } + await waitForInitialization() + await comfyPage.page.evaluate(() => + window.__cnRt!.captureInitialWidgetCounts() + ) + // Stage 2 - pristine snapshot + reload. + await comfyPage.page.evaluate(() => + window.__cnRt!.snapshotAndConfigure() + ) + await comfyPage.nextFrame() + await waitForInitialization() + // Stage 3 - pristine verdict, then write non-default values. + await comfyPage.page.evaluate(() => { + window.__cnRt!.compare('pristine', true) + window.__cnRt!.setAndStick() + }) + await comfyPage.nextFrame() + // Stage 4 - reload again with the written values. + await comfyPage.page.evaluate(() => + window.__cnRt!.snapshotAndConfigure() + ) + await comfyPage.nextFrame() + await waitForInitialization() + // Stage 5 - set-values verdict; collect and reset. + const result = await comfyPage.page.evaluate(() => { + window.__cnRt!.compare('set-values', false) + return window.__cnRt!.finish() + }) + mismatches.push(...result.problems) + for (const node of result.nodeLosses) observedNodeLosses.add(node) + for (const drift of result.topologyDrifts) { + mismatches.push( + `${drift.node}: widgets ${drift.before} [${drift.beforeNames}] -> ${drift.after} [${drift.afterNames}] on ${drift.label} reload` + ) + } + for (const [node, indices] of Object.entries(result.valueDrifts)) { + const observed = observedValueDrift.get(node) ?? new Set() + for (const index of indices) observed.add(index) + observedValueDrift.set(node, observed) + } + for (const [node, keys] of Object.entries(result.keyDrifts)) { + const observed = observedKeyDrift.get(node) ?? new Set() + for (const key of keys) observed.add(key) + observedKeyDrift.set(node, observed) + } + } + consoleErrors.stop() + expect( + unallowlistedGlobalExtensionErrorsForPacks( + installedManifestPacks, + unallowlistedErrors(entry.pack, consoleErrors.errors) + ).filter((error) => !isForeignExecutionNoise(error)), + `console errors during save/reload with VueNodes=${vueNodesEnabled}` + ).toEqual([]) + rendererMismatches.push( + ...mismatches.map( + (mismatch) => `VueNodes=${vueNodesEnabled}: ${mismatch}` + ) + ) + for (const ledgered of Object.keys(expectedNodeLosses)) + expect( + [...observedNodeLosses], + `stale ROUNDTRIP_NODE_LOSS_EXPECTATIONS entry: ${ledgered} now survives save/reload with VueNodes=${vueNodesEnabled}` + ).toContain(ledgered) + const staleValueIndices = staleValueDriftIndices( + allowedValueIndices, + Object.fromEntries( + [...observedValueDrift].map(([node, indices]) => [ + node, + [...indices] + ]) + ) + ) + expect( + staleValueIndices, + `stale ROUNDTRIP_VALUE_ALLOWED_INDICES entries with VueNodes=${vueNodesEnabled}: ${staleValueIndices.join(', ')}` + ).toEqual([]) + const staleValueKeys = staleValueDriftKeys( + allowedValueKeys, + Object.fromEntries( + [...observedKeyDrift].map(([node, keys]) => [node, [...keys]]) + ) + ) + expect( + staleValueKeys, + `stale ROUNDTRIP_VALUE_ALLOWED_KEYS entries with VueNodes=${vueNodesEnabled}: ${staleValueKeys.join(', ')}` + ).toEqual([]) + } + expect(rendererMismatches).toEqual([]) + await expectNoVisibleErrors(comfyPage.page, 'after save/reload sweep') + }) + + await runTier(['S9'], async () => { + const keys = registeredKeys + await comfyPage.settings.setSetting('Comfy.VueNodes.Enabled', false) + + // A prior pack's slow CPU execution can still be draining when this tier + // starts (all packs share one backend, locally and on CI alike; CI's + // unloaded runner keeps executions fast, a busy local machine may not). + // That work belongs to an earlier test's page, so it is invisible to + // drainBackendToIdle and must not be cancelled by us: wait it out + // instead. Slow-but-finite executions drain within the budget; only a + // truly wedged backend (which the exclusions already prevent) stays busy + // past it. The wait is free when the queue is already idle. + const queueBusy = await waitForQueueQuiet(comfyPage.page, 150_000) + expect( + queueBusy, + 'the backend still has a running prompt after a 150s wait - a genuinely wedged (non-interruptible) execution; restart the test backend' + ).toBe(0) + + const explicitRunExclusions = packLedgerFor( + CLOUD_RUN_EXCLUSIONS, + entry.pack + ) + const excluded = { + ...packLedgerFor(AUTO_RUN_EXCLUDE, entry.pack), + ...Object.fromEntries( + Object.entries(explicitRunExclusions).map(([node, exclusion]) => [ + node, + exclusion.reason + ]) + ) + } + for (const key of Object.keys(excluded)) + expect( + keys, + `stale AUTO_RUN_EXCLUDE entry: ${key} is not registered by ${entry.pack}` + ).toContain(key) + for (const [key, reason] of Object.entries(excluded)) + console.log( + `${entry.pack}: ${key} excluded from auto-run (${reason})` + ) + const verdicts = planAutoRuns( + defs, + keys.filter((key) => !(key in excluded)) + ) + const counts = new Map() + for (const verdict of verdicts) + counts.set(verdict.verdict, (counts.get(verdict.verdict) ?? 0) + 1) + console.log( + `${entry.pack} auto-run plan: ${[...counts.entries()] + .map(([verdict, count]) => `${verdict}=${count}`) + .join(' ')}` + ) + + const batches = batchAutoRunnable(verdicts, AUTO_RUN_BATCH) + const expectedRunnableCount = entry.expectedRunnableCount + if (expectedRunnableCount === undefined) + throw new Error(`${entry.pack} run tier has no runnable corpus`) + expect( + batches.flat(), + `${entry.pack} runnable corpus changed - inspect classifier or object_info drift before recalibrating` + ).toHaveLength(expectedRunnableCount) + const hardFailures: string[] = [] + const cannotRun = new Map() + const ranClean = new Set() + // A server-side 5xx means the backend is failing, not this batch's + // nodes: stop submitting, but keep every verdict already collected - + // real failures found before the fault take precedence over it and + // must still be reported (the abort is appended after them below). + let environmentAbort: string | undefined + batchLoop: for (const batch of batches) { + let outcome: string + try { + outcome = await runBatch(comfyPage.page, entry.pack, batch) + } catch (error) { + if (!isServerSideFault(error)) throw error + environmentAbort = `[${batch.map((verdict) => verdict.key).join(', ')}]: ${error.message} - remaining auto-runs skipped` + break + } + if (outcome === 'PASS') { + for (const verdict of batch) ranClean.add(verdict.key) + continue + } + // A jammed queue false-timeouts everything after it - stop here. + if (outcome.startsWith('HUNG_BACKEND')) { + hardFailures.push( + `[${batch.map((verdict) => verdict.key).join(', ')}]: ${outcome} - add the offender to AUTO_RUN_EXCLUDE with its mechanism` + ) + break + } + // Rerun singles so the bad node names itself, with a longer timeout + // so a slow-under-load node is not misread as a regression. + for (const verdict of batch) { + let single: string + try { + single = await runBatch( + comfyPage.page, + entry.pack, + [verdict], + SINGLE_RERUN_TIMEOUT + ) + } catch (error) { + if (!isServerSideFault(error)) throw error + environmentAbort = `${verdict.key}: ${error.message} - remaining auto-runs skipped` + break batchLoop + } + if (single === 'PASS') ranClean.add(verdict.key) + else if (single.startsWith('HUNG_BACKEND')) { + hardFailures.push( + `${verdict.key}: ${single} - add to AUTO_RUN_EXCLUDE with its mechanism` + ) + break batchLoop + } else cannotRun.set(verdict.key, single) + } + } + // Two-way reconciliation: unlisted failure = regression; listed node + // that runs clean (or is not auto-runnable) = stale entry. + const baseline = new Set(entry.cannotRunAlone ?? []) + const runnable = new Set( + batches.flatMap((batch) => batch.map((verdict) => verdict.key)) + ) + for (const [key, detail] of cannotRun) { + if (!baseline.has(key)) + hardFailures.push( + `${key}: ${detail} - not in cannotRunAlone; a regression, or a new baseline entry (attach the run log)` + ) + } + for (const key of baseline) { + if (ranClean.has(key)) + hardFailures.push( + `${key}: ran clean but is listed in cannotRunAlone - remove the stale entry, or re-derive the baseline if it predates execution_cached accounting (a PARTIAL from a warm backend cache used to read as cannot-run)` + ) + else if (!runnable.has(key)) + hardFailures.push( + `${key}: listed in cannotRunAlone but is not auto-runnable on this backend - remove the stale entry` + ) + } + console.log( + `${entry.pack} auto-ran ${ranClean.size} node(s) clean; ${cannotRun.size} cannot run alone (baseline ${baseline.size})` + ) + // Appended last: per-node failures keep precedence over the 5xx. + if (environmentAbort !== undefined) hardFailures.push(environmentAbort) + const autoRunFailureSummary = failureSummary( + `${entry.pack} auto-run failures`, + hardFailures, + 'auto-run-failures.json' + ) + if (hardFailures.length > 0) { + console.log(autoRunFailureSummary) + await attachPageDiagnosticEvidence( + test.info(), + 'auto-run-failures.json', + hardFailures + ) + } + expect(hardFailures.length === 0, autoRunFailureSummary).toBe(true) + }) + } + + allNodesTierRunners.push({ entry, run: expectAllNodesTier }) + } + + registerAllNodesTests() +} + +const tiers: Array<[AllNodesTier, string]> = [ + ['S1', 'every enrolled registered node mounts on the canvas renderer'], + ['S2', 'every enrolled registered node mounts on the DOM renderer'], + ['S3', 'enrolled registered-node save/reload outcomes match exact contracts'], + ['S9', 'calibrated model-free node corpus executes'] +] + +// The capability each tier needs from a pack's manifest row. Mounting and +// save/reload are frontend work every loadable pack owes; executing a node is +// not, and the row says so. These tiers used to sweep whatever the backend +// registered, which meant S9 executed every node of every installed pack no +// matter what its row asked for - 17.6 of one shard's 21.1 minutes on run +// 31842166428, for packs that declare load and connectivity only. +const TIER_REQUIRES: Record = { + S1: 'load', + S2: 'load', + S3: 'load', + S9: 'run' +} + +const runnersForTier = (tier: AllNodesTier): AllNodesTierRunner[] => + allNodesTierRunners.filter((runner) => + runner.entry.tiers.includes(TIER_REQUIRES[tier]) + ) + +test.describe('all nodes by tier @custom-nodes', () => { + for (const [tier, title] of tiers) { + const tierRunners = runnersForTier(tier) + // Registered only when a pack in this slice asks for it: a tier with no + // packs would otherwise be a skip, and the gate forbids skips because a + // skip is how lost coverage reads as a pass. + if (tierRunners.length === 0) continue + + test(`${tier}: ${title}`, async ({ comfyPage }) => { + // Run 32066027848 measured shard maxima of 14s/48s/74s for S1/S2/S3. + // Scale with the exact corpus and retain at least 4x observed headroom; + // S9 instead budgets each possible single-node diagnostic rerun. + const nodeCount = tierRunners.reduce( + (sum, runner) => sum + expectedNodeCountFor(runner.entry), + 0 + ) + const timeoutMs = + tier === 'S1' + ? 30_000 + nodeCount * 120 + : tier === 'S2' + ? 30_000 + nodeCount * 350 + : tier === 'S3' + ? 30_000 + nodeCount * 500 + : 60_000 + + tierRunners.reduce( + (sum, runner) => + sum + (runner.entry.expectedRunnableCount ?? 0) * 30_000, + 0 + ) + test.setTimeout(timeoutMs) + await comfyPage.page.evaluate((activeTier) => { + Object.assign(globalThis, { + __COMFY_CUSTOM_NODE_DETECTION_PROOF_TIER__: activeTier + }) + }, tier) + const pageId = await comfyPage.page.evaluate(() => { + const key = '__customNodeTierPageId' + const existing = sessionStorage.getItem(key) + if (existing) return existing + const created = crypto.randomUUID() + sessionStorage.setItem(key, created) + return created + }) + console.warn( + `[tier-session] pid=${process.pid} tier=${tier} pageId=${pageId}` + ) + + const defs = (await comfyPage.page.evaluate(() => + window.app!.api.getNodeDefs() + )) as unknown as Record + const failures: string[] = [] + for (const runner of tierRunners) { + let result = 'pass' + try { + await runner.run(comfyPage, tier, defs) + } catch (error) { + result = 'fail' + const errors = + error instanceof AggregateError ? error.errors : [error] + failures.push( + `[${runner.entry.pack}] ${errors + .map((entry) => + entry instanceof Error ? entry.message : String(entry) + ) + .join('\n')}` + ) + } + console.log( + `[tier-pack] tier=${tier} pack=${runner.entry.pack} result=${result}` + ) + } + const attachment = `${tier.toLowerCase()}-failures.json` + if (failures.length > 0) + await attachPageDiagnosticEvidence(test.info(), attachment, failures) + expect( + failures.length === 0, + failureSummary(`${tier} pack failures`, failures, attachment) + ).toBe(true) + }) + } +}) + +async function runBatch( + page: Page, + pack: string, + batch: Array<{ + key: string + needsPreviewSink?: boolean + requiredSockets?: RequiredSocket[] + }>, + timeoutMs: number = 20_000 +): Promise { + const batchWithInputs = batch.map((spec) => ({ + ...spec, + widgetInputs: packLedgerFor(AUTO_RUN_WIDGET_INPUTS, pack)[spec.key] + })) + const { ids, allIds, nodeIdByKey, sinkIdByKey } = await page.evaluate( + ([nodes, producers, spacingY]) => { + window.app!.graph.clear() + window.app!.graph.last_node_id = window.__cnIdBase ?? 0 + const ids: string[] = [] + const allIds: string[] = [] + const nodeIdByKey: Record = {} + const sinkIdByKey: Record = {} + for (const [index, spec] of nodes.entries()) { + const node = window.LiteGraph!.createNode(spec.key) + if (!node) continue + node.pos = [0, index * (spacingY as number)] + window.app!.graph.add(node) + ids.push(String(node.id)) + allIds.push(String(node.id)) + nodeIdByKey[spec.key] = String(node.id) + for (const [name, value] of Object.entries(spec.widgetInputs ?? {})) { + const widget = node.widgets?.find( + (candidate) => candidate.name === name + ) + if (!widget) + throw new Error( + `${spec.key}: required fixture widget ${name} is missing` + ) + widget.value = value + } + for (const [socketIndex, socket] of ( + spec.requiredSockets ?? [] + ).entries()) { + const inputIndex = node.inputs.findIndex( + (input) => input.name === socket.name + ) + // Pack JS may have rebuilt the socket as widget-only; the widget + // default then applies and no wire is needed. + if (inputIndex < 0) continue + const producer = producers[socket.type] + const producerNode = window.LiteGraph!.createNode(producer.nodeType) + if (!producerNode) continue + producerNode.pos = [ + -420 - socketIndex * 40, + index * (spacingY as number) + socketIndex * 90 + ] + window.app!.graph.add(producerNode) + allIds.push(String(producerNode.id)) + producerNode.connect(producer.outputIndex, node, inputIndex) + } + if (spec.needsPreviewSink) { + const sink = window.LiteGraph!.createNode('PreviewAny')! + sink.pos = [460, index * (spacingY as number)] + window.app!.graph.add(sink) + node.connect(0, sink, 0) + sinkIdByKey[spec.key] = String(sink.id) + allIds.push(String(sink.id)) + } + } + window.__cnIdBase = window.app!.graph.last_node_id + return { ids, allIds, nodeIdByKey, sinkIdByKey } + }, + [batchWithInputs, SYNTH_PRODUCERS, GRID_SPACING.y] as const + ) + // Batch default 20s = hung; the single re-run pass raises it (see caller) + // so a node that is merely slow under load is not misread as a regression. + const result = await target.runWorkflow(page, { + expectedNodeIds: ids, + graphNodeIds: allIds, + proofOutputNodeByExpectedNode: Object.fromEntries( + Object.entries(sinkIdByKey).map(([key, sinkId]) => [ + nodeIdByKey[key], + sinkId + ]) + ), + timeoutMs + }) + if (result.outcome === 'TIMEOUT') { + // A slow-but-finite CPU node (seam carving, heavy image ops) can exceed + // the batch budget under load; give the backend time to drain before + // calling it hung, so the caller can re-run it alone with the longer + // single timeout. Only a truly non-interruptible execution stays busy + // past this, and that genuinely needs a backend restart. + if ((await drainBackendToIdle(page, 90_000)) !== 0) + return 'HUNG_BACKEND (non-interruptible execution; backend restart required)' + } + if (result.outcome === 'PASS') { + // Executing-event coverage proves the node ran; the sink payload proves + // its output actually flowed. OUTPUT_NODE targets are their own terminus + // and promise no ui payload, so only PreviewAny sinks are asserted. + const silent = batch + .filter((spec) => { + const sinkId = sinkIdByKey[spec.key] + return sinkId !== undefined && result.outputsByNode[sinkId] == null + }) + .map((spec) => spec.key) + if (silent.length > 0) + return `NO_OUTPUT (PreviewAny sink emitted no payload for: ${silent.join(', ')})` + return 'PASS' + } + if (result.outcome === 'VALIDATION_FAIL' && result.clientError) + return `VALIDATION_FAIL (client threw: ${result.clientError.slice(0, 200)})` + return describeRunOutcome(result) +} diff --git a/browser_tests/tests/customNodes/autoRun.pure.spec.ts b/browser_tests/tests/customNodes/autoRun.pure.spec.ts new file mode 100644 index 00000000000..8e0367c22df --- /dev/null +++ b/browser_tests/tests/customNodes/autoRun.pure.spec.ts @@ -0,0 +1,312 @@ +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import { + batchAutoRunnable, + classifyAutoRunnable, + planAutoRuns +} from '@e2e/fixtures/customNode/autoRun' + +const SYNTH = new Set([ + 'IMAGE', + 'LATENT', + 'MASK', + 'INT', + 'FLOAT', + 'STRING', + 'BOOLEAN', + '*' +]) + +test.describe('autoRun classifier', () => { + test('widget-only node with outputs is runnable via a PreviewAny sink', () => { + const verdict = classifyAutoRunnable( + 'IntConstant', + { + input: { required: { value: ['INT', { default: 0 }] } }, + output: ['INT'], + output_node: false + }, + SYNTH + ) + expect(verdict.verdict).toBe('AUTO_RUNNABLE') + expect(verdict.needsPreviewSink).toBe(true) + }) + + test('widget-only OUTPUT_NODE runs standalone', () => { + const verdict = classifyAutoRunnable( + 'ShowValue', + { + input: { + required: { + text: ['STRING', {}], + mode: [['raw value', 'tensor shape']] + } + }, + output: [], + output_node: true + }, + SYNTH + ) + expect(verdict.verdict).toBe('AUTO_RUNNABLE') + expect(verdict.needsPreviewSink).toBe(false) + }) + + test('synthesizable sockets make a node CHAINABLE with its socket list', () => { + const verdict = classifyAutoRunnable( + 'MaskComposite', + { + input: { + required: { + destination: ['MASK'], + source: ['MASK'], + x: ['INT', { default: 0 }], + operation: [['multiply', 'add']] + } + }, + output: ['MASK'], + output_node: false + }, + SYNTH + ) + expect(verdict.verdict).toBe('CHAINABLE') + expect(verdict.requiredSockets).toEqual([ + { name: 'destination', type: 'MASK' }, + { name: 'source', type: 'MASK' } + ]) + expect(verdict.needsPreviewSink).toBe(true) + }) + + test('a union socket resolves to its first synthesizable member', () => { + const verdict = classifyAutoRunnable( + 'UnionConsumer', + { + input: { required: { pixels: ['VAE,IMAGE'] } }, + output: ['IMAGE'], + output_node: false + }, + SYNTH + ) + expect(verdict.verdict).toBe('CHAINABLE') + // The MEMBER is pushed, not the union string - the chain builder + // synthesizes a producer for exactly this type. + expect(verdict.requiredSockets).toEqual([{ name: 'pixels', type: 'IMAGE' }]) + }) + + test('a union socket with no synthesizable member means NEEDS_WIRES', () => { + const verdict = classifyAutoRunnable( + 'UnionNeedsWires', + { + input: { required: { model: ['MODEL,VAE'] } }, + output: ['IMAGE'], + output_node: false + }, + SYNTH + ) + expect(verdict.verdict).toBe('NEEDS_WIRES') + expect(verdict.reason).toContain('MODEL,VAE') + }) + + test('a socket with no model-free producer means NEEDS_WIRES', () => { + const verdict = classifyAutoRunnable( + 'VaeDecode', + { + input: { required: { samples: ['LATENT'], vae: ['VAE'] } }, + output: ['IMAGE'], + output_node: false + }, + SYNTH + ) + expect(verdict.verdict).toBe('NEEDS_WIRES') + expect(verdict.reason).toContain('vae') + }) + + test('forceInput STRING is a socket but STRING is synthesizable', () => { + const verdict = classifyAutoRunnable( + 'TextSink', + { + input: { required: { text: ['STRING', { forceInput: true }] } }, + output: ['STRING'], + output_node: true + }, + SYNTH + ) + expect(verdict.verdict).toBe('CHAINABLE') + expect(verdict.requiredSockets).toEqual([{ name: 'text', type: 'STRING' }]) + }) + + test('a declared widgetType makes an unproducible type a widget', () => { + const declared = classifyAutoRunnable( + 'PainterLike', + { + input: { + required: { mask: ['FILE_3D', { default: '', widgetType: 'STRING' }] } + }, + output: ['IMAGE'], + output_node: false + }, + SYNTH + ) + expect(declared.verdict).toBe('AUTO_RUNNABLE') + + const undeclared = classifyAutoRunnable( + 'PainterLike', + { + input: { required: { mask: ['FILE_3D', { default: '' }] } }, + output: ['IMAGE'], + output_node: false + }, + SYNTH + ) + expect(undeclared.verdict).toBe('NEEDS_WIRES') + }) + + test('forceInput beats a declared widgetType', () => { + const verdict = classifyAutoRunnable( + 'ForcedPainterLike', + { + input: { + required: { + mask: ['FILE_3D', { widgetType: 'STRING', forceInput: true }] + } + }, + output: ['IMAGE'], + output_node: false + }, + SYNTH + ) + expect(verdict.verdict).toBe('NEEDS_WIRES') + expect(verdict.reason).toContain('mask') + }) + + test('an empty required combo means NEEDS_MODELS', () => { + const verdict = classifyAutoRunnable( + 'CheckpointLoader', + { + input: { required: { ckpt_name: [[]] } }, + output: ['MODEL'], + output_node: false + }, + SYNTH + ) + expect(verdict.verdict).toBe('NEEDS_MODELS') + expect(verdict.reason).toContain('ckpt_name') + }) + + // Census-derived: transformed (V2-schema) defs carry combos as the string + // 'COMBO' with options in the opts object - the classifier must not read + // that as an unproducible socket type. + test('a V2-form combo with options is a widget', () => { + const verdict = classifyAutoRunnable( + 'LatentConcatLike', + { + input: { + required: { + dim: ['COMBO', { multiselect: false, options: ['x', '-x', 'y'] }] + } + }, + output: ['LATENT'], + output_node: false + }, + SYNTH + ) + expect(verdict.verdict).toBe('AUTO_RUNNABLE') + }) + + // Census-derived (DevToolsNodeWithOutputCombo.subset_options): a combo + // carrying forceInput is a socket in ANY form - no widget materializes, + // so its option list cannot satisfy the input. + test('forceInput on a list-form combo is a socket, not a widget', () => { + const verdict = classifyAutoRunnable( + 'OutputComboLike', + { + input: { + required: { subset_options: [['A', 'B'], { forceInput: true }] } + }, + output: ['COMBO'], + output_node: false + }, + SYNTH + ) + expect(verdict.verdict).toBe('NEEDS_WIRES') + expect(verdict.reason).toContain('subset_options') + }) + + test('a V2-form combo with no static options means NEEDS_MODELS', () => { + for (const spec of [ + ['COMBO', { multiselect: false, options: [] }], + ['COMBO', { remote: { route: '/internal/files/output' } }] + ]) { + const verdict = classifyAutoRunnable( + 'LoadImageOutputLike', + { + input: { required: { image: spec } }, + output: ['IMAGE'], + output_node: false + }, + SYNTH + ) + expect(verdict.verdict).toBe('NEEDS_MODELS') + expect(verdict.reason).toContain('image') + } + }) + + test('no outputs and not an OUTPUT_NODE means NO_OBSERVABLE_OUTPUT', () => { + const verdict = classifyAutoRunnable( + 'SideEffectOnly', + { + input: { required: { value: ['INT', {}] } }, + output: [], + output_node: false + }, + SYNTH + ) + expect(verdict.verdict).toBe('NO_OBSERVABLE_OUTPUT') + }) + + test('optional socket inputs do not block auto-running', () => { + const verdict = classifyAutoRunnable( + 'MathWithOptionalAny', + { + input: { + required: { expression: ['STRING', {}] }, + optional: { a: ['*'] } + }, + output: ['INT', 'FLOAT'], + output_node: true + }, + SYNTH + ) + expect(verdict.verdict).toBe('AUTO_RUNNABLE') + }) + + test('planAutoRuns validates producers against defs and batches runnables', () => { + const defs = { + A: { + input: { required: { v: ['INT', {}] } }, + output: ['INT'], + output_node: false + }, + B: { + input: { required: { x: ['SEGS'] } }, + output: ['SEGS'], + output_node: false + }, + C: { + input: { required: { img: ['IMAGE'] } }, + output: ['IMAGE'], + output_node: false + }, + EmptyImage: { input: { required: {} }, output: ['IMAGE'] } + } + const verdicts = planAutoRuns(defs, ['A', 'B', 'C']) + expect(verdicts.map((verdict) => verdict.verdict)).toEqual([ + 'AUTO_RUNNABLE', + 'NEEDS_WIRES', + 'CHAINABLE' + ]) + const batches = batchAutoRunnable(verdicts, 1) + expect(batches.map((batch) => batch[0].key)).toEqual(['A', 'C']) + }) +}) diff --git a/browser_tests/tests/customNodes/connectivity.spec.ts b/browser_tests/tests/customNodes/connectivity.spec.ts new file mode 100644 index 00000000000..7a77de4c9e0 --- /dev/null +++ b/browser_tests/tests/customNodes/connectivity.spec.ts @@ -0,0 +1,918 @@ +import type { Page } from '@playwright/test' + +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import { + customNodeSuiteSettings, + drainBackendToIdle, + trackSubmittedPrompts +} from '@e2e/fixtures/utils/customNodeSuite' +import { + isForeignExecutionNoise, + staleRequiredConnectivityErrorRulesForPacks, + unallowlistedConnectivityErrorsForPacks, + unallowlistedErrorsForPacks +} from '@e2e/fixtures/customNode/consoleErrorLedger' +import { + connectivityExpectations, + pairEndpointOwnershipIssues, + pairExpectationKeys, + pairExpectationNodeTypes +} from '@e2e/fixtures/customNode/connectivityExpectations' +import { failureSummary } from '@e2e/fixtures/customNode/failureReport' +import { + loadAllManifestPackNames, + loadFullManifest, + loadManifest, + packIdentity +} from '@e2e/fixtures/customNode/manifest' +import { eligibleNodeTypesForTier } from '@e2e/fixtures/customNode/tierNodeExclusions' +import type { + ConnectivityOutcome, + PlannedPair, + RawNodeDef +} from '@e2e/fixtures/customNode/typePairing' +import { + normalizeNodeDefs, + planPairs +} from '@e2e/fixtures/customNode/typePairing' +import { + attachPageDiagnosticEvidence, + collectConsoleErrors +} from '@e2e/fixtures/utils/consoleErrorCollector' +import { + expectNoVisibleErrors, + trackVisibleErrors +} from '@e2e/fixtures/utils/errorSurfaces' +import { fitToViewInstant } from '@e2e/fixtures/utils/fitToView' + +// Budget the sweep per pair instead of flat: the corpus grows with every pack +// added, and a flat cap silently becomes a hang the day it stops fitting. Run +// 30961895204 swept 16832 pairs and did not finish inside a flat 120s cap, so +// the real rate is above 6.5ms/pair; the multiplier below carries margin over +// that floor and the sweep logs its actual rate so it can be tightened. +const PLAN_SETUP_MS = 120_000 +const SWEEP_MS_PER_PAIR = 40 +const ISOLATED_MS_PER_PAIR = PLAN_SETUP_MS +const DYNAMIC_CLEANUP_SETTLE_MS = 50 +const PAIRS_PER_BATCH = 100 +// Same discipline for the drag pass, whose edge list grows with every +// connectivity pack: one drag per edge per renderer. This test carried a flat +// 120s cap over today's 6 packs (16 drags) until the since-removed cloud +// surface uncapped it, so 15s per drag is that budget with margin. +const DRAG_MS_PER_DRAG = 15_000 +const { + isolatedNodeTypes, + connectRejected: allConnectRejectedGroups, + deterministicSlotContractMismatch: allDeterministicSlotContractMismatchGroups, + dynamicSlotCleanupStalled: allDynamicSlotCleanupStalledGroups, + roundtripLost: allRoundtripLostGroups, + zeroPairDragExpectedNodeCounts +} = connectivityExpectations +const fullManifestPacks = new Set( + loadFullManifest().map((entry) => entry.pack.toLowerCase()) +) +const appliesToSelectedManifest = ({ pack }: { pack: string }) => + fullManifestPacks.has(pack.toLowerCase()) +const connectRejectedGroups = allConnectRejectedGroups.filter( + appliesToSelectedManifest +) +const deterministicSlotContractMismatchGroups = + allDeterministicSlotContractMismatchGroups.filter(appliesToSelectedManifest) +const dynamicSlotCleanupStalledGroups = + allDynamicSlotCleanupStalledGroups.filter(appliesToSelectedManifest) +const roundtripLostGroups = allRoundtripLostGroups.filter( + appliesToSelectedManifest +) +const connectRejected = pairExpectationKeys(connectRejectedGroups) +const deterministicSlotContractMismatch = pairExpectationKeys( + deterministicSlotContractMismatchGroups +) +const dynamicSlotCleanupStalled = pairExpectationKeys( + dynamicSlotCleanupStalledGroups +) +const roundtripLost = pairExpectationKeys(roundtripLostGroups) +const requiredPairKeys = [ + ...connectRejected, + ...deterministicSlotContractMismatch, + ...dynamicSlotCleanupStalled, + ...roundtripLost +] +const requiredEndpointNodeTypes = pairExpectationNodeTypes([ + ...connectRejectedGroups, + ...deterministicSlotContractMismatchGroups, + ...dynamicSlotCleanupStalledGroups, + ...roundtripLostGroups +]) + +test.use({ initialSettings: customNodeSuiteSettings }) + +test.beforeEach(async ({ comfyPage }) => { + trackSubmittedPrompts(comfyPage.page) +}) + +// This spec queues no prompt of its own, so the drain returns without a +// round-trip; it stays as the guard for pack JS that queues one behind our +// back, which would otherwise run on into the next test. +test.afterEach(async ({ comfyPage }) => { + expect( + await drainBackendToIdle(comfyPage.page, 10_000), + 'connectivity probe left test-owned backend work running' + ).toBe(0) +}) + +function isEntryInstalled( + nodeTypes: Set, + entry: { expectedNodes: string[] } +): boolean { + return entry.expectedNodes.every((type) => nodeTypes.has(type)) +} + +interface PairResult { + key: string + outcome: string + detail?: string +} + +async function runPairsInIsolatedPages( + page: Page, + pairs: PlannedPair[] +): Promise<{ results: PairResult[]; errors: string[] }> { + const results: PairResult[] = [] + const errors: string[] = [] + for (const pair of pairs) { + const pairKey = `${pair.producer.nodeType}.${pair.producer.slotName} -> ${pair.consumer.nodeType}.${pair.consumer.slotName}` + const pairStart = Date.now() + console.log(`connectivity isolated pair: ${pairKey} starting`) + const probe = await page.context().newPage() + try { + await trackVisibleErrors(probe) + await probe.goto(page.url()) + await probe.waitForFunction( + ([producerType, consumerType]) => + window.app?.extensionManager !== undefined && + window.LiteGraph?.registered_node_types[producerType] !== undefined && + window.LiteGraph.registered_node_types[consumerType] !== undefined, + [pair.producer.nodeType, pair.consumer.nodeType], + { timeout: 60_000 } + ) + await expectNoVisibleErrors( + probe, + `before isolated pair ${pair.producer.nodeType} -> ${pair.consumer.nodeType}` + ) + const consoleErrors = collectConsoleErrors(probe) + try { + const pairResults = await evaluatePairs(probe, [pair], { + resetAfter: false, + stalledCleanupKeys: dynamicSlotCleanupStalled + }) + results.push(...pairResults) + await expectNoVisibleErrors( + probe, + `after isolated pair ${pair.producer.nodeType} -> ${pair.consumer.nodeType}` + ) + } finally { + consoleErrors.stop() + errors.push(...consoleErrors.errors) + } + } finally { + if (!probe.isClosed()) await probe.close() + console.log( + `connectivity isolated pair: ${pairKey} completed in ${Date.now() - pairStart}ms` + ) + } + } + return { results, errors } +} + +const connectivityEntries = loadManifest().filter((entry) => + entry.tiers.includes('connectivity') +) +const activeIsolatedNodeTypes = Object.fromEntries( + Object.entries(isolatedNodeTypes).filter(([, isolation]) => + connectivityEntries.some((entry) => entry.pack === isolation.pack) + ) +) + +test('connectivity: representative edges cover every enrolled pairable slot through model, serialize, and prompt conversion @custom-nodes', async ({ + comfyPage +}) => { + test.setTimeout(PLAN_SETUP_MS) + const defs = (await comfyPage.page.evaluate(() => + window.app!.api.getNodeDefs() + )) as unknown as Record + const registeredNodes = normalizeNodeDefs(defs) + for (const [nodeType, isolation] of Object.entries(activeIsolatedNodeTypes)) { + const node = registeredNodes.find( + (candidate) => candidate.type === nodeType + ) + expect( + node?.pack, + `${nodeType} isolation is stale because the node is not registered` + ).toBe(isolation.pack) + console.log(`connectivity: isolating ${nodeType} - ${isolation.reason}`) + } + + // Pack-specific expectations apply only where the pack is installed; on a + // backend without it (e.g. a generic CI runner) the core sweep still runs + // and the absence is reported, never fake-failed or fake-passed. + const nodeTypes = new Set(registeredNodes.map((node) => node.type)) + const installedEntries = connectivityEntries.filter((entry) => + isEntryInstalled(nodeTypes, entry) + ) + for (const entry of connectivityEntries) + if (!installedEntries.includes(entry)) + console.log(`connectivity: ${entry.pack} not installed on this backend`) + // Corpus = every node the installed packs register, from the live backend. + const installedPacks = new Set(installedEntries.map((entry) => entry.pack)) + const eligibleTypesByPack = new Map( + installedEntries.map((entry) => [ + entry.pack, + new Set( + eligibleNodeTypesForTier( + { identity: packIdentity(entry), pack: entry.pack }, + 'S4', + registeredNodes + .filter((node) => node.pack === entry.pack) + .map((node) => node.type) + ) + ) + ]) + ) + const nodes = registeredNodes.filter( + (node) => + !installedPacks.has(node.pack) || + eligibleTypesByPack.get(node.pack)?.has(node.type) + ) + const packTypes = nodes + .filter((node) => installedPacks.has(node.pack)) + .map((node) => node.type) + expect( + pairEndpointOwnershipIssues( + requiredEndpointNodeTypes, + nodes, + installedPacks + ), + 'ledgered connectivity endpoint attribution' + ).toEqual([]) + const knownNodeTypes = new Set([ + ...nodes.map((node) => node.type), + ...requiredEndpointNodeTypes + ]) + const plan = planPairs(nodes, packTypes, requiredPairKeys, knownNodeTypes) + const isolatedTypes = new Set(Object.keys(activeIsolatedNodeTypes)) + const isolatedPairs = plan.pairs.filter( + (pair) => + isolatedTypes.has(pair.producer.nodeType) || + isolatedTypes.has(pair.consumer.nodeType) + ) + const sharedPairs = plan.pairs.filter( + (pair) => + !isolatedTypes.has(pair.producer.nodeType) && + !isolatedTypes.has(pair.consumer.nodeType) + ) + + expect(plan.pairs.length, 'pairing produced no edges').toBeGreaterThan(0) + expect( + plan.unknownShapes, + 'installed-pack slots with unrecognized object_info shapes' + ).toEqual([]) + expect( + plan.requiredPairIssues, + 'ledgered connectivity pair can no longer be planned exactly' + ).toEqual([]) + for (const nodeType of isolatedTypes) + expect( + isolatedPairs.some( + (pair) => + pair.producer.nodeType === nodeType || + pair.consumer.nodeType === nodeType + ), + `${nodeType} has no isolated pairs - remove its stale isolation entry` + ).toBe(true) + console.log( + `connectivity plan: ${plan.pairs.length} pairs (${isolatedPairs.length} isolated), ${plan.orphans.length} orphan slots, ${plan.wildcards.length} wildcard + ${plan.combos.length} combo slots (excluded by design), ${plan.unknownShapes.length} unknown-shape slots (recorded: ${plan.unknownShapes.join('; ') || 'none'})` + ) + + for (const entry of installedEntries) { + const contributes = plan.pairs.some( + (pair) => + pair.producer.pack === entry.pack || pair.consumer.pack === entry.pack + ) + expect( + contributes, + `${entry.pack} contributes no pairs - corpus or pack attribution broke` + ).toBe(true) + } + + // The breadth sweep runs under one renderer by design: it exercises + // graph-API link creation, the real isValidConnection veto, and + // serialize/configure survival - all renderer-independent paths (widget + // values and links flow through the same stores in both renderers). The + // curated drag test below covers real pointer wiring under BOTH renderers. + const consoleErrors = collectConsoleErrors(comfyPage.page) + test.setTimeout( + PLAN_SETUP_MS + + sharedPairs.length * SWEEP_MS_PER_PAIR + + isolatedPairs.length * ISOLATED_MS_PER_PAIR + ) + const sweepStart = Date.now() + const sharedStart = Date.now() + const sharedResults = await runPairsInPage(comfyPage.page, sharedPairs) + console.log( + `connectivity shared sweep: ${sharedPairs.length} pairs in ${Date.now() - sharedStart}ms` + ) + const isolated = await runPairsInIsolatedPages(comfyPage.page, isolatedPairs) + const results = [...sharedResults, ...isolated.results] + const sweepMs = Date.now() - sweepStart + consoleErrors.stop() + expect( + results, + 'the executor must return one outcome for every planned pair' + ).toHaveLength(plan.pairs.length) + console.log( + `connectivity sweep: ${plan.pairs.length} pairs in ${sweepMs}ms (${(sweepMs / plan.pairs.length).toFixed(1)}ms/pair)` + ) + const failures = results.filter( + (result) => + result.outcome !== ('PASS' satisfies ConnectivityOutcome) && + !( + result.outcome === ('CONNECT_REJECTED' satisfies ConnectivityOutcome) && + connectRejected.includes(result.key) + ) && + !( + result.outcome === + ('DYNAMIC_SLOT_CLEANUP_STALLED' satisfies ConnectivityOutcome) && + dynamicSlotCleanupStalled.includes(result.key) + ) && + !( + result.outcome === ('ROUNDTRIP_LOST' satisfies ConnectivityOutcome) && + roundtripLost.includes(result.key) + ) && + !( + result.outcome === + ('SLOT_CONTRACT_MISMATCH' satisfies ConnectivityOutcome) && + deterministicSlotContractMismatch.includes(result.key) + ) + ) + const passed = results.filter((result) => result.outcome === 'PASS').length + console.log(`connectivity sweep: ${passed}/${results.length} pairs PASS`) + // Ahead of the console gate below so the tier's own outcome signal is the + // failure message, not console noise a pack emitted alongside it. + expect(failures, JSON.stringify(failures, null, 1)).toEqual([]) + expect(passed).toBeGreaterThan(0) + + // Routed through the pack console ledger scoped to the packs actually in + // the corpus (the escape hatch this assert always documented): a KJNodes + // SplineEditor creation crash fired on 2026-07-18 when core's new partner + // nodes reshuffled the pair plan, and the ledger row carries its mechanism + // and upstream-report status. Every non-ledgered error still fails. The + // wiring sweep queues no prompts, so a prompt-execution error here is a + // prior tier's async stray, not this test's (isForeignExecutionNoise; + // ARCHITECTURE section 9 principle). + const sweepErrors = [...consoleErrors.errors, ...isolated.errors].filter( + (error) => !isForeignExecutionNoise(error) + ) + const unledgered = unallowlistedConnectivityErrorsForPacks( + [...installedPacks], + sweepErrors + ) + if (sweepErrors.length > unledgered.length) + console.log( + `connectivity sweep: ${sweepErrors.length - unledgered.length} console error(s) matched an installed pack's allowlist` + ) + if (unledgered.length > 0) + await attachPageDiagnosticEvidence( + test.info(), + 'connectivity-console-errors.json', + unledgered + ) + expect( + unledgered.length === 0, + failureSummary( + 'console errors during breadth sweep', + unledgered, + 'connectivity-console-errors.json' + ) + ).toBe(true) + expect( + staleRequiredConnectivityErrorRulesForPacks( + [...installedPacks], + sweepErrors + ), + 'stale required connectivity console mechanisms' + ).toEqual([]) + + // Two-way guard, same discipline as cannotRunAlone, for these expectations + // in the loop below: every key must still be OBSERVED failing in its + // recorded way. An entry whose pair now passes (or is no longer even + // planned) is stale and would silently hide the fixed bug behind it. + const outcomeByKey = new Map( + results.map((result) => [result.key, result.outcome]) + ) + // The expectations are global; a shard installs one slice of the manifest. Asking + // "are all packs installed" answers that for the SLICE, so it went true and + // every entry naming a node from another shard read as stale - 13 of them on + // run 31861114255. A shard can only speak to keys whose node types it + // actually has. Node types contain dots (MathExpression|pysssss.expression), + // so the slot name is split from the right. + const liveNodeTypes = new Set(nodes.map((node) => node.type)) + const keyIsAnswerable = (key: string): boolean | null => { + const sides = key.split(' -> ') + if (sides.length !== 2) return null + const nodeTypes = sides.map((side) => { + const separator = side.lastIndexOf('.') + return separator > 0 && separator < side.length - 1 + ? side.slice(0, separator) + : null + }) + if (nodeTypes.some((nodeType) => nodeType === null)) return null + return nodeTypes.every( + (nodeType) => nodeType !== null && liveNodeTypes.has(nodeType) + ) + } + const staleEntries: string[] = [] + for (const [expectedPairs, expectedOutcome] of [ + [connectRejected, 'CONNECT_REJECTED'], + [dynamicSlotCleanupStalled, 'DYNAMIC_SLOT_CLEANUP_STALLED'], + [roundtripLost, 'ROUNDTRIP_LOST'], + [deterministicSlotContractMismatch, 'SLOT_CONTRACT_MISMATCH'] + ] as const) + for (const key of expectedPairs) { + const observed = outcomeByKey.get(key) + const answerable = keyIsAnswerable(key) + if (answerable === null) { + staleEntries.push(`${key}: invalid pair key`) + continue + } + if (!answerable) continue + if (observed !== expectedOutcome) + staleEntries.push( + `${key}: expected ${expectedOutcome}, observed ${observed ?? 'nothing'} - remove the stale entry` + ) + } + expect(staleEntries, 'stale connectivity expectations').toEqual([]) + await expectNoVisibleErrors(comfyPage.page, 'after breadth sweep') +}) + +// First planned pair whose slots both exist on real instances (pack JS can +// rebuild declared inputs as widget-only controls). +function firstMaterializedPair( + page: Page, + pairs: PlannedPair[] +): Promise { + return page.evaluate((pairsInPage) => { + for (const pair of pairsInPage) { + const producer = window.LiteGraph!.createNode(pair.producer.nodeType) + const consumer = window.LiteGraph!.createNode(pair.consumer.nodeType) + const outFound = producer?.outputs.some( + (slot) => slot.name === pair.producer.slotName + ) + const inFound = consumer?.inputs.some( + (slot) => slot.name === pair.consumer.slotName + ) + if (outFound && inFound) return pair + } + return null + }, pairs) +} + +// The self-check below runs THIS SAME executor on poisoned pairs; if it stops +// being able to reject, every green sweep above is meaningless. +async function runPairsInPage( + page: Page, + pairs: PlannedPair[] +): Promise { + const results: PairResult[] = [] + for (let start = 0; start < pairs.length; start += PAIRS_PER_BATCH) { + const batch = pairs.slice(start, start + PAIRS_PER_BATCH) + const firstKey = `${batch[0].producer.nodeType}.${batch[0].producer.slotName}` + console.log( + `connectivity shared batch: ${start + 1}-${start + batch.length}/${pairs.length} starting at ${firstKey}` + ) + results.push(...(await evaluatePairs(page, batch))) + } + return results +} + +function evaluatePairs( + page: Page, + pairs: PlannedPair[], + { + resetAfter = true, + stalledCleanupKeys = [] + }: { resetAfter?: boolean; stalledCleanupKeys?: string[] } = {} +): Promise { + const payload = [ + pairs, + resetAfter, + stalledCleanupKeys, + DYNAMIC_CLEANUP_SETTLE_MS + ] as const + return page.evaluate(async (payloadInPage) => { + const [pairsInPage, resetAfter, stalledKeys, cleanupSettleMs] = + payloadInPage + const graph = window.app!.graph + let nextNodeId = graph.last_node_id + const resetGraph = () => { + nextNodeId = graph.last_node_id + graph.clear() + graph.last_node_id = nextNodeId + } + const report: Array<{ + key: string + outcome: string + detail?: string + }> = [] + for (const pair of pairsInPage) { + const key = `${pair.producer.nodeType}.${pair.producer.slotName} -> ${pair.consumer.nodeType}.${pair.consumer.slotName}` + try { + resetGraph() + const producer = window.LiteGraph!.createNode(pair.producer.nodeType) + const consumer = window.LiteGraph!.createNode(pair.consumer.nodeType) + if (!producer || !consumer) { + report.push({ + key, + outcome: 'SLOT_CONTRACT_MISMATCH', + detail: 'createNode returned null for a registered type' + }) + continue + } + graph.add(producer) + graph.add(consumer) + let stalledCleanup: string | undefined + if (stalledKeys.includes(key)) { + const removeInput = consumer.removeInput.bind(consumer) + let priorNoProgress: + | { slotIndex: number; inputCount: number } + | undefined + consumer.removeInput = (slotIndex) => { + const inputCount = consumer.inputs.length + const removed = removeInput(slotIndex) + if (consumer.inputs.length !== inputCount) { + priorNoProgress = undefined + return removed + } + if ( + priorNoProgress?.slotIndex === slotIndex && + priorNoProgress.inputCount === inputCount + ) { + stalledCleanup = `removeInput(${slotIndex}) left ${inputCount} inputs unchanged on consecutive cleanup iterations` + const unlinkedIndex = consumer.inputs.findIndex( + (input) => input.link == null + ) + if (unlinkedIndex >= 0) removeInput(unlinkedIndex) + priorNoProgress = undefined + return removed + } + priorNoProgress = { slotIndex, inputCount } + return removed + } + } + // Pack nodeCreated hooks commonly defer widget/editor initialization + // with setTimeout(0). Let that required mount work finish before this + // sweep serializes the pair. Without the yield, a 1,000-pair browser + // task snapshots half-initialized nodes and accumulates their deferred + // WebGL/editor work until after the batch has already removed them. + await new Promise((resolve) => setTimeout(resolve, 0)) + const outIndex = producer.outputs.findIndex( + (slot) => slot.name === pair.producer.slotName + ) + const inIndex = consumer.inputs.findIndex( + (slot) => slot.name === pair.consumer.slotName + ) + if (outIndex < 0 || inIndex < 0) { + // Pack JS may rebuild a declared input as widget-only (rgthree + // Seed.seed) - excluded; missing as slot AND widget stays a fail. + const widgetOnly = + outIndex >= 0 && + (consumer.widgets ?? []).some( + (widget) => widget.name === pair.consumer.slotName + ) + report.push({ + key, + outcome: widgetOnly + ? 'WIDGET_ONLY_ON_INSTANCE' + : 'SLOT_CONTRACT_MISMATCH', + detail: `declared slot missing on instance (out=${outIndex}, in=${inIndex})` + }) + continue + } + const link = producer.connect(outIndex, consumer, inIndex) + if (!link || consumer.inputs[inIndex]?.link == null) { + report.push({ key, outcome: 'CONNECT_REJECTED' }) + continue + } + if (stalledKeys.includes(key)) { + await new Promise((resolve) => setTimeout(resolve, cleanupSettleMs)) + if (stalledCleanup !== undefined) { + report.push({ + key, + outcome: 'DYNAMIC_SLOT_CLEANUP_STALLED', + detail: stalledCleanup + }) + continue + } + } + const serialized = graph.serialize() + graph.configure(serialized) + const restored = graph.getNodeById(consumer.id) + if (restored?.inputs?.[inIndex]?.link == null) { + report.push({ + key, + outcome: 'ROUNDTRIP_LOST', + detail: 'serialize/configure dropped the link' + }) + continue + } + let prompt: { + output?: Record }> + } + try { + prompt = (await window.app!.graphToPrompt()) as typeof prompt + } catch (error) { + const detail = String(error) + if ( + detail.startsWith('InvalidLinkError: No input node found for id') + ) { + report.push({ + key, + outcome: 'ROUNDTRIP_LOST', + detail: `graphToPrompt rejected a restored dangling link: ${detail}` + }) + continue + } + throw error + } + const promptInput = + prompt.output?.[String(consumer.id)]?.inputs?.[pair.consumer.slotName] + if (!Array.isArray(promptInput)) { + report.push({ + key, + outcome: 'ROUNDTRIP_LOST', + detail: 'link missing from graphToPrompt output' + }) + continue + } + report.push({ key, outcome: 'PASS' }) + } catch (error) { + report.push({ + key, + outcome: 'THREW', + detail: `threw: ${String(error)}` + }) + } + } + if (resetAfter) resetGraph() + return report + }, payload) +} + +test('connectivity self-check: the executor rejects broken pairs @custom-nodes', async ({ + comfyPage +}) => { + const slot = (nodeType: string, slotName: string, slotType: string) => ({ + nodeType, + pack: 'core', + slotName, + slotType + }) + const results = await runPairsInPage(comfyPage.page, [ + { + producer: slot('CheckpointLoaderSimple', 'MODEL', 'MODEL'), + consumer: slot('KSampler', 'latent_image', 'LATENT') + }, + { + producer: slot('EmptyLatentImage', 'LATENT', 'LATENT'), + consumer: slot('KSampler', 'does_not_exist', 'LATENT') + } + ]) + expect(results.map((result) => result.outcome)).toEqual([ + 'CONNECT_REJECTED', + 'SLOT_CONTRACT_MISMATCH' + ]) +}) + +test('connectivity drags: one materialized in-pack link per applicable pack connects under both renderers @custom-nodes', async ({ + comfyPage +}) => { + test.setTimeout(PLAN_SETUP_MS) + const defs = (await comfyPage.page.evaluate(() => + window.app!.api.getNodeDefs() + )) as unknown as Record + const nodes = normalizeNodeDefs(defs) + + // Native anchor pair plus one in-pack, link-typed pair per connectivity + // pack (derived from the same generator the breadth sweep uses). + const dragEdges: PlannedPair[] = [ + { + producer: { + nodeType: 'EmptyLatentImage', + pack: 'core', + slotName: 'LATENT', + slotType: 'LATENT' + }, + consumer: { + nodeType: 'KSampler', + pack: 'core', + slotName: 'latent_image', + slotType: 'LATENT' + } + }, + // Second-slot anchor: ImageBatch has two IMAGE inputs (image1, image2) + // and we target the SECOND. A slot hit-test regression that falls back + // to the first compatible input would land on image1, leaving image2 + // (the asserted index) unlinked - so this pair, unlike a first-slot + // pair, actually discriminates a broken drop-to-slot resolution. + { + producer: { + nodeType: 'EmptyImage', + pack: 'core', + slotName: 'IMAGE', + slotType: 'IMAGE' + }, + consumer: { + nodeType: 'ImageBatch', + pack: 'core', + slotName: 'image2', + slotType: 'IMAGE' + } + } + ] + const nodeTypes = new Set(nodes.map((node) => node.type)) + const observedZeroPairPacks = new Set() + for (const entry of connectivityEntries) { + if (!isEntryInstalled(nodeTypes, entry)) { + console.log( + `connectivity drag: ${entry.pack} not installed on this backend` + ) + continue + } + // Restrict the partner pool to the pack itself so the drag proves an + // in-pack wiring; widget-backed primitive inputs render real slot dots + // in Vue (verified empirically), so no slot type is excluded at plan time. + const registeredPackNodes = nodes.filter((node) => node.pack === entry.pack) + const eligiblePackNodeTypes = new Set( + eligibleNodeTypesForTier( + { identity: packIdentity(entry), pack: entry.pack }, + 'S5', + registeredPackNodes.map((node) => node.type) + ) + ) + const packNodes = registeredPackNodes.filter((node) => + eligiblePackNodeTypes.has(node.type) + ) + const packPlan = planPairs(packNodes, entry.expectedNodes) + if (packPlan.pairs.length === 0) { + expect( + zeroPairDragExpectedNodeCounts[entry.pack], + `${entry.pack} registers ${packNodes.length} nodes but contributes no in-pack draggable pair - drag coverage lost` + ).toBe(packNodes.length) + observedZeroPairPacks.add(entry.pack) + console.log( + `connectivity drag: ${entry.pack} is the verified ${packNodes.length}-node pack with no self-pair; S4 cross-pack coverage applies, S5 in-pack drag is not applicable` + ) + continue + } + // The plan comes from object_info, but a pack's own JS can rebuild a + // declared input as widget-only on the instance (rgthree's Seed does). + // Drag the first pair whose slots actually materialize; a pack whose + // every planned pair is customized away has no socket contract to drag. + const inPack = await firstMaterializedPair(comfyPage.page, packPlan.pairs) + if (!inPack) + throw new Error( + `${entry.pack} has planned pairs but every declared socket is widget-only on instances - add an exact reviewed applicability expectation` + ) + dragEdges.push(inPack) + } + for (const pack of Object.keys(zeroPairDragExpectedNodeCounts)) { + // Existence is a manifest-wide fact; whether this shard owns the pack is + // a separate question. Checking both against the slice made every shard + // that does not own a listed pack report it as stale. + expect( + loadAllManifestPackNames().includes(pack), + `${pack} has a zero-pair expectation but is not a manifest entry` + ).toBe(true) + const entry = connectivityEntries.find((entry) => entry.pack === pack) + if (!entry || !isEntryInstalled(nodeTypes, entry)) continue + expect( + observedZeroPairPacks.has(entry.pack), + `${pack} now contributes an in-pack draggable pair - remove the stale zero-pair expectation` + ).toBe(true) + } + + const rendererPasses = [false, true] + test.setTimeout( + PLAN_SETUP_MS + dragEdges.length * rendererPasses.length * DRAG_MS_PER_DRAG + ) + for (const vueNodesEnabled of rendererPasses) { + const consoleErrors = collectConsoleErrors(comfyPage.page) + await comfyPage.settings.setSetting( + 'Comfy.VueNodes.Enabled', + vueNodesEnabled + ) + + for (const edge of dragEdges) { + await comfyPage.nodeOps.clearGraph() + const producer = await comfyPage.nodeOps.addNode( + edge.producer.nodeType, + undefined, + { x: 150, y: 200 } + ) + await comfyPage.nextFrame() + const producerWidth = (await producer.getSize()).width + const consumer = await comfyPage.nodeOps.addNode( + edge.consumer.nodeType, + undefined, + { x: 150 + producerWidth + 150, y: 200 } + ) + await comfyPage.nextFrame() + await fitToViewInstant(comfyPage) + + const [outIndex, inIndex] = await comfyPage.page.evaluate( + ([producerId, consumerId, outName, inName]) => { + const byId = (id: string) => + window.app!.graph.nodes.find((node) => String(node.id) === id)! + const src = byId(producerId) + const dst = byId(consumerId) + return [ + src.outputs.findIndex((slot) => slot.name === outName), + dst.inputs.findIndex((slot) => slot.name === inName) + ] + }, + [ + String(producer.id), + String(consumer.id), + edge.producer.slotName, + edge.consumer.slotName + ] as const + ) + const key = `${edge.producer.nodeType}.${edge.producer.slotName} -> ${edge.consumer.nodeType}.${edge.consumer.slotName}` + expect(outIndex, `${key}: producer slot on instance`).toBeGreaterThan(-1) + expect(inIndex, `${key}: consumer slot on instance`).toBeGreaterThan(-1) + + if (vueNodesEnabled) { + await comfyPage.vueNodes.waitForNodes(2) + // Slot-key-addressed dots so shared-label ambiguity cannot misfire + // the drag. + const outDot = comfyPage.vueNodes.getOutputSlotConnectionDot( + String(producer.id), + outIndex + ) + const inDot = comfyPage.vueNodes.getInputSlotConnectionDot( + String(consumer.id), + inIndex + ) + await outDot.dragTo(inDot) + } else { + await producer.connectOutput(outIndex, consumer, inIndex) + } + + const linked = await comfyPage.page.evaluate( + ([consumerId, index]) => { + const node = window.app!.graph.nodes.find( + (candidate) => String(candidate.id) === consumerId + ) + return node?.inputs?.[Number(index)]?.link != null + }, + [String(consumer.id), String(inIndex)] as const + ) + expect(linked, `${key} with VueNodes=${vueNodesEnabled}`).toBe(true) + } + + consoleErrors.stop() + const dragPacks = [ + ...new Set( + dragEdges.flatMap((edge) => [edge.producer.pack, edge.consumer.pack]) + ) + ] + const dragErrors = consoleErrors.errors.filter( + (error) => !isForeignExecutionNoise(error) + ) + const unledgered = unallowlistedErrorsForPacks(dragPacks, dragErrors) + if (dragErrors.length > unledgered.length) + console.log( + `connectivity drag: ${dragErrors.length - unledgered.length} console error(s) matched the exact environment or installed-pack ledger` + ) + if (unledgered.length > 0) + await attachPageDiagnosticEvidence( + test.info(), + 'connectivity-drag-console-errors.json', + unledgered + ) + expect( + unledgered.length === 0, + failureSummary( + `console errors with VueNodes=${vueNodesEnabled}`, + unledgered, + 'connectivity-drag-console-errors.json' + ) + ).toBe(true) + await expectNoVisibleErrors( + comfyPage.page, + `after drag pass VueNodes=${vueNodesEnabled}` + ) + } +}) diff --git a/browser_tests/tests/customNodes/connectivityExpectations.pure.spec.ts b/browser_tests/tests/customNodes/connectivityExpectations.pure.spec.ts new file mode 100644 index 00000000000..e5ee0cd42cc --- /dev/null +++ b/browser_tests/tests/customNodes/connectivityExpectations.pure.spec.ts @@ -0,0 +1,71 @@ +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import { + connectivityExpectations, + pairEndpointOwnershipIssues, + pairEndpointPacks, + pairExpectationKeys, + pairExpectationNodeTypes +} from '@e2e/fixtures/customNode/connectivityExpectations' +import { loadAllManifestPackNames } from '@e2e/fixtures/customNode/manifest' + +test('connectivity pair expectations are attributable and disjoint', () => { + const manifestPacks = new Set( + loadAllManifestPackNames().map((pack) => pack.toLowerCase()) + ) + const groups = [ + ...connectivityExpectations.connectRejected, + ...connectivityExpectations.deterministicSlotContractMismatch, + ...connectivityExpectations.dynamicSlotCleanupStalled, + ...connectivityExpectations.roundtripLost + ] + for (const group of groups) { + expect(group.id).toBeTruthy() + expect(group.pack).toBeTruthy() + expect(group.reason).toBeTruthy() + expect(group.restore).toBeTruthy() + expect(group.pairs.length).toBeGreaterThan(0) + expect(manifestPacks.has(group.pack.toLowerCase())).toBe(true) + } + + expect(new Set(groups.map((group) => group.id)).size).toBe(groups.length) + const pairs = pairExpectationKeys(groups) + expect(new Set(pairs).size).toBe(pairs.length) + expect(Object.keys(pairEndpointPacks).sort()).toEqual( + pairExpectationNodeTypes(groups) + ) + for (const pack of Object.values(pairEndpointPacks)) + expect(pack === 'core' || manifestPacks.has(pack.toLowerCase())).toBe(true) +}) + +test('pair endpoint ownership is checked only on the shard that owns it', () => { + expect( + pairEndpointOwnershipIssues( + ['CrossShardNode'], + [], + new Set(['DifferentPack']) + ) + ).toEqual(['CrossShardNode: no endpoint pack attribution exists']) + + const required = ['FL_TimeLine'] + expect(pairEndpointOwnershipIssues(required, [], new Set())).toEqual([]) + expect( + pairEndpointOwnershipIssues(required, [], new Set(['ComfyUI_Fill-Nodes'])) + ).toEqual(['FL_TimeLine: not registered by ComfyUI_Fill-Nodes']) + expect( + pairEndpointOwnershipIssues( + required, + [{ type: 'FL_TimeLine', pack: 'OtherPack' }], + new Set(['ComfyUI_Fill-Nodes']) + ) + ).toEqual(['FL_TimeLine: expected ComfyUI_Fill-Nodes, observed OtherPack']) + expect( + pairEndpointOwnershipIssues( + required, + [{ type: 'FL_TimeLine', pack: 'ComfyUI_Fill-Nodes' }], + new Set(['ComfyUI_Fill-Nodes']) + ) + ).toEqual([]) +}) diff --git a/browser_tests/tests/customNodes/consoleErrorLedger.pure.spec.ts b/browser_tests/tests/customNodes/consoleErrorLedger.pure.spec.ts new file mode 100644 index 00000000000..0d613f6feb0 --- /dev/null +++ b/browser_tests/tests/customNodes/consoleErrorLedger.pure.spec.ts @@ -0,0 +1,324 @@ +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import { + consoleErrorExclusionsForPacks, + customExtensionStartupErrors, + isForeignExecutionNoise, + staleRequiredConnectivityErrorRulesForPacks, + staleRequiredStartupErrorRulesForPacks, + unallowlistedErrors, + unallowlistedConnectivityErrorsForPacks, + unallowlistedGlobalExtensionErrorsForPacks, + unallowlistedErrorsForPacks +} from '@e2e/fixtures/customNode/consoleErrorLedger' +import { loadAllManifestPackNames } from '@e2e/fixtures/customNode/manifest' + +// unallowlistedErrors is the sole enforcement point of the curated load and +// run console gates (pack startup/load and curated workflow execution): a degradation +// to "always empty" would turn those gates vacuously green, so the filter's +// three behaviors are pinned here directly. +test.describe('consoleErrorLedger', () => { + test('startup attribution excludes core boot noise and keeps extension failures', () => { + expect( + customExtensionStartupErrors([ + 'Failed to load resource: 404 [http://localhost:8188/user.css]', + 'ComfyApp graph accessed before initialization [http://localhost:8188/assets/app.js]', + 'Failed to load resource: 404 [http://localhost:8188/extensions/core/clipspace.js]', + 'Failed to load resource: 404 [http://localhost:8188/extensions/Pack/widget.js]', + "Error calling extension 'Pack.feature' method 'setup'", + "[vite:preloadError] {message: Unexpected token '}'} [http://localhost:8188/assets/main.js]" + ]) + ).toEqual([ + 'Failed to load resource: 404 [http://localhost:8188/extensions/Pack/widget.js]', + "Error calling extension 'Pack.feature' method 'setup'", + "[vite:preloadError] {message: Unexpected token '}'} [http://localhost:8188/assets/main.js]" + ]) + }) + + test('required startup errors fail stale and filter only their exact signatures', () => { + const error = + "[vite:preloadError] {url: null, message: Unexpected token '}'} [http://localhost:8188/assets/main-abc.js]" + expect( + staleRequiredStartupErrorRulesForPacks(['WhatDreamsCost-ComfyUI'], []) + ).toEqual(['WhatDreamsCost-ComfyUI/ltx-director-guide-syntax']) + expect( + staleRequiredStartupErrorRulesForPacks( + ['WhatDreamsCost-ComfyUI'], + [error] + ) + ).toEqual([]) + expect( + unallowlistedGlobalExtensionErrorsForPacks( + ['WhatDreamsCost-ComfyUI'], + [error, 'different failure'] + ) + ).toEqual(['different failure']) + }) + + test('reports every console acceptance with unique removal metadata', () => { + const exclusions = consoleErrorExclusionsForPacks([ + ...new Map( + loadAllManifestPackNames().map((pack) => [pack.toLowerCase(), pack]) + ).values() + ]) + expect(new Set(exclusions.map(({ label }) => label)).size).toBe( + exclusions.length + ) + expect(exclusions.every(({ reason }) => reason.length > 0)).toBe(true) + expect(exclusions.every(({ restore }) => restore.length > 0)).toBe(true) + expect(exclusions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + label: 'WhatDreamsCost-ComfyUI console ltx-director-guide-syntax', + mode: 'expected-failure' + }), + expect.objectContaining({ + label: + 'WhatDreamsCost-ComfyUI console whatdreams-set-value-probe-preview', + mode: 'conditional-console' + }) + ]) + ) + }) + + test('filters only errors matching the pack own patterns', () => { + const errors = [ + 'Failed to load resource: the server responded with a status of 404 () http://host/000_custom_node_probe.png', + 'Failed to load resource: the server responded with a status of 404 () http://host/000_custom_node_probe.wav', + 'Failed to load resource: the server responded with a status of 404 () http://host/001_custom_node_probe.wav', + 'TypeError: something real broke' + ] + expect(unallowlistedErrors('ComfyUI-Impact-Pack', errors)).toEqual([ + 'TypeError: something real broke' + ]) + }) + + test('a pattern never filters for a pack that does not own it', () => { + const error = "Cannot use 'in' operator to search for 'content' in null" + expect(unallowlistedErrors('ComfyUI-Impact-Pack', [error])).toEqual([error]) + expect(unallowlistedErrors('ComfyUI-Custom-Scripts', [error])).toEqual([]) + }) + + test('matches the pack key case-insensitively', () => { + const errors = [ + 'Failed to load resource: the server responded with a status of 404 () http://host/example.png', + 'TypeError: something real broke' + ] + expect(unallowlistedErrors('comfyui-impact-pack', errors)).toEqual([ + 'TypeError: something real broke' + ]) + }) + + test('unknown pack fails open: every error surfaces', () => { + // The first error would match an Impact pattern; with no ledger for the + // pack, nothing may be filtered. + const errors = [ + 'Failed to load resource: the server responded with a status of 404 () http://host/example.png', + 'boom' + ] + expect(unallowlistedErrors('Some-Future-Pack', errors)).toEqual(errors) + }) + + test('matches only the observed bare-backend resource requests', () => { + const cases = [ + [ + 'ComfyUI-UltraShape1', + 'Failed to load resource: 404 [http://localhost:8188/api/view?type=input&filename=%28upload+a+mesh+file%29&subfolder=&rand=0.5]' + ], + [ + 'WhatDreamsCost-ComfyUI', + 'Failed to load resource: 404 [http://localhost:8188/api/view?filename=_cn&type=input]' + ], + [ + 'ComfyUI_Fill-Nodes', + 'Failed to load resource: 404 [https://api.comfy.org/comfy-nodes//node]' + ], + [ + 'comfyui-impact-pack', + 'Failed to load resource: 404 [http://localhost:8188/beach.jpg]' + ] + ] as const + for (const [pack, error] of cases) { + expect(unallowlistedErrors(pack, [error])).toEqual([]) + expect(unallowlistedErrors('Some-Future-Pack', [error])).toEqual([error]) + } + }) + + test('attributes iTools no-saved-drawing log only to an installed iTools pack', () => { + const paintFileLog = + 'Error: File not found [http://localhost:8188/extensions/comfyui-itools/makadi/SmartPaintArea.js]' + expect(unallowlistedErrors('comfyui-itools', [paintFileLog])).toEqual([]) + expect(unallowlistedErrors('Some-Future-Pack', [paintFileLog])).toEqual([ + paintFileLog + ]) + expect( + unallowlistedGlobalExtensionErrorsForPacks( + ['comfyui-itools'], + [paintFileLog] + ) + ).toEqual([]) + }) + + test('cross-pack variant filters only via packs in scope', () => { + // Both observed editor_base subclasses match the mechanism pattern. + const kjErrors = [ + "Error creating SplineEditor: TypeError: Cannot read properties of null (reading 'replaceChild')", + "Error creating PointsEditor: TypeError: Cannot read properties of null (reading 'replaceChild')" + ] + // Owning pack in scope: ledgered. Absent: the error surfaces, so a pack + // outside the sweep corpus can never vouch for an error. + expect( + unallowlistedErrorsForPacks( + ['ComfyUI-Impact-Pack', 'ComfyUI-KJNodes'], + [...kjErrors, 'boom'] + ) + ).toEqual(['boom']) + expect( + unallowlistedErrorsForPacks(['ComfyUI-Impact-Pack'], kjErrors) + ).toEqual(kjErrors) + }) + + test('filters the conditional KJ and VHS connectivity errors', () => { + const points = + 'Error parsing stored points: SyntaxError: Unexpected end of JSON input\n at new PointsEditor (http://localhost/extensions/ComfyUI-KJNodes/js/editors/point_editor_canvas.js:77:26)' + const vhs = + "Uncaught page error: TypeError: Cannot read properties of undefined (reading 'target_id')\n at get_links (http://localhost/extensions/ComfyUI-VideoHelperSuite/js/VHS.core.js:2096:71)" + const spline = + "Error creating SplineEditor: TypeError: Cannot read properties of undefined (reading 'x')\n at Object.buildPathD (http://localhost/extensions/ComfyUI-KJNodes/js/editors/interpolation.js:132:27)" + expect( + unallowlistedConnectivityErrorsForPacks( + ['ComfyUI-KJNodes', 'ComfyUI-VideoHelperSuite'], + [points, vhs, spline] + ) + ).toEqual([spline]) + expect( + staleRequiredConnectivityErrorRulesForPacks( + ['ComfyUI-KJNodes', 'ComfyUI-VideoHelperSuite'], + [points, vhs] + ) + ).toEqual([]) + expect( + staleRequiredConnectivityErrorRulesForPacks( + ['ComfyUI-KJNodes', 'ComfyUI-VideoHelperSuite'], + [] + ) + ).toEqual([]) + expect(consoleErrorExclusionsForPacks(['ComfyUI-KJNodes'])).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + label: + 'ComfyUI-KJNodes connectivity console kj-points-empty-bbox-json', + mode: 'conditional-console', + scope: 'S4 connectivity sweep', + tier: 'S8' + }) + ]) + ) + }) + + test('requires the exact LTX sparse-editor initialization failure', () => { + const ltx = + "Uncaught page error: TypeError: Cannot read properties of null (reading 'imgH')\n" + + ' at widget.computeSize (http://localhost:8188/extensions/ComfyUI-LTXVideo/js/sparse_track_editor.js:224:29)' + expect( + unallowlistedConnectivityErrorsForPacks(['ComfyUI-LTXVideo'], [ltx]) + ).toEqual([]) + expect( + unallowlistedConnectivityErrorsForPacks(['Some-Future-Pack'], [ltx]) + ).toEqual([ltx]) + expect( + staleRequiredConnectivityErrorRulesForPacks(['ComfyUI-LTXVideo'], [ltx]) + ).toEqual([]) + expect( + staleRequiredConnectivityErrorRulesForPacks(['ComfyUI-LTXVideo'], []) + ).toEqual(['ltx-sparse-track-null-image-size']) + }) + + test('matches lower-cased VHS paths', () => { + const vhs = + "TypeError: Cannot read properties of undefined (reading 'target_id')\n" + + 'at get_links (http://localhost:8188/extensions/comfyui-videohelpersuite/js/VHS.core.js:2088:71)' + expect( + unallowlistedConnectivityErrorsForPacks( + ['comfyui-videohelpersuite'], + [vhs] + ) + ).toEqual([]) + expect( + staleRequiredConnectivityErrorRulesForPacks( + ['comfyui-videohelpersuite'], + [vhs] + ) + ).toEqual([]) + }) + + test('allows only the exact pysssss None-default 404 without requiring it', () => { + const pysssss404 = + 'Failed to load resource: the server responded with a status of 404 (Not Found) [http://localhost:8188/api/pysssss/examples/loras%2FNone]' + expect( + unallowlistedConnectivityErrorsForPacks( + ['ComfyUI-Custom-Scripts'], + [pysssss404] + ) + ).toEqual([]) + expect( + staleRequiredConnectivityErrorRulesForPacks( + ['ComfyUI-Custom-Scripts'], + [] + ) + ).toEqual([]) + expect( + staleRequiredConnectivityErrorRulesForPacks( + ['ComfyUI-Custom-Scripts'], + [pysssss404] + ) + ).toEqual([]) + expect( + unallowlistedConnectivityErrorsForPacks( + ['ComfyUI-Custom-Scripts'], + [ + pysssss404.replace('404', '500'), + pysssss404.replace('localhost:8188', 'localhost:4040'), + pysssss404.replace('loras%2FNone', 'loras%2FOther') + ] + ) + ).toHaveLength(3) + expect( + unallowlistedConnectivityErrorsForPacks( + ['ComfyUI-VideoHelperSuite'], + [pysssss404] + ) + ).toEqual([pysssss404]) + }) +}) + +// Filters a prior tier's async execution error out of the non-executing +// tiers; must match execution-domain lines and nothing a mount/wiring tier +// should legitimately catch. +test.describe('isForeignExecutionNoise', () => { + test('matches the execution-domain console surfaces', () => { + expect(isForeignExecutionNoise('PromptExecutionError: boom')).toBe(true) + expect(isForeignExecutionNoise('Prompt execution failed')).toBe(true) + expect( + isForeignExecutionNoise( + 'Failed to load resource: the server responded with a status of 400 (Bad Request) http://127.0.0.1:8288/api/prompt' + ) + ).toBe(true) + }) + + test('does not match render or unrelated resource errors a tier must catch', () => { + expect( + isForeignExecutionNoise('TypeError: cannot read x of undefined') + ).toBe(false) + expect( + isForeignExecutionNoise( + 'Failed to load resource: 404 http://127.0.0.1:8288/api/view?filename=x.png' + ) + ).toBe(false) + expect( + isForeignExecutionNoise('Uncaught page error: something rendered wrong') + ).toBe(false) + }) +}) diff --git a/browser_tests/tests/customNodes/coreSmoke.spec.ts b/browser_tests/tests/customNodes/coreSmoke.spec.ts new file mode 100644 index 00000000000..12a6dfaa43d --- /dev/null +++ b/browser_tests/tests/customNodes/coreSmoke.spec.ts @@ -0,0 +1,91 @@ +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' + +import { validateComfyWorkflow } from '@/platform/workflow/validation/schemas/workflowSchema' +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import { isForeignExecutionNoise } from '@e2e/fixtures/customNode/consoleErrorLedger' +import { + customNodeSuiteSettings, + drainBackendToIdle, + trackSubmittedPrompts +} from '@e2e/fixtures/utils/customNodeSuite' +import { collectConsoleErrors } from '@e2e/fixtures/utils/consoleErrorCollector' +import { expectNoVisibleErrors } from '@e2e/fixtures/utils/errorSurfaces' +import { assetPath } from '@e2e/fixtures/utils/paths' + +// Core-only, model-free workflow: the bundled default template references +// model files a scoped test backend does not have, which rightly trips the +// error surfaces this suite asserts are clean. +const smokeWorkflowInput: unknown = JSON.parse( + readFileSync(resolve(assetPath('customNodes/core_smoke.json')), 'utf-8') +) + +test.use({ initialSettings: customNodeSuiteSettings }) + +test.beforeEach(async ({ comfyPage }) => { + trackSubmittedPrompts(comfyPage.page) +}) + +// This spec queues no prompt of its own, so the drain returns without a +// round-trip; it stays as the guard for pack JS that queues one behind our +// back, which would otherwise run on into the next test. +test.afterEach(async ({ comfyPage }) => { + expect( + await drainBackendToIdle(comfyPage.page, 10_000), + 'smoke test left test-owned backend work running' + ).toBe(0) +}) + +test.describe('smoke: core workflow @custom-nodes', () => { + test('starts without onboarding and with a blank graph', async ({ + comfyPage + }) => { + await expect( + comfyPage.templatesDialog.root.filter({ + has: comfyPage.page.getByTestId('template-filter-bar') + }) + ).toBeHidden() + await expect( + comfyPage.page.getByTestId('getting-started-blank') + ).toBeHidden() + expect(await comfyPage.nodeOps.getGraphNodesCount()).toBe(0) + }) + + test('loads without console errors in both renderers', async ({ + comfyPage + }) => { + const validationErrors: string[] = [] + const smokeWorkflow = await validateComfyWorkflow( + smokeWorkflowInput, + (error) => validationErrors.push(error) + ) + expect(validationErrors, 'core smoke fixture schema errors').toEqual([]) + expect( + smokeWorkflow, + 'core smoke fixture must be a valid workflow' + ).not.toBeNull() + if (!smokeWorkflow) throw new Error('core smoke fixture validation failed') + for (const vueNodesEnabled of [false, true]) { + const consoleErrors = collectConsoleErrors(comfyPage.page) + await comfyPage.settings.setSetting( + 'Comfy.VueNodes.Enabled', + vueNodesEnabled + ) + await comfyPage.workflow.loadGraphData(smokeWorkflow) + await comfyPage.nextFrame() + consoleErrors.stop() + + expect(await comfyPage.nodeOps.getGraphNodesCount()).toBeGreaterThan(0) + // Core smoke loads a graph but queues no prompt; a prompt-execution + // error here is a prior tier's async stray (isForeignExecutionNoise). + expect( + consoleErrors.errors.filter((error) => !isForeignExecutionNoise(error)), + `console errors (VueNodes=${vueNodesEnabled})` + ).toEqual([]) + await expectNoVisibleErrors(comfyPage.page, `VueNodes=${vueNodesEnabled}`) + } + }) +}) diff --git a/browser_tests/tests/customNodes/customNode.regression.spec.ts b/browser_tests/tests/customNodes/customNode.regression.spec.ts new file mode 100644 index 00000000000..01fe57d4434 --- /dev/null +++ b/browser_tests/tests/customNodes/customNode.regression.spec.ts @@ -0,0 +1,530 @@ +import { existsSync, readFileSync } from 'node:fs' +import { basename, resolve } from 'node:path' + +import type { Page } from '@playwright/test' + +import type { ComfyWorkflowJSON } from '@/platform/workflow/validation/schemas/workflowSchema' +import type { CuratedOutputHashes } from '@e2e/fixtures/customNode/outputHashes' +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import { + customNodeSuiteSettings, + drainBackendToIdle, + trackSubmittedPrompts +} from '@e2e/fixtures/utils/customNodeSuite' +import { LocalDesktopTarget } from '@e2e/fixtures/customNode/ComfyTarget' +import { + customExtensionStartupErrors, + isForeignExecutionNoise, + staleRequiredStartupErrorRulesForPacks, + unallowlistedGlobalExtensionErrorsForPacks, + unallowlistedErrors +} from '@e2e/fixtures/customNode/consoleErrorLedger' +import { + AUTOGROW_CASES, + customNodesManifest, + expectedExtensionsFor, + FRONTEND_ASSET_EXCLUSIONS, + loadManifest, + packIdentity, + servesFrontendAssetsForPack, + staleAutogrowApplicabilityIssues +} from '@e2e/fixtures/customNode/manifest' +import { missingExpectedNodes } from '@e2e/fixtures/customNode/objectInfoValidator' +import { + compareOutputHashes, + hashSinkPayloads, + recordObservedHashes +} from '@e2e/fixtures/customNode/outputHashes' +import { + collectConsoleErrors, + startupConsoleErrors +} from '@e2e/fixtures/utils/consoleErrorCollector' +import { + errorSurfaces, + expectNoVisibleErrors +} from '@e2e/fixtures/utils/errorSurfaces' +import { assetPath } from '@e2e/fixtures/utils/paths' + +const target = new LocalDesktopTarget() +const OBJECT_INFO_SANITY_FLOOR = 50 +// Display sinks used by the curated workflows; each is an output node whose +// `executed` event carries a ui payload, so "the workflow ran" can be +// upgraded to "data actually arrived at the sink". Console-style sinks +// (WAS `Text to Console`) emit NO ui payload and stay off this list, so a +// pack whose only sink prints to console gets execution-completed proof +// only. +const CURATED_SINK_TYPES = ['PreviewAny', 'DisplayAny', 'ShowText|pysssss'] + +test.use({ initialSettings: customNodeSuiteSettings }) + +test.beforeEach(async ({ comfyPage }) => { + trackSubmittedPrompts(comfyPage.page) +}) + +// Leave the shared backend idle so the next test starts clean (drainBackendToIdle). +test.afterEach(async ({ comfyPage }) => { + expect( + await drainBackendToIdle(comfyPage.page, 10_000), + 'test-owned backend work did not reach idle during cleanup' + ).toBe(0) +}) + +function readWorkflow(relativePath: string): ComfyWorkflowJSON { + return JSON.parse( + readFileSync(resolve(relativePath), 'utf-8') + ) as ComfyWorkflowJSON +} + +async function nodeIdsByType( + page: Page, + classTypes: string[] +): Promise { + return await page.evaluate((types) => { + const nodes = window.app!.graph.nodes ?? [] + return nodes + .filter((node) => { + const n = node as { comfyClass?: string; type?: string } + return types.includes(n.comfyClass ?? n.type ?? '') + }) + .map((node) => String(node.id)) + }, classTypes) +} + +const manifestEntries = loadManifest() +const installedManifestPacks = manifestEntries.map((entry) => entry.pack) + +test('Pack startup/load: custom extensions import without unallowlisted errors @custom-nodes', async ({ + comfyPage +}) => { + const startupErrors = customExtensionStartupErrors( + startupConsoleErrors(comfyPage.page) + ) + expect( + unallowlistedGlobalExtensionErrorsForPacks( + installedManifestPacks, + startupErrors + ), + 'custom extension failed while the application loaded it' + ).toEqual([]) + expect( + staleRequiredStartupErrorRulesForPacks( + installedManifestPacks, + startupErrors + ), + 'stale required startup extension errors' + ).toEqual([]) +}) + +for (const entry of manifestEntries) { + const workflowRelative = `browser_tests/${entry.workflow}` + + test.describe(`custom node: ${entry.pack} @custom-nodes`, () => { + test('Pack startup/load: expected nodes register, render in both renderers, and frontend integration is present', async ({ + comfyPage + }) => { + test.setTimeout(entry.timeoutMs) + const objectInfo = await target.getObjectInfo(comfyPage.page) + expect( + Object.keys(objectInfo).length, + 'object_info sanity floor' + ).toBeGreaterThan(OBJECT_INFO_SANITY_FLOOR) + const missing = missingExpectedNodes(objectInfo, entry.expectedNodes) + expect( + missing, + `${entry.pack} not installed on this backend (missing: ${missing.join(', ')})` + ).toEqual([]) + await expectNoVisibleErrors(comfyPage.page, 'at startup') + + // Backend registration alone does not prove the pack's FRONTEND JS + // loaded: a wrong web dir or a loadExtensions regression leaves nodes + // in object_info while every JS-driven behavior silently vanishes + // (and this suite would then be testing vanilla nodes). Assert the + // pack's boot-registered extensions actually arrived in the browser. + const ownedAutogrowCases = AUTOGROW_CASES.filter( + ({ pack }) => pack.toLowerCase() === entry.pack.toLowerCase() + ) + const webDirectory = + 'webDirectory' in entry ? entry.webDirectory : undefined + if ( + expectedExtensionsFor(entry).length > 0 || + ownedAutogrowCases.length > 0 || + webDirectory !== undefined + ) { + const registered = await comfyPage.page.evaluate(() => + window.app!.extensions.map((extension) => extension.name) + ) + const servedExtensionPaths = + webDirectory !== undefined || + ownedAutogrowCases.some( + ({ extensionName }) => + !expectedExtensionsFor(entry).includes(extensionName) + ) + ? await comfyPage.page.evaluate(() => + window.app!.api.getExtensions() + ) + : [] + for (const name of expectedExtensionsFor(entry)) + expect( + registered, + `${entry.pack}: frontend extension "${name}" not registered - pack JS did not load` + ).toContain(name) + if (webDirectory !== undefined) { + const servesAssets = servesFrontendAssetsForPack( + servedExtensionPaths, + entry.pack + ) + const exclusion = FRONTEND_ASSET_EXCLUSIONS[entry.pack] + if (exclusion) { + expect(packIdentity(entry)).toBe(exclusion.deployRef) + expect(webDirectory).toBe(exclusion.webDirectory) + expect( + servesAssets, + `${entry.pack}: frontend assets are now served - remove the stale exclusion` + ).toBe(false) + } else + expect( + servesAssets, + `${entry.pack}: supported_nodes.yaml declares web_directory=${webDirectory}, but the backend serves no extension path for this pack` + ).toBe(true) + } + const staleAutogrowApplicability = staleAutogrowApplicabilityIssues( + { + pack: entry.pack, + expectedExtensions: expectedExtensionsFor(entry) + }, + registered, + servedExtensionPaths + ) + expect( + staleAutogrowApplicability, + `${entry.pack}: ${staleAutogrowApplicability.join('; ')}` + ).toEqual([]) + } + + for (const vueNodesEnabled of [false, true]) { + const consoleErrors = collectConsoleErrors(comfyPage.page) + await comfyPage.settings.setSetting( + 'Comfy.VueNodes.Enabled', + vueNodesEnabled + ) + await comfyPage.nodeOps.clearGraph() + + const addedIds: string[] = [] + for (const classType of entry.expectedNodes) { + const node = await comfyPage.nodeOps.addNode(classType) + addedIds.push(String(node.id)) + } + await comfyPage.nextFrame() + + expect(await comfyPage.nodeOps.getGraphNodesCount()).toBe( + entry.expectedNodes.length + ) + // Vue Nodes 2.0 mounts each node as a [data-node-id] element; assert + // the pack's own nodes rendered, not just any node count. + if (vueNodesEnabled) + for (const id of addedIds) + await expect(comfyPage.vueNodes.getNodeLocator(id)).toBeVisible() + + consoleErrors.stop() + // Pack startup/load renders nodes but queues no prompt; a prompt-execution + // error here is a prior tier's async stray (isForeignExecutionNoise). + // Mounting a pack's nodes is one of the surfaces its ledgered noise + // emits on, so this tier reads the ledger like curated workflow + // tiers do - the environment rules in particular apply to every pack + // and were never reaching this gate. + expect( + unallowlistedErrors( + entry.pack, + consoleErrors.errors.filter( + (error) => !isForeignExecutionNoise(error) + ) + ), + `console errors with VueNodes=${vueNodesEnabled}` + ).toEqual([]) + await expectNoVisibleErrors( + comfyPage.page, + `after VueNodes=${vueNodesEnabled} pass` + ) + } + }) + + if (entry.tiers.includes('run')) + test('Curated workflow execution: completes without error', async ({ + comfyPage + }) => { + test.setTimeout(entry.timeoutMs + 15_000) + const objectInfo = await target.getObjectInfo(comfyPage.page) + const missing = missingExpectedNodes(objectInfo, entry.expectedNodes) + expect( + missing, + `run-tier nodes unavailable for ${entry.pack}: ${missing.join(', ')}` + ).toEqual([]) + await expectNoVisibleErrors(comfyPage.page, 'at startup') + + // Pack scripts can throw during workflow load or execution without + // any visible error surface; collect console + uncaught page errors + // across the whole run, filtered through the shared pack ledger. + const consoleErrors = collectConsoleErrors(comfyPage.page) + const workflow = readWorkflow(workflowRelative) + await comfyPage.workflow.loadGraphData(workflow) + // A drifted fixture that dropped an expected node would silently + // shrink the executed-set assertion (an empty id list PASSes on + // execution_success alone): require every expected type to actually + // be present in the loaded workflow before running it. + const expectedNodeIds: string[] = [] + for (const type of entry.expectedNodes) { + const ids = await nodeIdsByType(comfyPage.page, [type]) + expect( + ids.length, + `expectedNodes drift: ${type} is not in the curated workflow ${entry.workflow}` + ).toBeGreaterThan(0) + expectedNodeIds.push(...ids) + } + const result = await target.runWorkflow(comfyPage.page, { + expectedNodeIds, + timeoutMs: entry.timeoutMs + }) + + // A run that executed and errored carries an ExecutionError; a run the + // backend rejected before executing (VALIDATION_FAIL) carries only the + // captured node_errors text in clientError - surface whichever exists so + // a red names the cause instead of printing an empty object. + expect( + result.outcome, + result.clientError ?? JSON.stringify(result.error ?? {}) + ).toBe('PASS') + // PASS proves execution completed; the sinks prove data ARRIVED. + // Every display sink in the curated workflow must have emitted a ui + // payload through its executed event. + const sinkIds = await nodeIdsByType(comfyPage.page, CURATED_SINK_TYPES) + expect( + sinkIds.length, + `curated workflow ${entry.workflow} has no observable display sink` + ).toBeGreaterThan(0) + for (const sinkId of sinkIds) + expect( + result.outputsByNode[sinkId], + `sink node ${sinkId} produced no ui payload` + ).toBeTruthy() + + const recordMode = process.env.RECORD_OUTPUT_HASHES + if (recordMode !== undefined && recordMode !== '1') + throw new Error( + `unrecognized RECORD_OUTPUT_HASHES value '${recordMode}' - the only mode is '1'` + ) + const compareMode = process.env.CN_ENABLE_S15 + if ( + compareMode !== undefined && + compareMode !== '0' && + compareMode !== '1' + ) + throw new Error( + `unrecognized CN_ENABLE_S15 value '${compareMode}' - expected '0' or '1'` + ) + if (recordMode === '1' || compareMode === '1') { + expect( + customNodesManifest(), + 'S15 is calibrated only for the six-pack Core manifest' + ).toBe('core') + const outputHashesPath = resolve( + 'browser_tests/fixtures/data/curatedOutputHashes.core.json' + ) + const curatedOutputHashes: CuratedOutputHashes | null = existsSync( + outputHashesPath + ) + ? (JSON.parse( + readFileSync(outputHashesPath, 'utf-8') + ) as CuratedOutputHashes) + : null + const observed = await hashSinkPayloads( + result.outputsByNode as Record, + async (ref) => { + const encoded = await comfyPage.page.evaluate(async (file) => { + const query = new URLSearchParams({ + filename: file.filename, + subfolder: file.subfolder, + type: file.type + }) + const response = await window.app!.api.fetchApi( + `/view?${query}` + ) + if (!response.ok) return { status: response.status, body: '' } + const bytes = new Uint8Array(await response.arrayBuffer()) + let binary = '' + for (const byte of bytes) binary += String.fromCharCode(byte) + return { status: response.status, body: btoa(binary) } + }, ref) + expect(encoded.status, `S15: /api/view ${ref.filename}`).toBe(200) + return Buffer.from(encoded.body, 'base64') + } + ) + const workflowKey = `${entry.pack}/${basename(entry.workflow)}` + if (recordMode === '1') { + recordObservedHashes( + 'test-results/curatedOutputHashes.recorded.json', + workflowKey, + observed + ) + expect( + null, + `RECORD_OUTPUT_HASHES: wrote ${Object.keys(observed).length} hash(es) for ${workflowKey} - the artifact is the product, this is not a pass` + ).not.toBeNull() + } else if (!process.env.CI) { + console.log( + `S15 compare skipped off-CI for ${workflowKey} - baselines encode the CI recording environment; CI enforces` + ) + } else { + expect( + curatedOutputHashes, + `S15: no committed hashes for Core (${outputHashesPath}) - record them with RECORD_OUTPUT_HASHES=1` + ).not.toBeNull() + expect( + curatedOutputHashes!.recordedAt.core, + 'S15 baseline must match the pinned ComfyUI core under test' + ).toBe(process.env.CN_OUTPUT_HASHES_CORE) + expect( + compareOutputHashes({ + workflowKey, + observed, + committed: curatedOutputHashes! + }), + 'S15 output regression' + ).toEqual([]) + } + } + + await expectNoVisibleErrors(comfyPage.page, 'after run') + consoleErrors.stop() + expect( + unallowlistedErrors(entry.pack, consoleErrors.errors), + 'console errors during curated run' + ).toEqual([]) + }) + }) +} + +test('harness self-check: captures a real execution error @custom-nodes', async ({ + comfyPage +}) => { + test.setTimeout(30_000) + const objectInfo = await target.getObjectInfo(comfyPage.page) + expect( + Object.keys(objectInfo).length, + 'object_info sanity floor' + ).toBeGreaterThan(OBJECT_INFO_SANITY_FLOOR) + expect( + 'DevToolsErrorRaiseNode' in objectInfo, + 'harness self-check requires ComfyUI_devtools' + ).toBe(true) + + await comfyPage.workflow.loadGraphData( + readWorkflow(assetPath('nodes/execution_error.json')) + ) + const result = await target.runWorkflow(comfyPage.page, { + expectedNodeIds: [], + timeoutMs: 15000 + }) + + expect(result.outcome).toBe('EXECUTION_ERROR') + expect(result.error?.exceptionType).toBeTruthy() + expect(result.error?.exceptionMessage).toBe('Error node was called!') + // Proves the event tap captures node ids from the live `executing` stream + // (its detail is a bare string): the failing node starts before it raises. + expect(result.executedNodes.length).toBeGreaterThan(0) + // Positive control for the zero-visible-errors invariant: a real execution + // error MUST surface in the app's error overlay. If this fails, the + // expectNoVisibleErrors selectors have rotted and every clean assertion in + // this suite is meaningless. + await expect(errorSurfaces(comfyPage.page).errorOverlay).toBeVisible() +}) + +test('collector self-check: captures uncaught page exceptions @custom-nodes', async ({ + comfyPage +}) => { + // Positive control for the console collector: an uncaught async throw + // never reaches console.error, so this proves the pageerror listener + // works. If this fails, every zero-console-errors assertion in the suite + // is blind to the whole uncaught-exception class. + const collected = collectConsoleErrors(comfyPage.page) + await comfyPage.page.evaluate(() => { + setTimeout(() => { + throw new Error('cn-collector-self-check') + }, 0) + }) + await expect + .poll( + () => + collected.errors.find((error) => + error.includes('cn-collector-self-check') + ) ?? '' + ) + .toMatch(/Error: cn-collector-self-check\n\s+at /) + collected.stop() +}) + +test('attribution self-check: a foreign-prompt terminal event cannot fail this run @custom-nodes', async ({ + comfyPage +}) => { + test.setTimeout(30_000) + const objectInfo = await target.getObjectInfo(comfyPage.page) + expect( + ['PrimitiveInt', 'PreviewAny'].filter((node) => !(node in objectInfo)), + 'attribution self-check requires core PrimitiveInt and PreviewAny' + ).toEqual([]) + await comfyPage.workflow.loadGraphData( + readWorkflow(assetPath('customNodes/core_primitive_preview_run.json')) + ) + // Once the run's event tap starts filling, inject ONE terminal error under + // a prompt id this page never queued. The positive prompt-id filter must + // discard it; the pre-capture harness let the never-seen id through the + // seen-set and misclassified the run as EXECUTION_ERROR. This is the + // discriminating guard for the foreign-attribution bug class. + await comfyPage.page.evaluate(() => { + const w = window as unknown as { + __cnEvents?: object[] + __cnSelfCheckTimer?: ReturnType + } + w.__cnSelfCheckTimer = setInterval(() => { + const sink = w.__cnEvents + if (!sink || sink.length === 0) return + sink.push({ + type: 'execution_error', + prompt_id: 'cn-foreign-self-check', + exception_type: 'ForeignError', + node_id: '424242' + }) + clearInterval(w.__cnSelfCheckTimer) + }, 25) + }) + const result = await target.runWorkflow(comfyPage.page, { + expectedNodeIds: await nodeIdsByType(comfyPage.page, [ + 'PrimitiveInt', + 'PreviewAny' + ]), + timeoutMs: 15000 + }) + // Prove the stimulus actually landed before trusting the PASS: without + // this, a run that finishes before the injector's next tick never injects + // the foreign event, and PASS then holds for the wrong reason (it would + // hold identically against a harness with the prompt-id filter removed). + // Clearing a not-yet-fired timer stops a post-run push from faking it. + const injectionLanded = await comfyPage.page.evaluate(() => { + const w = window as unknown as { + __cnEvents?: { prompt_id?: string }[] + __cnSelfCheckTimer?: ReturnType + } + clearInterval(w.__cnSelfCheckTimer) + return (w.__cnEvents ?? []).some( + (event) => event.prompt_id === 'cn-foreign-self-check' + ) + }) + expect( + injectionLanded, + 'positive control: the foreign terminal event was injected during the run' + ).toBe(true) + expect(result.outcome, JSON.stringify(result.error ?? {})).toBe('PASS') + expect(result.error).toBeUndefined() +}) diff --git a/browser_tests/tests/customNodes/customNodeSuite.pure.spec.ts b/browser_tests/tests/customNodes/customNodeSuite.pure.spec.ts new file mode 100644 index 00000000000..56b78b803cf --- /dev/null +++ b/browser_tests/tests/customNodes/customNodeSuite.pure.spec.ts @@ -0,0 +1,284 @@ +import type { Page } from '@playwright/test' + +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import { + drainBackendToIdle, + installCustomNodeBlankStartup, + runWithCollectedCleanup, + trackSubmittedPrompts, + waitForQueueQuiet +} from '@e2e/fixtures/utils/customNodeSuite' +import type { + ActivePathPointer, + DraftIndexV2, + DraftPayloadV2, + OpenPathsPointer +} from '@/platform/workflow/persistence/base/draftTypes' +import { StorageKeys } from '@/platform/workflow/persistence/base/storageKeys' + +interface QueueSnapshot { + Running: { id: string }[] + Pending: { id: string }[] +} + +// The drain's in-page callbacks are plain functions over `window.app.api`, so +// running them against a scripted api proves the real call sequence (which ids +// get interrupted vs deleted, and whether a cancellation pass happens at all) +// without a browser. The last snapshot repeats once the script runs out. +function scriptedPage(reads: QueueSnapshot[]) { + const listeners = new Set<(response: unknown) => void>() + const interrupted: string[] = [] + const deleted: string[] = [] + let read = 0 + const api = { + getQueue: () => Promise.resolve(reads[Math.min(read++, reads.length - 1)]), + interrupt: (id: string) => { + interrupted.push(id) + return Promise.resolve() + }, + deleteItem: (type: string, id: string) => { + deleted.push(`${type}:${id}`) + return Promise.resolve() + } + } + const page = { + on: (event: string, listener: (response: unknown) => void) => { + if (event === 'response') listeners.add(listener) + }, + off: (event: string, listener: (response: unknown) => void) => { + if (event === 'response') listeners.delete(listener) + }, + evaluate: async (fn: (arg?: unknown) => unknown, arg?: unknown) => { + const saved = Reflect.get(globalThis, 'window') as unknown + Reflect.set(globalThis, 'window', { app: { api } }) + try { + return await fn(arg) + } finally { + Reflect.set(globalThis, 'window', saved) + } + } + } + return { + page: page as unknown as Page, + listenerCount: () => listeners.size, + interrupted, + deleted, + submit: (promptId: string) => { + for (const listener of listeners) + listener({ + request: () => ({ method: () => 'POST' }), + url: () => 'http://backend/api/prompt', + status: () => 200, + json: () => Promise.resolve({ prompt_id: promptId }) + }) + } + } +} + +const IDLE: QueueSnapshot = { Running: [], Pending: [] } + +test('preseeds a restorable blank workflow before first boot', async ({ + page +}) => { + const path = 'workflows/Custom Nodes E2E Blank Workflow.json' + const draftKey = StorageKeys.draftKey(path) + const keys = { + index: StorageKeys.draftIndex('personal'), + payload: StorageKeys.draftPayload(path, 'personal'), + active: StorageKeys.lastActivePath('personal'), + open: StorageKeys.lastOpenPaths('personal') + } + await installCustomNodeBlankStartup(page) + await page.route('http://guard.test/', (route) => + route.fulfill({ contentType: 'text/html', body: '' }) + ) + await page.goto('http://guard.test/') + + const state = await page.evaluate( + ({ keys, path }) => { + const index = JSON.parse( + localStorage.getItem(keys.index)! + ) as DraftIndexV2 + const payload = JSON.parse( + localStorage.getItem(keys.payload)! + ) as DraftPayloadV2 + return { + storedKeys: Object.keys(localStorage) + .filter((key) => key.startsWith('Comfy.Workflow.')) + .sort(), + index, + payload, + active: JSON.parse( + localStorage.getItem(keys.active)! + ) as ActivePathPointer, + open: JSON.parse(localStorage.getItem(keys.open)!) as OpenPathsPointer, + path + } + }, + { keys, path } + ) + + const { updatedAt } = state.index + expect(state.storedKeys).toEqual(Object.values(keys).sort()) + expect(updatedAt).toEqual(expect.any(Number)) + expect(state.index).toEqual({ + v: 2, + updatedAt, + order: [draftKey], + entries: { + [draftKey]: { + path, + name: 'Custom Nodes E2E Blank Workflow.json', + isTemporary: true, + updatedAt + } + } + }) + expect(state.payload).toEqual({ + data: JSON.stringify({ + last_node_id: 0, + last_link_id: 0, + nodes: [], + links: [], + groups: [], + config: {}, + extra: {}, + version: 0.4 + }), + updatedAt + }) + expect(state.active).toEqual({ workspaceId: 'personal', path }) + expect(state.open).toEqual({ + workspaceId: 'personal', + paths: [path], + activeIndex: 0 + }) +}) + +test('preserves the test failure while every cleanup runs', async () => { + const calls: string[] = [] + const testError = new Error('original test failure') + const guardError = new Error('guard teardown failure') + + let thrown: unknown + await runWithCollectedCleanup(async () => { + calls.push('test') + throw testError + }, [ + async () => { + calls.push('guard') + throw guardError + }, + async () => { + calls.push('perf') + } + ]).catch((error: unknown) => { + thrown = error + }) + + expect(calls).toEqual(['test', 'guard', 'perf']) + expect(thrown).toBeInstanceOf(AggregateError) + expect((thrown as AggregateError).errors).toEqual([testError, guardError]) +}) + +test('rethrows a sole run or cleanup error by identity', async () => { + const runError = new Error('run failed') + const cleanupError = new Error('cleanup failed') + + await expect( + runWithCollectedCleanup(async () => { + throw runError + }, []) + ).rejects.toBe(runError) + await expect( + runWithCollectedCleanup(async () => {}, [ + async () => { + throw cleanupError + } + ]) + ).rejects.toBe(cleanupError) +}) + +test.describe('drainBackendToIdle', () => { + test('installing twice leaves one response listener', () => { + const fake = scriptedPage([IDLE]) + trackSubmittedPrompts(fake.page) + trackSubmittedPrompts(fake.page) + expect(fake.listenerCount()).toBe(1) + }) + + test('refuses to run on a page whose submissions were never tracked', async () => { + const fake = scriptedPage([IDLE]) + let message = '' + await drainBackendToIdle(fake.page, 0).catch((error: unknown) => { + message = error instanceof Error ? error.message : String(error) + }) + expect(message).toContain('call trackSubmittedPrompts(page) in beforeEach') + }) + + test('cancels our running and pending entries before giving up on the budget', async () => { + const fake = scriptedPage([ + { Running: [{ id: 'ours-run' }], Pending: [{ id: 'ours-pend' }] } + ]) + trackSubmittedPrompts(fake.page) + fake.submit('ours-run') + fake.submit('ours-pend') + expect(await drainBackendToIdle(fake.page, 0)).toBe(1) + expect(fake.interrupted).toEqual(['ours-run']) + expect(fake.deleted).toEqual(['queue:ours-pend']) + }) + + test('reports idle once our entries clear', async () => { + const fake = scriptedPage([ + { Running: [{ id: 'ours' }], Pending: [] }, + IDLE + ]) + trackSubmittedPrompts(fake.page) + fake.submit('ours') + expect(await drainBackendToIdle(fake.page, 60_000)).toBe(0) + expect(fake.interrupted).toEqual(['ours']) + }) + + test('never touches or fails on a queue owned by another client', async () => { + const fake = scriptedPage([ + { Running: [{ id: 'theirs' }], Pending: [{ id: 'theirs-2' }] } + ]) + trackSubmittedPrompts(fake.page) + fake.submit('ours') + expect(await drainBackendToIdle(fake.page, 0)).toBe(0) + expect(fake.interrupted).toEqual([]) + expect(fake.deleted).toEqual([]) + }) + + // Pins the reason the cancellation pass sits INSIDE the poll loop: deleting a + // pending entry cannot stop it once the backend promotes it to running, so a + // one-shot cancel would leak it onto the shared queue. + test('re-cancels a pending entry promoted to running between polls', async () => { + const fake = scriptedPage([ + { Running: [], Pending: [{ id: 'ours' }] }, + { Running: [{ id: 'ours' }], Pending: [] }, + IDLE + ]) + trackSubmittedPrompts(fake.page) + fake.submit('ours') + expect(await drainBackendToIdle(fake.page, 60_000)).toBe(0) + expect(fake.deleted).toEqual(['queue:ours']) + expect(fake.interrupted).toEqual(['ours']) + }) +}) + +test.describe('waitForQueueQuiet', () => { + test('reports a busy queue it did not submit without cancelling it', async () => { + const fake = scriptedPage([{ Running: [{ id: 'theirs' }], Pending: [] }]) + expect(await waitForQueueQuiet(fake.page, 0)).toBe(1) + expect(fake.interrupted).toEqual([]) + expect(fake.deleted).toEqual([]) + }) + + test('reports an empty queue as quiet', async () => { + expect(await waitForQueueQuiet(scriptedPage([IDLE]).page, 0)).toBe(0) + }) +}) diff --git a/browser_tests/tests/customNodes/detection-proof/row-01-mount-v1-input-dropped.patch b/browser_tests/tests/customNodes/detection-proof/row-01-mount-v1-input-dropped.patch new file mode 100644 index 00000000000..23c09b3ae30 --- /dev/null +++ b/browser_tests/tests/customNodes/detection-proof/row-01-mount-v1-input-dropped.patch @@ -0,0 +1,23 @@ +diff --git a/src/services/litegraphService.ts b/src/services/litegraphService.ts +--- a/src/services/litegraphService.ts ++++ b/src/services/litegraphService.ts +@@ -338,4 +338,17 @@ export const useLitegraphService = () => { + // Create sockets and widgets in the determined order +- for (const inputSpec of orderedInputSpecs) addInputSocket(node, inputSpec) +- for (const inputSpec of orderedInputSpecs) addInputWidget(node, inputSpec) ++ // DETECTION PROOF (row 1, mount v1): drop ImpactInt's declared value ++ // only in the canvas renderer. Expected: S1 red and S2 remains green. ++ const detectionProofTier = ( ++ globalThis as { ++ __COMFY_CUSTOM_NODE_DETECTION_PROOF_TIER__?: string ++ } ++ ).__COMFY_CUSTOM_NODE_DETECTION_PROOF_TIER__ ++ const detectionProof = ++ detectionProofTier === 'S1' && ++ nodeDefImpl.name === 'ImpactInt' && ++ !useSettingStore().get('Comfy.VueNodes.Enabled') ++ ? orderedInputSpecs.slice(0, -1) ++ : orderedInputSpecs ++ for (const inputSpec of detectionProof) addInputSocket(node, inputSpec) ++ for (const inputSpec of detectionProof) addInputWidget(node, inputSpec) + } diff --git a/browser_tests/tests/customNodes/detection-proof/row-02-mount-v2-vue-widget-dropped.patch b/browser_tests/tests/customNodes/detection-proof/row-02-mount-v2-vue-widget-dropped.patch new file mode 100644 index 00000000000..22bc7a80b23 --- /dev/null +++ b/browser_tests/tests/customNodes/detection-proof/row-02-mount-v2-vue-widget-dropped.patch @@ -0,0 +1,22 @@ +diff --git a/src/renderer/extensions/vueNodes/composables/useProcessedWidgets.ts b/src/renderer/extensions/vueNodes/composables/useProcessedWidgets.ts +index e7e5549d59..4ca1314e3c 100644 +--- a/src/renderer/extensions/vueNodes/composables/useProcessedWidgets.ts ++++ b/src/renderer/extensions/vueNodes/composables/useProcessedWidgets.ts +@@ -316,4 +316,17 @@ export function computeProcessedWidgets({ + isVisible: visible, + identity: { renderKey } + } of uniqueWidgets) { ++ // DETECTION PROOF (row 2, mount v2): hide ImpactInt's value widget only ++ // in Vue Nodes. Expected: S2 red and every other tier remains green. ++ const detectionProofTier = ( ++ globalThis as { ++ __COMFY_CUSTOM_NODE_DETECTION_PROOF_TIER__?: string ++ } ++ ).__COMFY_CUSTOM_NODE_DETECTION_PROOF_TIER__ ++ if ( ++ detectionProofTier === 'S2' && ++ nodeData.type === 'ImpactInt' && ++ widget.name === 'value' ++ ) ++ continue + const bareWidgetId = stripGraphPrefix(widget.nodeId ?? nodeId) diff --git a/browser_tests/tests/customNodes/detection-proof/row-03-persistence-offbyone.patch b/browser_tests/tests/customNodes/detection-proof/row-03-persistence-offbyone.patch new file mode 100644 index 00000000000..7be06fc5cc9 --- /dev/null +++ b/browser_tests/tests/customNodes/detection-proof/row-03-persistence-offbyone.patch @@ -0,0 +1,26 @@ +diff --git a/src/lib/litegraph/src/LGraphNode.ts b/src/lib/litegraph/src/LGraphNode.ts +index a928d93992..65563d0dd2 100644 +--- a/src/lib/litegraph/src/LGraphNode.ts ++++ b/src/lib/litegraph/src/LGraphNode.ts +@@ -987,7 +987,20 @@ export class LGraphNode + let i = 0 + for (const widget of this.widgets ?? []) { + if (widget.serialize === false) continue +- if (i >= info.widgets_values.length) break ++ // DETECTION PROOF (row 3, persistence): ImpactInt drops its last ++ // widget value. Expected: S3 red and every other tier remains green. ++ const detectionProofTier = ( ++ globalThis as { ++ __COMFY_CUSTOM_NODE_DETECTION_PROOF_TIER__?: string ++ } ++ ).__COMFY_CUSTOM_NODE_DETECTION_PROOF_TIER__ ++ if ( ++ detectionProofTier === 'S3' && ++ this.type === 'ImpactInt' && ++ i >= info.widgets_values.length - 1 ++ ) ++ break ++ if (i >= info.widgets_values.length) break + widget.value = info.widgets_values[i++] + } + } diff --git a/browser_tests/tests/customNodes/detection-proof/row-15-serialized-value-drifts.patch b/browser_tests/tests/customNodes/detection-proof/row-15-serialized-value-drifts.patch new file mode 100644 index 00000000000..8b4943bc00d --- /dev/null +++ b/browser_tests/tests/customNodes/detection-proof/row-15-serialized-value-drifts.patch @@ -0,0 +1,32 @@ +diff --git a/src/utils/executionUtil.ts b/src/utils/executionUtil.ts +index 0b6ad40610..fffd6e6991 100644 +--- a/src/utils/executionUtil.ts ++++ b/src/utils/executionUtil.ts +@@ -103,15 +103,23 @@ export const graphToPrompt = async ( + const widgetValue = widget.serializeValue + ? await widget.serializeValue(node, i) + : widget.value ++ // DETECTION PROOF (row 15, serialization): drift ImpactInt's integer ++ // widget value. The workflow dispatch isolates this to the S15 test. ++ const drifted = ++ node.type === 'ImpactInt' && ++ typeof widgetValue === 'number' && ++ Number.isInteger(widgetValue) ++ ? widgetValue + 1 ++ : widgetValue + // By default, Array values are reserved to represent node connections. + // We need to wrap the array as an object to avoid the misinterpretation + // of the array as a node connection. + // The backend automatically unwraps the object to an array during + // execution. +- inputs[widget.name] = Array.isArray(widgetValue) ++ inputs[widget.name] = Array.isArray(drifted) + ? widget.type === 'curve' +- ? { __type__: 'CURVE', __value__: widgetValue } +- : { __value__: widgetValue } +- : widgetValue ++ ? { __type__: 'CURVE', __value__: drifted } ++ : { __value__: drifted } ++ : drifted + } + } diff --git a/browser_tests/tests/customNodes/dynamicInputs.spec.ts b/browser_tests/tests/customNodes/dynamicInputs.spec.ts new file mode 100644 index 00000000000..5e5e28edc79 --- /dev/null +++ b/browser_tests/tests/customNodes/dynamicInputs.spec.ts @@ -0,0 +1,276 @@ +import type { Page } from '@playwright/test' + +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import { LocalDesktopTarget } from '@e2e/fixtures/customNode/ComfyTarget' +import { isForeignExecutionNoise } from '@e2e/fixtures/customNode/consoleErrorLedger' +import { missingExpectedNodes } from '@e2e/fixtures/customNode/objectInfoValidator' +import { collectConsoleErrors } from '@e2e/fixtures/utils/consoleErrorCollector' +import { loadApplicableAutogrowCases } from '@e2e/fixtures/customNode/manifest' +import { customNodeSuiteSettings } from '@e2e/fixtures/utils/customNodeSuite' +import { errorSurfaces } from '@e2e/fixtures/utils/errorSurfaces' + +// Dynamic-input (autogrow) tier: packs whose JS adds an input slot when the +// last one is connected and removes trailing empties on disconnect. That +// behavior lives in pack JS (onConnectionsChange overrides), NOT in +// /object_info - the def declares only the initial slot, so the mount and +// connectivity tiers are structurally blind to it. This tier asserts the +// BEHAVIOR: connect -> the node grows; disconnect -> it shrinks back. +// +// Both connect paths run on purpose: the drag path and the programmatic +// path go through different frontend pipelines (pointer -> hit-test -> +// LinkConnector vs a direct node.connect), and Impact's handler inspects +// `new Error().stack` (on its disconnect/removal branch), so the two paths +// can break independently. Both renderers run because a Vue reactivity gap +// can grow the graph-side array without rendering the new slot row. +// +// One curated node per mechanism: every Impact autogrow node shares one +// onConnectionsChange block, so a second node of the same mechanism adds +// runtime, not detection. ImpactMakeImageList is the IMAGE-typed member, +// wireable from the model-free EmptyImage. +// +// Disconnect runs single-path (programmatic disconnectInput) on purpose: the +// pack contract under test fires on the disconnect EVENT regardless of how +// the link was severed, and the suite has no generic drag-detach vocabulary +// (drag-detach pointer mechanics are core-interaction territory). +const applicableAutogrowCases = loadApplicableAutogrowCases() + +const target = new LocalDesktopTarget() + +test.use({ initialSettings: customNodeSuiteSettings }) + +async function consumerShape( + page: Page, + consumerId: string +): Promise<{ inputCount: number; domSlotDots: number }> { + return await page.evaluate((id) => { + const node = window.app!.graph.nodes.find( + (candidate) => String(candidate.id) === id + ) + const root = document.querySelector(`[data-node-id="${id}"]`) + return { + inputCount: node?.inputs?.length ?? -1, + domSlotDots: + root?.querySelectorAll('[data-testid="slot-connection-dot"]').length ?? + -1 + } + }, consumerId) +} + +for (const { autogrowCase } of applicableAutogrowCases) { + test.describe(`dynamic inputs: ${autogrowCase.pack} @custom-nodes`, () => { + test(`${autogrowCase.consumerType} grows on connect and shrinks on disconnect (drag + programmatic, both renderers)`, async ({ + comfyPage + }) => { + test.setTimeout(120_000) + const objectInfo = await target.getObjectInfo(comfyPage.page) + expect( + Object.keys(objectInfo).length, + 'object_info sanity floor' + ).toBeGreaterThan(50) + const missing = missingExpectedNodes(objectInfo, [ + autogrowCase.consumerType, + autogrowCase.producerType + ]) + expect( + missing, + `${autogrowCase.pack} not installed on this backend (missing: ${missing.join(', ')})` + ).toEqual([]) + expect( + await comfyPage.page.evaluate( + (extensionName) => + window.app!.extensions.some( + (extension) => extension.name === extensionName + ), + autogrowCase.extensionName + ), + `${autogrowCase.pack} autogrow extension ${autogrowCase.extensionName} is registered before S12` + ).toBe(true) + + for (const vueNodesEnabled of [false, true]) { + const consoleErrors = collectConsoleErrors(comfyPage.page) + await comfyPage.settings.setSetting( + 'Comfy.VueNodes.Enabled', + vueNodesEnabled + ) + + for (const connectPath of ['drag', 'programmatic'] as const) { + const context = `${autogrowCase.consumerType} via ${connectPath} with VueNodes=${vueNodesEnabled}` + await comfyPage.nodeOps.clearGraph() + const producer = await comfyPage.nodeOps.addNode( + autogrowCase.producerType, + undefined, + { x: 150, y: 200 } + ) + const consumer = await comfyPage.nodeOps.addNode( + autogrowCase.consumerType, + undefined, + { x: 700, y: 200 } + ) + await comfyPage.nextFrame() + const consumerId = String(consumer.id) + + // The DOM baseline below is a one-shot census, so the Vue nodes + // must be MOUNTED first or it captures a mid-mount undercount and + // the +1 growth poll chases a wrong absolute target. + if (vueNodesEnabled) { + await comfyPage.vueNodes.waitForNodes(2) + // Input 0 exists for every autogrow case; one visible dot proves + // the slot rows mounted, so the census below reads settled DOM. + await expect( + comfyPage.vueNodes.getInputSlotConnectionDot(consumerId, 0), + `${context}: consumer slot dots mounted before baseline` + ).toBeVisible() + } + + const before = await consumerShape(comfyPage.page, consumerId) + expect( + before.inputCount, + `${context}: consumer instantiates with at least one input` + ).toBeGreaterThan(0) + const lastIndex = before.inputCount - 1 + const outIndex = await comfyPage.page.evaluate( + ([producerId, slotName]) => { + const node = window.app!.graph.nodes.find( + (candidate) => String(candidate.id) === producerId + )! + return node.outputs.findIndex((slot) => slot.name === slotName) + }, + [String(producer.id), autogrowCase.producerSlot] as const + ) + expect( + outIndex, + `${context}: producer slot on instance` + ).toBeGreaterThan(-1) + + if (connectPath === 'drag') { + if (vueNodesEnabled) { + const outDot = comfyPage.vueNodes.getOutputSlotConnectionDot( + String(producer.id), + outIndex + ) + const inDot = comfyPage.vueNodes.getInputSlotConnectionDot( + consumerId, + lastIndex + ) + await outDot.dragTo(inDot) + } else { + await producer.connectOutput(outIndex, consumer, lastIndex) + } + } else { + await comfyPage.page.evaluate( + ([producerId, consumerId, out, input]) => { + const byId = (id: string) => + window.app!.graph.nodes.find( + (node) => String(node.id) === id + )! + byId(producerId).connect( + Number(out), + byId(consumerId), + Number(input) + ) + }, + [ + String(producer.id), + consumerId, + String(outIndex), + String(lastIndex) + ] as const + ) + } + await comfyPage.nextFrame() + + // The link itself must land on the LAST input: stricter than the + // handler needs (it grows on any connect), but a drag that falls + // back to an earlier slot is a hit-test regression this tier + // should surface, not silently absorb. + await expect + .poll( + () => + comfyPage.page.evaluate( + ([id, index]) => { + const node = window.app!.graph.nodes.find( + (candidate) => String(candidate.id) === id + ) + return node?.inputs?.[Number(index)]?.link != null + }, + [consumerId, String(lastIndex)] as const + ), + { message: `${context}: link lands on the last input` } + ) + .toBe(true) + + // The behavior under test: the pack's JS appends a fresh input. + await expect + .poll( + async () => + (await consumerShape(comfyPage.page, consumerId)).inputCount, + { message: `${context}: input count grows by one on connect` } + ) + .toBe(before.inputCount + 1) + + // Rendered growth: the new input must also EXIST as a slot row. + // Graph-side growth without a rendered row is the Vue reactivity + // gap the CombineRegionalPrompts incident exposed. + if (vueNodesEnabled) + await expect + .poll( + async () => + (await consumerShape(comfyPage.page, consumerId)).domSlotDots, + { message: `${context}: grown input renders a slot dot` } + ) + .toBe(before.domSlotDots + 1) + + await comfyPage.page.evaluate( + ([id, index]) => { + const node = window.app!.graph.nodes.find( + (candidate) => String(candidate.id) === id + )! + node.disconnectInput(Number(index)) + }, + [consumerId, String(lastIndex)] as const + ) + await comfyPage.nextFrame() + await expect + .poll( + async () => + (await consumerShape(comfyPage.page, consumerId)).inputCount, + { + message: `${context}: trailing empty input removed on disconnect` + } + ) + .toBe(before.inputCount) + + // Symmetric rendered-shrink assert: the reactivity gap applies to + // slot removal too - a stale rendered row after disconnect would + // otherwise pass. + if (vueNodesEnabled) + await expect + .poll( + async () => + (await consumerShape(comfyPage.page, consumerId)).domSlotDots, + { message: `${context}: removed input's slot dot unrenders` } + ) + .toBe(before.domSlotDots) + } + + consoleErrors.stop() + expect( + consoleErrors.errors.filter( + (error) => !isForeignExecutionNoise(error) + ), + `console errors with VueNodes=${vueNodesEnabled}` + ).toEqual([]) + for (const [surface, locator] of Object.entries( + errorSurfaces(comfyPage.page) + )) + await expect( + locator, + `after VueNodes=${vueNodesEnabled} pass: ${surface}` + ).toHaveCount(0) + } + }) + }) +} diff --git a/browser_tests/tests/customNodes/errorSurfaces.pure.spec.ts b/browser_tests/tests/customNodes/errorSurfaces.pure.spec.ts new file mode 100644 index 00000000000..11748d35332 --- /dev/null +++ b/browser_tests/tests/customNodes/errorSurfaces.pure.spec.ts @@ -0,0 +1,159 @@ +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import { + expectNoVisibleErrors, + trackVisibleErrors +} from '@e2e/fixtures/utils/errorSurfaces' + +test.beforeEach(async ({ page }) => { + await trackVisibleErrors(page) +}) + +test('a clean page passes every surface', async ({ page }) => { + await page.setContent('
no errors here
') + await expect(expectNoVisibleErrors(page, 'clean')).resolves.toBeUndefined() +}) + +test('a persistent error toast fails carrying its visible text', async ({ + page +}) => { + await page.setContent( + '
Failed to load workspace: HTTP 502
' + + '
Settings seed rejected
' + ) + const failure = await expectNoVisibleErrors(page, 'at startup').then( + () => undefined, + (error: unknown) => error + ) + expect(failure).toBeInstanceOf(Error) + const message = (failure as Error).message + expect(message).toContain('errorToasts') + expect(message).toContain('Failed to load workspace: HTTP 502') + expect(message).toContain('Settings seed rejected') +}) + +test('a closed page fails immediately with the real reason, not the sentinel', async ({ + page +}) => { + await page.setContent('
') + await page.close() + const failure = await expectNoVisibleErrors(page, 'after close').then( + () => undefined, + (error: unknown) => error + ) + expect(String(failure)).toMatch(/has been closed/) + expect(String(failure)).not.toContain('unreadable mid-poll') +}) + +test('a visible error toast fails after it clears before the assertion', async ({ + page +}) => { + await page.setContent('
') + await page.evaluate(() => { + const toast = document.createElement('div') + toast.id = 't' + toast.className = 'p-toast-message-error' + toast.textContent = 'momentary' + document.getElementById('root')!.append(toast) + setTimeout(() => toast.remove(), 800) + }) + await expect(page.locator('#t')).toHaveCount(0) + const failure = await expectNoVisibleErrors(page, 'transient').then( + () => undefined, + (error: unknown) => error + ) + expect(failure).toBeInstanceOf(Error) + expect(String(failure)).toContain('momentary') +}) + +test('an error class added after insertion fails after it clears', async ({ + page +}) => { + await page.setContent('
late error
') + await page.evaluate(() => { + const toast = document.getElementById('t')! + toast.classList.add('p-toast-message-error') + setTimeout(() => toast.classList.remove('p-toast-message-error'), 800) + }) + await expect(page.locator('#t')).not.toHaveClass('p-toast-message-error') + const failure = await expectNoVisibleErrors(page, 'class-added').then( + () => undefined, + (error: unknown) => error + ) + expect(failure).toBeInstanceOf(Error) + expect(String(failure)).toContain('late error') +}) + +test('a hidden error fails when an ancestor mutation reveals it', async ({ + page +}) => { + await page.setContent( + '' + ) + await page.evaluate(() => { + document.getElementById('container')!.removeAttribute('hidden') + }) + const failure = await expectNoVisibleErrors(page, 'revealed').then( + () => undefined, + (error: unknown) => error + ) + expect(failure).toBeInstanceOf(Error) + expect(String(failure)).toContain('revealed error') +}) + +test('DOM churn is sampled at a fixed rate, not per mutation', async ({ + page +}) => { + const probe = await page.context().newPage() + try { + const clockStart = new Date('2026-08-19T00:00:00Z') + await probe.clock.install({ time: clockStart }) + await probe.clock.pauseAt(clockStart) + await probe.setContent('
') + await probe.evaluate(() => { + const target = window as Window & + typeof globalThis & { __recorderQueries?: number } + const originalDocumentQuery = document.querySelectorAll.bind(document) + target.__recorderQueries = 0 + document.querySelectorAll = ((selector: string) => { + target.__recorderQueries! += 1 + return originalDocumentQuery(selector) + }) as typeof document.querySelectorAll + }) + await trackVisibleErrors(probe) + + const state = () => + probe.evaluate( + () => + ( + window as Window & + typeof globalThis & { __recorderQueries?: number } + ).__recorderQueries + ) + await expect(state()).resolves.toBe(1) + + await probe.evaluate(() => { + let parent = document.getElementById('root')! + for (let index = 0; index < 1_000; index += 1) { + parent.classList.toggle('connectivity-churn', index % 2 === 0) + const child = document.createElement('div') + parent.append(child) + parent = child + } + }) + await expect(state()).resolves.toBe(1) + await probe.clock.runFor(99) + await expect(state()).resolves.toBe(1) + await probe.clock.runFor(1) + await expect(state()).resolves.toBe(2) + await expect( + expectNoVisibleErrors(probe, 'after DOM churn') + ).resolves.toBeUndefined() + } finally { + await probe.close() + } +}) diff --git a/browser_tests/tests/customNodes/failureReport.pure.spec.ts b/browser_tests/tests/customNodes/failureReport.pure.spec.ts new file mode 100644 index 00000000000..7df39f0ee97 --- /dev/null +++ b/browser_tests/tests/customNodes/failureReport.pure.spec.ts @@ -0,0 +1,31 @@ +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import { failureSummary } from '@e2e/fixtures/customNode/failureReport' + +test.describe('failureSummary', () => { + test('keeps an empty failure label concise', () => { + expect(failureSummary('tier failures', [], 'tier-failures.json')).toBe( + 'tier failures' + ) + }) + + test('shows only the first line of the first ten failures', () => { + const failures = Array.from( + { length: 12 }, + (_, index) => `failure ${index + 1}\nlarge nested detail` + ) + const summary = failureSummary( + 'Example-Pack tier failures', + failures, + 'tier-failures.json' + ) + expect(summary).toContain('12 failure(s); first 10') + expect(summary).toContain('1. failure 1') + expect(summary).toContain('10. failure 10') + expect(summary).not.toContain('failure 11') + expect(summary).not.toContain('large nested detail') + expect(summary).toContain('full list in tier-failures.json') + }) +}) diff --git a/browser_tests/tests/customNodes/interactionProfiles.pure.spec.ts b/browser_tests/tests/customNodes/interactionProfiles.pure.spec.ts new file mode 100644 index 00000000000..a9799845598 --- /dev/null +++ b/browser_tests/tests/customNodes/interactionProfiles.pure.spec.ts @@ -0,0 +1,99 @@ +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import type { NodeInteractionProfile } from '@e2e/fixtures/customNode/interactionProfiles' +import { + comparePackProfiles, + diffShapes +} from '@e2e/fixtures/customNode/interactionProfiles' + +const GROWS: NodeInteractionProfile = { + connectFirst: ['+input:image2:IMAGE'], + connectLast: 'SAME_AS_FIRST', + disconnect: ['-input:image2:IMAGE'] +} + +function committedWith(nodes: Record) { + return { + recordedAt: { core: 'abc123', pin: 'def456' }, + schema: 1 as const, + nodes + } +} + +test.describe('S13 interaction profiles', () => { + // The sort is load-bearing: probesEqual compares deltas element by element. + test('diffShapes is a sorted, facet-tagged symmetric difference blind to slot order', () => { + const before = { + inputs: ['input:a:IMAGE', 'input:c:MASK'], + outputs: ['output:out:IMAGE'], + widgets: ['widget:w:INT'] + } + const after = { + inputs: ['input:c:MASK', 'input:b:IMAGE'], + outputs: ['output:out:IMAGE'], + widgets: ['widget:w:INT', 'widget:z:FLOAT'] + } + expect(diffShapes(before, after)).toEqual([ + '+input:b:IMAGE', + '+widget:z:FLOAT', + '-input:a:IMAGE' + ]) + expect(diffShapes(before, { ...before })).toEqual([]) + const reordered = { + ...before, + inputs: [...before.inputs].reverse() + } + expect(diffShapes(before, reordered)).toEqual([]) + }) + + test('compare fails closed on every drift class', () => { + const committed = committedWith({ NodeA: GROWS }) + expect( + comparePackProfiles({ pack: 'p', observed: {}, committed: null })[0] + ).toContain('no committed interaction profiles') + expect( + comparePackProfiles({ pack: 'p', observed: {}, committed })[0] + ).toContain('was not probed') + expect( + comparePackProfiles({ + pack: 'p', + observed: { NodeA: GROWS, NodeB: GROWS }, + committed + })[0] + ).toContain('no baseline entry') + const drifted = comparePackProfiles({ + pack: 'p', + observed: { NodeA: { ...GROWS, disconnect: [] } }, + committed + }) + expect(drifted[0]).toContain('interaction delta drifted') + expect(drifted[0]).toContain('disconnect') + expect(drifted[0]).toContain('recorded at core abc123') + expect( + comparePackProfiles({ pack: 'p', observed: { NodeA: GROWS }, committed }) + ).toEqual([]) + }) + + test('marker-vs-delta mismatches drift both directions', () => { + const committed = committedWith({ NodeA: GROWS }) + expect( + comparePackProfiles({ + pack: 'p', + observed: { NodeA: { ...GROWS, connectFirst: 'NO_PRODUCER' } }, + committed + })[0] + ).toContain('connectFirst') + const markerCommitted = committedWith({ + NodeA: { ...GROWS, connectFirst: 'NO_INPUTS' } + }) + expect( + comparePackProfiles({ + pack: 'p', + observed: { NodeA: GROWS }, + committed: markerCommitted + })[0] + ).toContain('connectFirst') + }) +}) diff --git a/browser_tests/tests/customNodes/interactionProfiles.spec.ts b/browser_tests/tests/customNodes/interactionProfiles.spec.ts new file mode 100644 index 00000000000..792107cbaf8 --- /dev/null +++ b/browser_tests/tests/customNodes/interactionProfiles.spec.ts @@ -0,0 +1,282 @@ +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import { SYNTH_PRODUCERS } from '@e2e/fixtures/customNode/autoRun' +import type { NodeInteractionProfile } from '@e2e/fixtures/customNode/interactionProfiles' +import { + INTERACTION_UNSTABLE_NODES, + comparePackProfiles, + hasCommittedProfile, + loadPackProfiles, + recordPackProfiles +} from '@e2e/fixtures/customNode/interactionProfiles' +import { + customNodesManifest, + loadManifest, + packIdentity +} from '@e2e/fixtures/customNode/manifest' +import type { RawNodeDef } from '@e2e/fixtures/customNode/typePairing' +import { + isTypeCompatible, + normalizeNodeDefs +} from '@e2e/fixtures/customNode/typePairing' +import { + customNodeSuiteSettings, + drainBackendToIdle, + trackSubmittedPrompts +} from '@e2e/fixtures/utils/customNodeSuite' + +// S13: lock every registered node's interaction-triggered shape deltas. +// Probes are pure browser-side graph interactions (no prompt is ever +// queued), so this tier carries no backend-queue exclusivity constraint. +const PROBE_CHUNK = 40 + +test.use({ initialSettings: customNodeSuiteSettings }) + +interface ProbePlan { + type: string + // absent = the node declares no connectable inputs + first?: { inputName: string; producer: string; producerOutput: number } + last?: { inputName: string; producer: string; producerOutput: number } +} + +function producerFor( + inputType: string +): { producer: string; producerOutput: number } | null { + const direct = SYNTH_PRODUCERS[inputType] + if (direct) + return { producer: direct.nodeType, producerOutput: direct.outputIndex } + for (const [outType, synth] of Object.entries(SYNTH_PRODUCERS)) + if (isTypeCompatible(outType, inputType)) + return { producer: synth.nodeType, producerOutput: synth.outputIndex } + return null +} + +function planProbes( + defs: Record, + pack: string +): ProbePlan[] { + return normalizeNodeDefs(defs) + .filter((node) => node.pack === pack) + .map((node) => { + const connectable = node.inputs + const plan: ProbePlan = { type: node.type } + const firstInput = connectable[0] + const lastInput = connectable[connectable.length - 1] + if (firstInput) { + const producer = producerFor(firstInput.type) + if (producer) plan.first = { inputName: firstInput.name, ...producer } + } + if (lastInput && lastInput !== firstInput) { + const producer = producerFor(lastInput.type) + if (producer) plan.last = { inputName: lastInput.name, ...producer } + } + return plan + }) + .sort((a, b) => a.type.localeCompare(b.type)) +} + +test.beforeEach(({ comfyPage }) => { + trackSubmittedPrompts(comfyPage.page) +}) + +const interactionProfileEntries = + customNodesManifest() === 'core' + ? loadManifest().filter((row) => + hasCommittedProfile(row.pack, packIdentity(row)) + ) + : [] + +for (const entry of interactionProfileEntries) { + test(`interaction profiles: ${entry.pack} @custom-nodes`, async ({ + comfyPage + }) => { + test.setTimeout(entry.timeoutMs + 120_000) + const defs = (await comfyPage.page.evaluate(() => + window.app!.api.getNodeDefs() + )) as unknown as Record + const plans = planProbes(defs, entry.pack) + expect( + plans.length, + `${entry.pack} has a committed S13 profile but registered no nodes` + ).toBeGreaterThan(0) + + const observed: Record = {} + const probeThrows: Record = {} + for (let start = 0; start < plans.length; start += PROBE_CHUNK) { + const chunk = plans.slice(start, start + PROBE_CHUNK) + const probed = await comfyPage.page.evaluate((probePlans) => { + // Shape entries are `kind:name:type`, sorted by the node-side diff; + // capture reads the INSTANCE (what pack JS materialized), not the def. + function shapeOf(node: { + inputs?: Array<{ name: string; type: unknown }> + outputs?: Array<{ name: string; type: unknown }> + widgets?: Array<{ name?: string; type?: string }> + }): { inputs: string[]; outputs: string[]; widgets: string[] } { + return { + inputs: (node.inputs ?? []).map( + (slot) => `input:${slot.name}:${String(slot.type)}` + ), + outputs: (node.outputs ?? []).map( + (slot) => `output:${slot.name}:${String(slot.type)}` + ), + widgets: (node.widgets ?? []).map( + (widget) => `widget:${widget.name ?? '?'}:${widget.type ?? '?'}` + ) + } + } + function diff( + before: ReturnType, + after: ReturnType + ): string[] { + const delta: string[] = [] + for (const facet of ['inputs', 'outputs', 'widgets'] as const) { + const beforeSet = new Set(before[facet]) + const afterSet = new Set(after[facet]) + for (const item of afterSet) + if (!beforeSet.has(item)) delta.push(`+${item}`) + for (const item of beforeSet) + if (!afterSet.has(item)) delta.push(`-${item}`) + } + return delta.sort() + } + const app = window.app! + const graph = app.graph + // Never reuse node ids within a page (widgetValueStore keys by id). + window.__cnIdBase = Math.max(window.__cnIdBase ?? 0, graph.last_node_id) + const results: Record< + string, + { + connectFirst: string[] | 'NO_PRODUCER' | 'NO_INPUTS' + connectLast: + | string[] + | 'NO_PRODUCER' + | 'NO_INPUTS' + | 'SAME_AS_FIRST' + disconnect: string[] | 'NO_PRODUCER' | 'NO_INPUTS' + } + > = {} + // One throwing node must not take its whole chunk with it: name it, + // drop what it created, and probe the rest of the chunk anyway. + const threw: Record = {} + for (const plan of probePlans) { + const node = window.LiteGraph!.createNode(plan.type) + if (!node) continue + try { + graph.last_node_id = ++window.__cnIdBase! + graph.add(node) + const fresh = shapeOf(node) + const probeConnect = (spec: { + inputName: string + producer: string + producerOutput: number + }) => { + const producerNode = window.LiteGraph!.createNode(spec.producer) + if (!producerNode) return null + try { + graph.last_node_id = ++window.__cnIdBase! + graph.add(producerNode) + const inputIndex = (node.inputs ?? []).findIndex( + (slot: { name: string }) => slot.name === spec.inputName + ) + if (inputIndex === -1) return null + producerNode.connect(spec.producerOutput, node, inputIndex) + const connected = shapeOf(node) + node.disconnectInput(inputIndex) + const disconnected = shapeOf(node) + return { connected, disconnected } + } finally { + if (producerNode.graph) graph.remove(producerNode) + } + } + const hasInputs = (node.inputs ?? []).length > 0 + if (!hasInputs) { + results[plan.type] = { + connectFirst: 'NO_INPUTS', + connectLast: 'NO_INPUTS', + disconnect: 'NO_INPUTS' + } + } else { + const first = plan.first ? probeConnect(plan.first) : null + const last = plan.last ? probeConnect(plan.last) : null + const anchor = last ?? first + results[plan.type] = { + connectFirst: first + ? diff(fresh, first.connected) + : 'NO_PRODUCER', + connectLast: plan.last + ? last + ? diff(fresh, last.connected) + : 'NO_PRODUCER' + : 'SAME_AS_FIRST', + disconnect: anchor + ? diff(anchor.connected, anchor.disconnected) + : 'NO_PRODUCER' + } + } + } catch (error) { + threw[plan.type] = String(error) + } finally { + if (node.graph) graph.remove(node) + } + } + return { results, threw } + }, chunk) + Object.assign(observed, probed.results) + Object.assign(probeThrows, probed.threw) + } + // The probes queue nothing, so this returns without a round-trip; it stays + // as the guard for pack JS that queues behind our back while being probed. + expect( + await drainBackendToIdle(comfyPage.page, 10_000), + 'interaction probe left test-owned backend work running' + ).toBe(0) + + // Ahead of the ledger and profile gates: a node that threw is absent from + // `observed`, which those would report as a stale baseline instead. + expect( + Object.entries(probeThrows).map( + ([node, error]) => `${entry.pack}/${node}: probe threw: ${error}` + ), + 'nodes that threw while being probed' + ).toEqual([]) + + for (const node of Object.keys( + INTERACTION_UNSTABLE_NODES[entry.pack] ?? {} + )) + expect( + node in observed, + `${entry.pack}/${node} is ledgered unstable but no longer in the corpus - stale ledger entry` + ).toBe(true) + + const recordMode = process.env.CN_INTERACTION + if (recordMode !== undefined && recordMode !== 'record') + throw new Error( + `unrecognized CN_INTERACTION value '${recordMode}' - the only mode is 'record'` + ) + if (recordMode === 'record') { + recordPackProfiles(entry.pack, observed, { + core: process.env.CN_INTERACTION_CORE ?? 'unpinned-local', + pin: packIdentity(entry) + }) + expect( + null, + `CN_INTERACTION=record: wrote ${Object.keys(observed).length} profile(s) for ${entry.pack} - the artifact is the product, this is not a pass` + ).not.toBeNull() + } else if (process.env.CI) { + expect( + comparePackProfiles({ + pack: entry.pack, + observed, + committed: loadPackProfiles(entry.pack) + }), + 'S13 interaction profiles' + ).toEqual([]) + } else { + console.log( + `S13 compare skipped off-CI for ${entry.pack} - baselines encode the pinned record environment; CI enforces` + ) + } + }) +} diff --git a/browser_tests/tests/customNodes/manifest.pure.spec.ts b/browser_tests/tests/customNodes/manifest.pure.spec.ts new file mode 100644 index 00000000000..7a554c19b9d --- /dev/null +++ b/browser_tests/tests/customNodes/manifest.pure.spec.ts @@ -0,0 +1,330 @@ +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import type { CoreManifestEntry } from '@e2e/fixtures/customNode/manifest' +import { + assertCoreEntry, + expectedNodeCountFor, + loadApplicableAutogrowCases, + loadFullManifest, + loadManifest, + servesFrontendAssetsForPack, + shardOf +} from '@e2e/fixtures/customNode/manifest' + +function validEntry(): CoreManifestEntry { + return { + pack: 'Example-Pack', + repo: 'https://github.com/example/Example-Pack', + pin: 'a1'.repeat(20), + tiers: ['load', 'connectivity', 'run'], + workflow: 'assets/customNodes/vhs_video_pipeline_run.json', + expectedNodes: ['ExampleNode'], + expectedRunnableCount: 1, + expectedNodeCount: 1, + expectedExtensions: ['Example.Extension'], + requiresGpu: false, + requiresModels: [], + timeoutMs: 60_000 + } +} + +test.describe('customNode manifest', () => { + test('loads entries with the shape the regression spec depends on', () => { + const entries = loadFullManifest() + expect(entries.length).toBeGreaterThan(0) + for (const entry of entries) { + expect(entry.pack).toBeTruthy() + expect(entry.expectedNodes.length).toBeGreaterThan(0) + expect(entry.tiers.length).toBeGreaterThan(0) + } + }) + + test('a local node-count baseline applies only to a local cloud-pack run', () => { + const priorBackend = process.env.CUSTOM_NODES_BACKEND + const priorManifest = process.env.CUSTOM_NODES_MANIFEST + const entry = { + ...validEntry(), + pack: 'comfyui-videohelpersuite', + expectedNodeCount: 32 + } + try { + process.env.CUSTOM_NODES_BACKEND = 'local' + process.env.CUSTOM_NODES_MANIFEST = 'cloud' + expect(expectedNodeCountFor(entry)).toBe(40) + process.env.CUSTOM_NODES_BACKEND = 'cloud' + expect(expectedNodeCountFor(entry)).toBe(32) + process.env.CUSTOM_NODES_BACKEND = 'local' + process.env.CUSTOM_NODES_MANIFEST = 'core' + expect(expectedNodeCountFor(entry)).toBe(32) + } finally { + if (priorBackend === undefined) delete process.env.CUSTOM_NODES_BACKEND + else process.env.CUSTOM_NODES_BACKEND = priorBackend + if (priorManifest === undefined) delete process.env.CUSTOM_NODES_MANIFEST + else process.env.CUSTOM_NODES_MANIFEST = priorManifest + } + }) + + test('pin must be a full commit SHA', () => { + expect(() => assertCoreEntry(validEntry(), 0)).not.toThrow() + expect(() => assertCoreEntry({ ...validEntry(), workflow: '' }, 0)).toThrow( + /workflow/ + ) + expect(() => + assertCoreEntry( + { + ...validEntry(), + workflow: 'assets/customNodes/not_committed.json' + }, + 0 + ) + ).toThrow(/workflow/) + expect(() => assertCoreEntry({ ...validEntry(), pin: '' }, 0)).toThrow( + /pin/ + ) + expect(() => + assertCoreEntry({ ...validEntry(), pin: 'abc123' }, 0) + ).toThrow(/pin/) + }) + + test('expectedExtensions is required; empty explicitly expects no healthy registration', () => { + // Omission must fail (a new pack row cannot silently opt out of the + // extension-loaded assert); an explicit [] is the deliberate opt-out. + const { expectedExtensions: _omitted, ...withoutField } = validEntry() + expect(() => assertCoreEntry(withoutField as CoreManifestEntry, 0)).toThrow( + /expectedExtensions/ + ) + expect(() => + assertCoreEntry({ ...validEntry(), expectedExtensions: [] }, 0) + ).not.toThrow() + expect(() => + assertCoreEntry({ ...validEntry(), expectedExtensions: [''] }, 0) + ).toThrow(/expectedExtensions/) + expect(() => + assertCoreEntry( + { ...validEntry(), expectedExtensions: [42 as unknown as string] }, + 0 + ) + ).toThrow(/expectedExtensions/) + expect(() => + assertCoreEntry({ ...validEntry(), expectedExtensions: ['A', 'A'] }, 0) + ).toThrow(/expectedExtensions/) + }) + + test('expectedNodeCount must be a positive integer', () => { + const { expectedNodeCount: _omitted, ...withoutField } = validEntry() + expect(() => assertCoreEntry(withoutField as CoreManifestEntry, 0)).toThrow( + /expectedNodeCount/ + ) + for (const bad of [0, -3, 1.5, Number.NaN]) { + expect(() => + assertCoreEntry({ ...validEntry(), expectedNodeCount: bad }, 0) + ).toThrow(/expectedNodeCount/) + } + expect(() => + assertCoreEntry({ ...validEntry(), expectedNodeCount: 197 }, 0) + ).not.toThrow() + }) + + test('run tiers require an exact nonzero runnable corpus', () => { + const { expectedRunnableCount: _omitted, ...withoutField } = validEntry() + expect(() => assertCoreEntry(withoutField as CoreManifestEntry, 0)).toThrow( + /expectedRunnableCount/ + ) + for (const bad of [0, -1, 1.5, Number.NaN]) + expect(() => + assertCoreEntry({ ...validEntry(), expectedRunnableCount: bad }, 0) + ).toThrow(/expectedRunnableCount/) + expect(() => + assertCoreEntry( + { + ...validEntry(), + tiers: ['load'], + workflow: '', + expectedRunnableCount: 1 + }, + 0 + ) + ).toThrow(/expectedRunnableCount/) + expect(() => + assertCoreEntry({ ...validEntry(), requiresGpu: true }, 0) + ).toThrow(/model-free CPU gate/) + expect(() => + assertCoreEntry( + { ...validEntry(), requiresModels: ['model.safetensors'] }, + 0 + ) + ).toThrow(/model-free CPU gate/) + }) + + test('pack must be a plain path segment (it becomes the install dirname)', () => { + for (const bad of ['../escape', 'a/b', '.hidden', 'sp ace', '']) + expect( + () => assertCoreEntry({ ...validEntry(), pack: bad }, 0), + `pack '${bad}' must be rejected` + ).toThrow(/pack/) + }) + + test('matches Impact frontend applicability to what the target serves', () => { + const priorBackend = process.env.CUSTOM_NODES_BACKEND + const priorManifest = process.env.CUSTOM_NODES_MANIFEST + const expectedCase = { + pack: 'ComfyUI-Impact-Pack', + extensionName: 'Comfy.Impack', + extensionPathPack: 'comfyui-impact-pack', + consumerType: 'ImpactMakeImageList', + producerType: 'EmptyImage', + producerSlot: 'IMAGE' + } + try { + process.env.CUSTOM_NODES_MANIFEST = 'core' + process.env.CUSTOM_NODES_BACKEND = 'local' + expect( + loadApplicableAutogrowCases().map(({ autogrowCase }) => autogrowCase) + ).toEqual([expectedCase]) + + process.env.CUSTOM_NODES_MANIFEST = 'cloud' + expect( + loadApplicableAutogrowCases().map(({ autogrowCase }) => autogrowCase) + ).toEqual([expectedCase]) + + process.env.CUSTOM_NODES_BACKEND = 'cloud' + expect(loadApplicableAutogrowCases()).toEqual([]) + } finally { + if (priorBackend === undefined) delete process.env.CUSTOM_NODES_BACKEND + else process.env.CUSTOM_NODES_BACKEND = priorBackend + if (priorManifest === undefined) delete process.env.CUSTOM_NODES_MANIFEST + else process.env.CUSTOM_NODES_MANIFEST = priorManifest + } + expect( + servesFrontendAssetsForPack( + ['/extensions/ComfyUI-Impact-Pack/js/impact.js'], + 'comfyui-impact-pack' + ) + ).toBe(true) + expect( + servesFrontendAssetsForPack( + ['/extensions/another-pack/main.js'], + 'comfyui-impact-pack' + ) + ).toBe(false) + }) + + test('a pack keeps its shard when another pack is excluded', () => { + // All packs in a shard share one Python environment, so a pack's + // neighbours decide which of its optional imports resolve and how many + // node classes it registers. Bin-packing the FILTERED list moved 26 of 81 + // packs when four left the quarantine, and every recorded node count moved + // with them - calibration could never converge. + const packs = Array.from({ length: 40 }, (_, i) => ({ + pack: `p${i}`, + weight: (i % 7) + 1 + })) + const original = process.env.CUSTOM_NODES_SHARD + const shardIndexOf = (excluded: string[]) => { + const dropped = new Set(excluded) + const where = new Map() + for (let index = 1; index <= 4; index++) { + process.env.CUSTOM_NODES_SHARD = `${index}/4` + for (const entry of shardOf(packs, (e) => e.weight)) + if (!dropped.has(entry.pack)) where.set(entry.pack, index) + } + return where + } + try { + const before = shardIndexOf([]) + const after = shardIndexOf(['p3', 'p11', 'p27']) + const moved = [...after].filter(([pack, i]) => before.get(pack) !== i) + expect(moved).toEqual([]) + } finally { + if (original === undefined) delete process.env.CUSTOM_NODES_SHARD + else process.env.CUSTOM_NODES_SHARD = original + } + }) + + test('cloud local calibration rejects an unreviewed shard count', () => { + const priorManifest = process.env.CUSTOM_NODES_MANIFEST + const priorBackend = process.env.CUSTOM_NODES_BACKEND + const priorShard = process.env.CUSTOM_NODES_SHARD + try { + process.env.CUSTOM_NODES_MANIFEST = 'cloud' + process.env.CUSTOM_NODES_BACKEND = 'local' + process.env.CUSTOM_NODES_SHARD = '1/4' + expect(() => loadManifest()).toThrow(/calibrated for 5 shards, got 4/) + process.env.CUSTOM_NODES_SHARD = '1/5' + expect(() => loadManifest()).not.toThrow() + } finally { + if (priorManifest === undefined) delete process.env.CUSTOM_NODES_MANIFEST + else process.env.CUSTOM_NODES_MANIFEST = priorManifest + if (priorBackend === undefined) delete process.env.CUSTOM_NODES_BACKEND + else process.env.CUSTOM_NODES_BACKEND = priorBackend + if (priorShard === undefined) delete process.env.CUSTOM_NODES_SHARD + else process.env.CUSTOM_NODES_SHARD = priorShard + } + }) + + test('sharding balances by weight, not by count', () => { + // The distribution that matters: expectedNodeCount runs 1..285 with a + // median of 6, so equal pack counts are not equal work. Counting packs + // per shard would pass on the 4.86x spread this replaced. + // Heavy packs on a stride the old round-robin shared a residue with: + // striping sent every one of them to the same shard (10.2x spread), + // which is the real distribution this replaced. + const heavy = Array.from({ length: 32 }, (_, i) => ({ + pack: `p${i}`, + weight: i % 4 === 2 ? 250 : 5 + })) + const original = process.env.CUSTOM_NODES_SHARD + try { + const loads: number[] = [] + for (let index = 1; index <= 4; index++) { + process.env.CUSTOM_NODES_SHARD = `${index}/4` + loads.push( + shardOf(heavy, (entry) => entry.weight).reduce( + (sum, entry) => sum + entry.weight, + 0 + ) + ) + } + // Striping this fixture gives 50x; balanced divides exactly. + expect(Math.max(...loads) / Math.min(...loads)).toBe(1) + } finally { + if (original === undefined) delete process.env.CUSTOM_NODES_SHARD + else process.env.CUSTOM_NODES_SHARD = original + } + }) + + test('sharding partitions the manifest - every pack exactly once', () => { + const packs = Array.from({ length: 87 }, (_, i) => `pack-${i}`) + const original = process.env.CUSTOM_NODES_SHARD + try { + for (const total of [10, 14, 18]) { + const seen: string[] = [] + for (let index = 1; index <= total; index++) { + process.env.CUSTOM_NODES_SHARD = `${index}/${total}` + seen.push(...shardOf(packs)) + } + // Loss is the failure that matters: a pack dropped from every shard + // reads as a smaller green run, not as missing coverage. + expect(seen.slice().sort()).toEqual(packs.slice().sort()) + } + } finally { + if (original === undefined) delete process.env.CUSTOM_NODES_SHARD + else process.env.CUSTOM_NODES_SHARD = original + } + }) + + test('an unshaped CUSTOM_NODES_SHARD fails rather than running everything', () => { + const original = process.env.CUSTOM_NODES_SHARD + try { + process.env.CUSTOM_NODES_SHARD = '11/10' + expect(() => shardOf(['a'])).toThrow(/out of range/) + process.env.CUSTOM_NODES_SHARD = 'half' + expect(() => shardOf(['a'])).toThrow(/index\/total/) + } finally { + if (original === undefined) delete process.env.CUSTOM_NODES_SHARD + else process.env.CUSTOM_NODES_SHARD = original + } + }) +}) diff --git a/browser_tests/tests/customNodes/objectInfoValidator.pure.spec.ts b/browser_tests/tests/customNodes/objectInfoValidator.pure.spec.ts new file mode 100644 index 00000000000..a6e0afad603 --- /dev/null +++ b/browser_tests/tests/customNodes/objectInfoValidator.pure.spec.ts @@ -0,0 +1,22 @@ +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import type { ObjectInfo } from '@e2e/fixtures/customNode/objectInfoValidator' +import { missingExpectedNodes } from '@e2e/fixtures/customNode/objectInfoValidator' + +const objectInfo: ObjectInfo = { + KSampler: { input: { required: { model: {}, seed: {} } } } +} + +test.describe('objectInfoValidator', () => { + test('missingExpectedNodes returns only the names absent from object_info', () => { + expect( + missingExpectedNodes(objectInfo, ['KSampler', 'Missing (rgthree)']) + ).toEqual(['Missing (rgthree)']) + }) + + test('missingExpectedNodes returns empty when every expected node is registered', () => { + expect(missingExpectedNodes(objectInfo, ['KSampler'])).toEqual([]) + }) +}) diff --git a/browser_tests/tests/customNodes/outputHashes.pure.spec.ts b/browser_tests/tests/customNodes/outputHashes.pure.spec.ts new file mode 100644 index 00000000000..591382605e3 --- /dev/null +++ b/browser_tests/tests/customNodes/outputHashes.pure.spec.ts @@ -0,0 +1,210 @@ +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import { mkdtempSync, readFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { + compareOutputHashes, + hashPngPixels, + hashSinkPayloads, + recordObservedHashes +} from '@e2e/fixtures/customNode/outputHashes' + +function pngChunk(type: string, data: Buffer): Buffer { + const length = Buffer.alloc(4) + length.writeUInt32BE(data.length) + const crc = Buffer.alloc(4) + return Buffer.concat([length, Buffer.from(type, 'latin1'), data, crc]) +} + +function pngWith(input: { idat: Buffer; text?: string }): Buffer { + const signature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]) + const chunks = [signature, pngChunk('IHDR', Buffer.alloc(13))] + if (input.text !== undefined) + chunks.push(pngChunk('tEXt', Buffer.from(input.text, 'latin1'))) + chunks.push(pngChunk('IDAT', input.idat)) + chunks.push(pngChunk('IEND', Buffer.alloc(0))) + return Buffer.concat(chunks) +} + +const never = () => Promise.reject(new Error('no file fetch expected')) + +test.describe('S15 output hashes', () => { + test('identical pixels hash identically regardless of embedded metadata', () => { + const idat = Buffer.from([1, 2, 3, 4, 5]) + const plain = hashPngPixels(pngWith({ idat })) + const withPrompt = hashPngPixels( + pngWith({ idat, text: 'prompt{"seed":42,"widgets":[1,2,3]}' }) + ) + expect(plain).toBe(withPrompt) + expect(plain).toMatch(/^sha256:[0-9a-f]{64}$/) + }) + + test('different pixels hash differently', () => { + const a = hashPngPixels(pngWith({ idat: Buffer.from([1, 2, 3]) })) + const b = hashPngPixels(pngWith({ idat: Buffer.from([1, 2, 4]) })) + expect(a).not.toBe(b) + }) + + test('non-PNG bytes and IDAT-less files throw instead of hashing', () => { + expect(() => hashPngPixels(Buffer.from('GIF89a not a png'))).toThrow( + /not a PNG/ + ) + const signature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]) + const noIdat = Buffer.concat([ + signature, + pngChunk('IHDR', Buffer.alloc(13)), + pngChunk('IEND', Buffer.alloc(0)) + ]) + expect(() => hashPngPixels(noIdat)).toThrow(/no IDAT/) + }) + + test('payload hashing is key-order independent and value sensitive', async () => { + const a = await hashSinkPayloads({ '5': { text: ['7'], b: 1 } }, never) + const b = await hashSinkPayloads({ '5': { b: 1, text: ['7'] } }, never) + const c = await hashSinkPayloads({ '5': { b: 1, text: ['8'] } }, never) + expect(a['5']).toBe(b['5']) + expect(a['5']).not.toBe(c['5']) + }) + + test('file refs canonicalize by extension: run-varying names do not churn, pixels do', async () => { + const idat = Buffer.from([9, 9, 9]) + const run1 = await hashSinkPayloads( + { '3': { images: [{ filename: 'ComfyUI_00001_.png', type: 'temp' }] } }, + async () => pngWith({ idat }) + ) + const run2 = await hashSinkPayloads( + { '3': { images: [{ filename: 'ComfyUI_00042_.png', type: 'temp' }] } }, + async () => pngWith({ idat, text: 'other-metadata' }) + ) + const changed = await hashSinkPayloads( + { '3': { images: [{ filename: 'ComfyUI_00001_.png', type: 'temp' }] } }, + async () => pngWith({ idat: Buffer.from([9, 9, 8]) }) + ) + expect(run1['3']).toBe(run2['3']) + expect(run1['3']).not.toBe(changed['3']) + // non-PNG refs canonicalize to the extension alone (no content fetch) + const video = await hashSinkPayloads( + { '3': { gifs: [{ filename: 'out_00001.mp4', type: 'temp' }] } }, + never + ) + expect(video['3']).toMatch(/^sha256:/) + }) + + test('compare fails closed on every drift class', () => { + const committed = { + recordedAt: { core: 'abc123', run: '42' }, + schema: 1 as const, + workflows: { 'pack/wf.json': { '5': 'sha256:aa', '7': 'sha256:bb' } } + } + expect( + compareOutputHashes({ + workflowKey: 'pack/other.json', + observed: {}, + committed + })[0] + ).toContain('no committed hashes') + expect( + compareOutputHashes({ + workflowKey: 'pack/wf.json', + observed: { '5': 'sha256:aa', '7': 'sha256:XX' }, + committed + })[0] + ).toContain('output hash changed') + expect( + compareOutputHashes({ + workflowKey: 'pack/wf.json', + observed: { '5': 'sha256:aa', '7': 'sha256:XX' }, + committed + })[0] + ).toContain('recorded at core abc123') + expect( + compareOutputHashes({ + workflowKey: 'pack/wf.json', + observed: { '5': 'sha256:aa' }, + committed + })[0] + ).toContain('output is gone') + expect( + compareOutputHashes({ + workflowKey: 'pack/wf.json', + observed: { '5': 'sha256:aa', '7': 'sha256:bb', '9': 'sha256:cc' }, + committed + })[0] + ).toContain('new output') + expect( + compareOutputHashes({ + workflowKey: 'pack/wf.json', + observed: { '5': 'sha256:aa', '7': 'sha256:bb' }, + committed + }) + ).toEqual([]) + }) + + test('record accumulates across calls to the same file (worker-restart survival)', () => { + const dir = mkdtempSync(join(tmpdir(), 's15-')) + const file = join(dir, 'recorded.json') + recordObservedHashes(file, 'a/one.json', { '1': 'sha256:aa' }) + recordObservedHashes(file, 'b/two.json', { '2': 'sha256:bb' }) + const written = JSON.parse(readFileSync(file, 'utf-8')) + expect(Object.keys(written.workflows).sort()).toEqual([ + 'a/one.json', + 'b/two.json' + ]) + expect(written.schema).toBe(1) + expect(written.recordedAt.run).toBeTruthy() + }) + + test('an empty committed entry fails because it proves no output content', () => { + const committed = { + recordedAt: { core: 'abc', run: '1' }, + schema: 1 as const, + workflows: { 'was/wf.json': {} } + } + expect( + compareOutputHashes({ + workflowKey: 'was/wf.json', + observed: {}, + committed + }) + ).toEqual([ + "S15: 'was/wf.json' has no committed output hashes - add an observable sink before recording its baseline" + ]) + }) + + test('a truncated PNG chunk throws instead of hashing', () => { + const good = pngWith({ idat: Buffer.from([1, 2, 3]) }) + expect(() => hashPngPixels(good.subarray(0, good.length - 12))).toThrow( + /no IEND/ + ) + // Cut inside the IDAT chunk (13 = IEND's 12 bytes + 1 byte of IDAT crc) + // so the chunk header is readable but its declared length overruns. + expect(() => hashPngPixels(good.subarray(0, good.length - 13))).toThrow( + /overruns the buffer/ + ) + }) + + test('value objects that merely contain a filename keep their sibling keys', async () => { + const bare = await hashSinkPayloads( + { '9': { filename: 'display.txt', text: ['A'] } }, + never + ) + const changed = await hashSinkPayloads( + { '9': { filename: 'display.txt', text: ['B'] } }, + never + ) + expect(bare['9']).not.toBe(changed['9']) + const ref1 = await hashSinkPayloads( + { '9': { gifs: [{ filename: 'a.mp4', type: 'temp', frame_rate: 8 }] } }, + never + ) + const ref2 = await hashSinkPayloads( + { '9': { gifs: [{ filename: 'b.mp4', type: 'temp', frame_rate: 9 }] } }, + never + ) + expect(ref1['9']).not.toBe(ref2['9']) + }) +}) diff --git a/browser_tests/tests/customNodes/packLedger.pure.spec.ts b/browser_tests/tests/customNodes/packLedger.pure.spec.ts new file mode 100644 index 00000000000..ddf0f97492e --- /dev/null +++ b/browser_tests/tests/customNodes/packLedger.pure.spec.ts @@ -0,0 +1,69 @@ +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import { + assertPackLedgerKeys, + packLedgerFor +} from '@e2e/fixtures/customNode/packLedger' + +const ledger = { + 'ComfyUI-VideoHelperSuite': { + VHS_LoadVideo: 'pack-owned upload widget' + } +} + +test.describe('packLedgerFor', () => { + test('matches cloud pack ids case-insensitively', () => { + expect(packLedgerFor(ledger, 'comfyui-videohelpersuite')).toEqual( + ledger['ComfyUI-VideoHelperSuite'] + ) + }) + + test('returns an empty ledger for an unknown pack', () => { + expect(packLedgerFor(ledger, 'Other-Pack')).toEqual({}) + }) + + test('rejects ambiguous folded keys', () => { + expect(() => + packLedgerFor( + { + Example: { NodeA: 'first' }, + example: { NodeB: 'second' } + }, + 'EXAMPLE' + ) + ).toThrow(/duplicate case-insensitive keys.*Example, example/) + }) +}) + +test.describe('assertPackLedgerKeys', () => { + test('accepts a differently-cased manifest key', () => { + expect(() => + assertPackLedgerKeys('ROUNDTRIP_VALUE_ALLOWLIST', ledger, [ + 'comfyui-videohelpersuite' + ]) + ).not.toThrow() + }) + + test('rejects a ledger pack absent from the manifest', () => { + expect(() => + assertPackLedgerKeys('ROUNDTRIP_VALUE_ALLOWLIST', ledger, ['Other-Pack']) + ).toThrow( + /ROUNDTRIP_VALUE_ALLOWLIST.*not in manifest: ComfyUI-VideoHelperSuite/ + ) + }) + + test('rejects duplicate case-insensitive ledger keys', () => { + expect(() => + assertPackLedgerKeys( + 'ROUNDTRIP_VALUE_ALLOWLIST', + { + Example: { NodeA: 'first' }, + example: { NodeB: 'second' } + }, + ['example'] + ) + ).toThrow(/case-insensitive duplicates: example/) + }) +}) diff --git a/browser_tests/tests/customNodes/promptError.pure.spec.ts b/browser_tests/tests/customNodes/promptError.pure.spec.ts new file mode 100644 index 00000000000..f919bfcfc07 --- /dev/null +++ b/browser_tests/tests/customNodes/promptError.pure.spec.ts @@ -0,0 +1,289 @@ +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import { + LocalDesktopTarget, + isServerSideFault, + summarizePromptError, + toPromptEvent +} from '@e2e/fixtures/customNode/ComfyTarget' + +import type { Page, Response } from '@playwright/test' + +// The curated-run happy path never executes summarizePromptError (it only +// runs on a VALIDATION_FAIL), so these cases keep a curated workflow rejection +// naming the node+input instead of rotting back to `{}`. +test.describe('summarizePromptError', () => { + test('names the node class and the failing input from node_errors', () => { + const body = { + error: { type: 'prompt_outputs_failed_validation', message: 'failed' }, + node_errors: { + '7': { + class_type: 'ImpactInt', + errors: [ + { type: 'value_not_in_list', message: 'msg', details: 'value' } + ], + dependent_outputs: [] + } + } + } + expect(summarizePromptError(body)).toBe('failed; ImpactInt: value') + }) + + test('accepts a string top-level error', () => { + expect(summarizePromptError({ error: 'bad request' })).toBe('bad request') + }) + + test('falls back to the node message when details is empty', () => { + const body = { + node_errors: { + '3': { + class_type: 'KSampler', + errors: [ + { type: 'x', message: 'required input missing', details: '' } + ], + dependent_outputs: [] + } + } + } + expect(summarizePromptError(body)).toBe('KSampler: required input missing') + }) + + test('returns undefined for an empty or non-object body', () => { + expect(summarizePromptError({})).toBeUndefined() + expect(summarizePromptError(null)).toBeUndefined() + expect(summarizePromptError('not an object')).toBeUndefined() + }) +}) + +test('normalizes trailing backend whitespace without changing the message', () => { + expect( + toPromptEvent({ + type: 'execution_error', + exception_message: 'Error node was called!\n' + }) + ).toMatchObject({ + type: 'execution_error', + error: { exceptionMessage: 'Error node was called!' } + }) + expect( + toPromptEvent({ + type: 'execution_error', + exception_message: ' leading whitespace is content \r\n' + }) + ).toMatchObject({ + error: { exceptionMessage: ' leading whitespace is content' } + }) +}) + +// A page whose /prompt POST answers with one scripted rejection. The first +// evaluate installs the event tap; every later one is a queue attempt, which +// delivers the response before reporting whether app.queuePrompt accepted it. +function rejectingPage(script: { + status: number + json: () => Promise + queueAccepted: boolean +}) { + const listeners = new Set<(response: Response) => void>() + let evaluateCalls = 0 + let waitedForExecution = false + const response = { + request: () => ({ method: () => 'POST' }), + url: () => 'http://backend/api/prompt', + status: () => script.status, + json: script.json + } as unknown as Response + const page = { + on: (_event: string, listener: (value: Response) => void) => { + listeners.add(listener) + }, + off: (_event: string, listener: (value: Response) => void) => { + listeners.delete(listener) + }, + evaluate: async () => { + evaluateCalls += 1 + if (evaluateCalls === 1) return [] + for (const listener of listeners) listener(response) + return script.queueAccepted + }, + waitForFunction: async () => { + waitedForExecution = true + throw new Error('a rejected submission cannot emit execution events') + } + } as unknown as Page + return { + run: () => + new LocalDesktopTarget().runWorkflow(page, { + expectedNodeIds: ['1'], + timeoutMs: 60_000 + }), + evaluateCalls: () => evaluateCalls, + waitedForExecution: () => waitedForExecution, + listenerCount: () => listeners.size + } +} + +test('returns a captured prompt rejection without waiting for execution events', async () => { + const fake = rejectingPage({ + status: 400, + json: () => + Promise.resolve({ + error: { message: 'prompt rejected' }, + node_errors: {} + }), + queueAccepted: true + }) + + expect(await fake.run()).toEqual({ + outcome: 'VALIDATION_FAIL', + executedNodes: [], + outputsByNode: {}, + clientError: 'prompt rejected' + }) + expect(fake.waitedForExecution()).toBe(false) + expect(fake.listenerCount()).toBe(0) +}) + +test('a 5xx rejection without a JSON body is an environment fault, not a pack verdict', async () => { + const fake = rejectingPage({ + status: 502, + json: () => Promise.reject(new SyntaxError('proxy HTML')), + queueAccepted: true + }) + + await expect(fake.run()).rejects.toThrow( + 'prompt submission failed server-side (HTTP 502 POST /prompt) - backend/environment fault, not a pack validation reject' + ) + expect(fake.waitedForExecution()).toBe(false) + expect(fake.listenerCount()).toBe(0) +}) + +// The 2026-08-09 testcloud incident shape: an out-of-order migration left +// ingest unable to INSERT jobs, every POST /prompt answered 500 +// DATABASE_ERROR, and the suite reported 1,563 per-node VALIDATION_FAIL +// "regressions" across 68 packs. A 5xx during submission must fail the tier +// once, naming the backend - it can never become a per-node verdict. +test('a backend 5xx during submission cannot be pinned on the node under test', async () => { + // The incident path: the 500 body carries NO node_errors, so + // app.queuePrompt's `queueResultOverride = !nodeErrors` resolves the + // submission TRUE and the environment verdict comes from the + // resolved-with-rejection classifier, not the double-refusal retry. + const fake = rejectingPage({ + status: 500, + json: () => + Promise.resolve({ + error: { + message: 'Failed to create job record', + type: 'DATABASE_ERROR', + details: '' + } + }), + queueAccepted: true + }) + + const failure = await fake.run().then( + () => undefined, + (error: unknown) => error + ) + // The batch driver catches THIS shape to stop submitting while still + // reporting every verdict collected before the fault - the predicate is + // part of the contract, not just the message text. + expect(isServerSideFault(failure)).toBe(true) + expect((failure as Error).message).toBe( + 'prompt submission failed server-side (HTTP 500 POST /prompt) - Failed to create job record [type: DATABASE_ERROR] - backend/environment fault, not a pack validation reject' + ) + expect( + isServerSideFault(new Error('VALIDATION_FAIL (client threw: boom)')) + ).toBe(false) + expect(fake.evaluateCalls()).toBe(2) + expect(fake.waitedForExecution()).toBe(false) + expect(fake.listenerCount()).toBe(0) +}) + +// The double-refusal side of the same fault: app.queuePrompt returns false +// only when the rejection body CARRIES node_errors (queueResultOverride = +// !nodeErrors), so this shape spends the client-flap retry before the +// environment verdict. +test('a 5xx whose body carries node_errors is refused twice and still faults to the environment', async () => { + const fake = rejectingPage({ + status: 500, + json: () => + Promise.resolve({ + error: { + message: 'Failed to create job record', + type: 'DATABASE_ERROR', + details: '' + }, + node_errors: { + '3': { + class_type: 'KSampler', + errors: [{ type: 'x', message: 'boom', details: 'exploded' }], + dependent_outputs: [] + } + } + }), + queueAccepted: false + }) + + const failure = await fake.run().then( + () => undefined, + (error: unknown) => error + ) + expect(isServerSideFault(failure)).toBe(true) + expect((failure as Error).message).toBe( + 'prompt submission failed server-side (HTTP 500 POST /prompt) - Failed to create job record; KSampler: exploded [type: DATABASE_ERROR] - backend/environment fault, not a pack validation reject' + ) + // Seen-set install + two refused queue attempts: the retry was spent. + expect(fake.evaluateCalls()).toBe(3) + expect(fake.waitedForExecution()).toBe(false) + expect(fake.listenerCount()).toBe(0) +}) + +test('keeps a successful retry when the older rejection body parses last', async () => { + const listeners = new Set<(response: Response) => void>() + let evaluateCalls = 0 + const response = (status: number, body: unknown, delayMs = 0): Response => + ({ + request: () => ({ method: () => 'POST' }), + url: () => 'http://backend/api/prompt', + status: () => status, + json: () => + new Promise((resolve) => setTimeout(() => resolve(body), delayMs)) + }) as unknown as Response + const page = { + on: (_event: string, listener: (value: Response) => void) => { + listeners.add(listener) + }, + off: (_event: string, listener: (value: Response) => void) => { + listeners.delete(listener) + }, + evaluate: async () => { + evaluateCalls += 1 + if (evaluateCalls === 1) return [] + if (evaluateCalls === 2) { + for (const listener of listeners) + listener( + response(500, { error: { message: 'older rejection' } }, 300) + ) + return false + } + if (evaluateCalls === 3) { + for (const listener of listeners) + listener(response(200, { prompt_id: 'successful-retry' })) + return true + } + return [{ type: 'execution_success', prompt_id: 'successful-retry' }] + }, + waitForFunction: async () => {} + } as unknown as Page + + const result = await new LocalDesktopTarget().runWorkflow(page, { + expectedNodeIds: [], + timeoutMs: 60_000 + }) + + expect(result.outcome).toBe('PASS') + expect(result.clientError).toBeUndefined() + expect(listeners.size).toBe(0) +}) diff --git a/browser_tests/tests/customNodes/runResult.pure.spec.ts b/browser_tests/tests/customNodes/runResult.pure.spec.ts new file mode 100644 index 00000000000..9cecc6a5319 --- /dev/null +++ b/browser_tests/tests/customNodes/runResult.pure.spec.ts @@ -0,0 +1,212 @@ +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import { + classifyRun, + describeRunOutcome +} from '@e2e/fixtures/customNode/runResult' + +test.describe('classifyRun', () => { + test('PASS when every expected node appears in the executing stream', () => { + const result = classifyRun({ + events: [ + { type: 'execution_start' }, + { type: 'executing', node: '1' }, + { type: 'executing', node: '2' }, + { type: 'executing', node: null }, + { type: 'execution_success' } + ], + expectedNodeIds: ['1', '2'] + }) + expect(result.outcome).toBe('PASS') + expect(result.executedNodes).toEqual(['1', '2']) + }) + + test('PASS when an expected node was served from cache instead of run', () => { + const result = classifyRun({ + events: [ + { type: 'executing', node: '1' }, + { type: 'execution_cached', nodes: ['2'] }, + { type: 'execution_success' } + ], + expectedNodeIds: ['1', '2'] + }) + expect(result.outcome).toBe('PASS') + expect(result.executedNodes).toEqual(['1', '2']) + }) + + test('PARTIAL when a node appears in neither the executing nor the cached stream', () => { + const result = classifyRun({ + events: [ + { type: 'execution_cached', nodes: ['1'] }, + { type: 'execution_success' } + ], + expectedNodeIds: ['1', '2'] + }) + expect(result.outcome).toBe('PARTIAL') + expect(result.executedNodes).toEqual(['1']) + }) + + test('PASS when a missing execution event has a non-null downstream output', () => { + const result = classifyRun({ + events: [ + { type: 'executed', node: 'sink', output: { text: ['value'] } }, + { type: 'execution_success' } + ], + expectedNodeIds: ['source'], + proofOutputNodeByExpectedNode: { source: 'sink' } + }) + expect(result.outcome).toBe('PASS') + }) + + test('PARTIAL when a downstream output is missing or null', () => { + for (const output of [undefined, null]) { + const result = classifyRun({ + events: [ + { type: 'executed', node: 'sink', output }, + { type: 'execution_success' } + ], + expectedNodeIds: ['source'], + proofOutputNodeByExpectedNode: { source: 'sink' } + }) + expect(result.outcome).toBe('PARTIAL') + } + }) + + test('a node named by both streams is counted once', () => { + const result = classifyRun({ + events: [ + { type: 'executing', node: '1' }, + { type: 'execution_cached', nodes: ['1', '2'] }, + { type: 'execution_success' } + ], + expectedNodeIds: ['1', '2'] + }) + expect(result.outcome).toBe('PASS') + expect(result.executedNodes).toEqual(['1', '2']) + }) + + test('EXECUTION_ERROR captures the failing node details', () => { + const result = classifyRun({ + events: [ + { type: 'executing', node: '1' }, + { + type: 'execution_error', + error: { + exceptionMessage: 'inputs have different channel counts', + exceptionType: 'ValueError', + nodeId: '1', + nodeType: 'AudioConcatenate' + } + } + ], + expectedNodeIds: ['1'] + }) + expect(result.outcome).toBe('EXECUTION_ERROR') + expect(result.error?.exceptionType).toBe('ValueError') + expect(describeRunOutcome(result)).toBe( + 'EXECUTION_ERROR (AudioConcatenate: ValueError - inputs have different channel counts)' + ) + }) + + test('describes an execution error even when Cloud omits node identity', () => { + expect( + describeRunOutcome({ + outcome: 'EXECUTION_ERROR', + executedNodes: [], + outputsByNode: {}, + error: { exceptionMessage: 'allocation failed' } + }) + ).toBe('EXECUTION_ERROR (allocation failed)') + }) + + test('an execution_error naming a node outside the queued graph is not this run', () => { + const result = classifyRun({ + events: [ + { type: 'executing', node: '1' }, + { + type: 'execution_error', + error: { exceptionMessage: 'stray', nodeId: '99' } + }, + { type: 'execution_success' } + ], + expectedNodeIds: ['1'], + graphNodeIds: ['1'] + }) + expect(result.outcome).toBe('PASS') + expect(result.error).toBeUndefined() + }) + + test('an execution_error inside the queued graph, or carrying no node id, is reported', () => { + const inGraph = classifyRun({ + events: [ + { type: 'executing', node: '1' }, + { + type: 'execution_error', + error: { exceptionMessage: 'ours', nodeId: '1' } + }, + { type: 'execution_success' } + ], + expectedNodeIds: ['1'], + graphNodeIds: ['1'] + }) + const anonymous = classifyRun({ + events: [ + { type: 'executing', node: '1' }, + { type: 'execution_error', error: { exceptionMessage: 'allocation' } }, + { type: 'execution_success' } + ], + expectedNodeIds: ['1'], + graphNodeIds: ['1'] + }) + expect(inGraph.outcome).toBe('EXECUTION_ERROR') + expect(inGraph.error?.exceptionMessage).toBe('ours') + expect(anonymous.outcome).toBe('EXECUTION_ERROR') + expect(anonymous.error?.exceptionMessage).toBe('allocation') + }) + + test('an executed event carries the ui payload without counting as a run', () => { + const result = classifyRun({ + events: [ + { type: 'executing', node: '1' }, + { type: 'executed', node: '1', output: { images: ['a.png'] } }, + { type: 'executed', node: '2', output: { images: ['b.png'] } }, + { type: 'executed', node: null, output: { images: [] } }, + { type: 'execution_success' } + ], + expectedNodeIds: ['1', '2'] + }) + expect(result.outputsByNode).toEqual({ + '1': { images: ['a.png'] }, + '2': { images: ['b.png'] } + }) + expect(result.executedNodes).toEqual(['1']) + expect(result.outcome).toBe('PARTIAL') + }) + + test('EXECUTION_ERROR when the run is interrupted', () => { + const result = classifyRun({ + events: [ + { type: 'executing', node: '1' }, + { type: 'execution_interrupted' } + ], + expectedNodeIds: ['1'] + }) + expect(result.outcome).toBe('EXECUTION_ERROR') + }) + + test('TIMEOUT when flagged or when no terminal event arrived', () => { + const flagged = classifyRun({ + events: [{ type: 'executing', node: '1' }], + expectedNodeIds: ['1'], + timedOut: true + }) + const noTerminal = classifyRun({ + events: [{ type: 'executing', node: '1' }], + expectedNodeIds: ['1'] + }) + expect(flagged.outcome).toBe('TIMEOUT') + expect(noTerminal.outcome).toBe('TIMEOUT') + }) +}) diff --git a/browser_tests/tests/customNodes/tierNodeExclusions.pure.spec.ts b/browser_tests/tests/customNodes/tierNodeExclusions.pure.spec.ts new file mode 100644 index 00000000000..e7d5fd22d70 --- /dev/null +++ b/browser_tests/tests/customNodes/tierNodeExclusions.pure.spec.ts @@ -0,0 +1,61 @@ +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import { + CUSTOM_NODE_TIER_NODE_EXCLUSIONS, + eligibleNodeTypesForTier, + tierNodeExclusionProblems +} from '@e2e/fixtures/customNode/tierNodeExclusions' + +test.describe('tier node exclusions', () => { + test('excludes only the named node from its exact tiers', () => { + expect( + eligibleNodeTypesForTier( + { identity: 'comfyui-itools@0.6.8', pack: 'comfyui-itools' }, + 'S1', + ['iToolsAddOverlay', 'iToolsCropImage'] + ) + ).toEqual(['iToolsAddOverlay']) + expect( + eligibleNodeTypesForTier( + { identity: 'comfyui-itools@0.6.8', pack: 'comfyui-itools' }, + 'S4', + ['iToolsAddOverlay', 'iToolsCropImage'] + ) + ).toEqual(['iToolsAddOverlay']) + }) + + test('fails when the pinned artifact or registered node changes', () => { + expect(() => + eligibleNodeTypesForTier( + { identity: 'comfyui-itools@0.6.9', pack: 'comfyui-itools' }, + 'S1', + ['iToolsCropImage'] + ) + ).toThrow(/manifest now uses comfyui-itools@0\.6\.9/) + expect(() => + eligibleNodeTypesForTier( + { identity: 'comfyui-itools@0.6.8', pack: 'comfyui-itools' }, + 'S1', + ['iToolsAddOverlay'] + ) + ).toThrow(/node that no longer registers/) + }) + + test('reports unknown and repinned entries and has unique tier keys', () => { + expect( + tierNodeExclusionProblems([ + { identity: 'comfyui-itools@0.6.9', pack: 'comfyui-itools' } + ]) + ).toEqual([ + 'comfyui-itools/iToolsCropImage is pinned to comfyui-itools@0.6.8, but the manifest now uses comfyui-itools@0.6.9' + ]) + expect(tierNodeExclusionProblems([])).toEqual([ + 'comfyui-itools/iToolsCropImage is not in the manifest' + ]) + expect(new Set(CUSTOM_NODE_TIER_NODE_EXCLUSIONS[0].tiers).size).toBe( + CUSTOM_NODE_TIER_NODE_EXCLUSIONS[0].tiers.length + ) + }) +}) diff --git a/browser_tests/tests/customNodes/typePairing.pure.spec.ts b/browser_tests/tests/customNodes/typePairing.pure.spec.ts new file mode 100644 index 00000000000..775f6774430 --- /dev/null +++ b/browser_tests/tests/customNodes/typePairing.pure.spec.ts @@ -0,0 +1,424 @@ +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import type { RawNodeDef } from '@e2e/fixtures/customNode/typePairing' +import { + isTypeCompatible, + normalizeNodeDefs, + packOf, + planPairs +} from '@e2e/fixtures/customNode/typePairing' + +const DEFS: Record = { + LatentSource: { + input: { required: {} }, + output: ['LATENT'], + output_name: ['LATENT'], + python_module: 'nodes' + }, + LatentSink: { + input: { required: { latent: ['LATENT', {}] } }, + output: [], + python_module: 'custom_nodes.SomePack' + }, + UnionSource: { + input: { required: {} }, + output: ['STRING,INT'], + output_name: ['value'], + python_module: 'nodes' + }, + IntSink: { + input: { required: { value: ['int', {}] } }, + output: [], + python_module: 'nodes' + }, + ComboNode: { + input: { required: { choice: [['a', 'b'], {}] } }, + output: [], + python_module: 'nodes' + }, + SocketlessNode: { + input: { required: { hidden: ['STRING', { socketless: true }] } }, + output: [], + python_module: 'nodes' + }, + WildcardNode: { + input: { required: { anything: ['*', {}] } }, + output: ['*'], + output_name: ['out'], + python_module: 'nodes' + }, + OrphanNode: { + input: { required: {} }, + output: ['NOBODY_CONSUMES_THIS'], + output_name: ['orphan'], + python_module: 'custom_nodes.OrphanPack' + } +} + +test.describe('typePairing', () => { + test('isTypeCompatible mirrors the real validator semantics', () => { + expect(isTypeCompatible('LATENT', 'LATENT')).toBe(true) + expect(isTypeCompatible('latent', 'LATENT')).toBe(true) + expect(isTypeCompatible('LATENT', 'IMAGE')).toBe(false) + expect(isTypeCompatible('STRING,INT', 'INT')).toBe(true) + expect(isTypeCompatible('STRING,INT', 'FLOAT')).toBe(false) + expect(isTypeCompatible('BBOX,', 'BBOX')).toBe(true) + expect(isTypeCompatible('BBOX,', 'IMAGE')).toBe(true) + expect(isTypeCompatible('*', 'ANYTHING')).toBe(true) + expect(isTypeCompatible('', 'ANYTHING')).toBe(true) + }) + + test('packOf attributes core vs custom pack', () => { + expect(packOf('nodes')).toBe('core') + expect(packOf('comfy_extras.nodes_x')).toBe('core') + expect(packOf('custom_nodes.ComfyUI-Impact-Pack')).toBe( + 'ComfyUI-Impact-Pack' + ) + expect(packOf(undefined)).toBe('core') + }) + + test('normalize maps COMBO literals and drops socketless inputs', () => { + const nodes = normalizeNodeDefs(DEFS) + const combo = nodes.find((n) => n.type === 'ComboNode')! + expect(combo.inputs).toEqual([ + { name: 'choice', type: 'COMBO', comboOptions: ['a', 'b'] } + ]) + const socketless = nodes.find((n) => n.type === 'SocketlessNode')! + expect(socketless.inputs).toEqual([]) + // socketless is a recognized shape deliberately left out of the matrix; + // it must never be recorded as an unknown slot. + expect(socketless.unknownSlots).toBeUndefined() + }) + + test('unrecognizable slot specs are recorded, never silently dropped', () => { + // A numeric input type and a numeric output type have no connectable + // socket type (slotTypeOf null): the slot leaves the corpus, but the + // drop must surface on the node and in the plan. + const nodes = normalizeNodeDefs({ + WeirdNode: { + input: { required: { strange: [42, {}], ok: ['INT', {}] } }, + output: [7, 'INT'], + python_module: 'custom_nodes.weird-pack' + } + }) + const weird = nodes.find((n) => n.type === 'WeirdNode')! + expect(weird.unknownSlots).toEqual(['strange', 'output[0]']) + expect(weird.inputs.map((s) => s.name)).toEqual(['ok']) + const plan = planPairs(nodes, ['WeirdNode']) + expect(plan.unknownShapes).toEqual([ + 'WeirdNode.strange', + 'WeirdNode.output[0]' + ]) + }) + + test('a short output_name list uses the production output_ fallback', () => { + const shortNodes = normalizeNodeDefs({ + ShortNames: { + input: { required: {} }, + output: ['IMAGE', 'MASK', ['alpha', 'beta']], + output_name: ['image'], + python_module: 'custom_nodes.short-names' + }, + MaskSink: { + input: { required: { mask: ['MASK', {}] } }, + output: [], + python_module: 'nodes' + } + }) + const short = shortNodes.find((node) => node.type === 'ShortNames')! + expect(short.outputs).toEqual([ + { name: 'image', type: 'IMAGE' }, + { name: 'output_1', type: 'MASK' }, + { + name: 'output_2', + type: 'COMBO', + comboOptions: ['alpha', 'beta'] + } + ]) + expect(short.unknownSlots).toBeUndefined() + expect( + planPairs(shortNodes, ['ShortNames', 'MaskSink']).pairs.map( + (pair) => + `${pair.producer.nodeType}.${pair.producer.slotName}->${pair.consumer.nodeType}.${pair.consumer.slotName}` + ) + ).toContain('ShortNames.output_1->MaskSink.mask') + + const [unnamed] = normalizeNodeDefs({ + Unnamed: { + input: { required: {} }, + output: ['IMAGE', 'MASK'], + python_module: 'custom_nodes.unnamed' + } + }) + expect(unnamed.outputs).toEqual([ + { name: 'output_0', type: 'IMAGE' }, + { name: 'output_1', type: 'MASK' } + ]) + + const [falsyName] = normalizeNodeDefs({ + FalsyName: { + input: { required: {} }, + output: ['IMAGE'], + output_name: [null], + python_module: 'custom_nodes.falsy-name' + } + }) + expect(falsyName.outputs).toEqual([{ name: 'output_0', type: 'IMAGE' }]) + + const [emptyName] = normalizeNodeDefs({ + EmptyName: { + input: { required: {} }, + output: ['IMAGE'], + output_name: [''], + python_module: 'custom_nodes.empty-name' + } + }) + expect(emptyName.outputs).toEqual([{ name: 'output_0', type: 'IMAGE' }]) + }) + + test('planPairs pairs exact and union types, deterministically', () => { + const nodes = normalizeNodeDefs(DEFS) + const plan = planPairs(nodes, ['LatentSink', 'IntSink']) + const keys = plan.pairs.map( + (p) => + `${p.producer.nodeType}.${p.producer.slotName}->${p.consumer.nodeType}.${p.consumer.slotName}` + ) + expect(keys).toContain('LatentSource.LATENT->LatentSink.latent') + expect(keys).toContain('UnionSource.value->IntSink.value') + const again = planPairs(nodes, ['LatentSink', 'IntSink']) + expect(again.pairs).toEqual(plan.pairs) + // The DEFS corpus is fully recognizable; unknownShapes stays empty. + expect(plan.unknownShapes).toEqual([]) + }) + + test('ledgered pairs are planned exactly in addition to representatives', () => { + const nodes = normalizeNodeDefs({ + ImageSource: { + input: { required: {} }, + output: ['IMAGE'], + output_name: ['image'], + python_module: 'custom_nodes.ExamplePack' + }, + AlphaSink: { + input: { required: { image: ['IMAGE', {}] } }, + output: [], + python_module: 'nodes' + }, + ExpectedSink: { + input: { required: { image: ['IMAGE', {}] } }, + output: [], + python_module: 'nodes' + } + }) + const plan = planPairs( + nodes, + ['ImageSource'], + ['ImageSource.image -> ExpectedSink.image'] + ) + const keys = plan.pairs.map( + (pair) => + `${pair.producer.nodeType}.${pair.producer.slotName} -> ${pair.consumer.nodeType}.${pair.consumer.slotName}` + ) + + expect(keys).toEqual([ + 'ImageSource.image -> AlphaSink.image', + 'ImageSource.image -> ExpectedSink.image' + ]) + expect(plan.requiredPairIssues).toEqual([]) + }) + + test('ledgered pairs fail closed when a live slot contract changes', () => { + const nodes = normalizeNodeDefs(DEFS) + const plan = planPairs( + nodes, + ['LatentSink'], + [ + 'LatentSource.missing -> LatentSink.latent', + 'AbsentSource.output -> LatentSink.latent' + ] + ) + + expect(plan.requiredPairIssues).toEqual([ + 'LatentSource.missing -> LatentSink.latent: declared slot is no longer present', + 'AbsentSource.output -> LatentSink.latent: unknown node type(s): AbsentSource' + ]) + }) + + test('COMBO slots with different vocabularies stay excluded', () => { + const nodes = normalizeNodeDefs({ + ComboSource: { + input: { required: {} }, + output: [['A', 'B', 'C']], + output_name: ['COMBO'], + python_module: 'nodes' + }, + ...DEFS + }) + const source = nodes.find((n) => n.type === 'ComboSource')! + expect(source.outputs).toEqual([ + { name: 'COMBO', type: 'COMBO', comboOptions: ['A', 'B', 'C'] } + ]) + // ComboNode.choice offers [a, b] - not the same vocabulary as [A, B, C]. + const plan = planPairs(nodes, ['ComboSource', 'ComboNode']) + expect(plan.pairs).toEqual([]) + expect(plan.combos.map((s) => `${s.nodeType}.${s.slotName}`)).toEqual([ + 'ComboSource.COMBO', + 'ComboNode.choice' + ]) + }) + + test('COMBO slots with an identical vocabulary pair up', () => { + const nodes = normalizeNodeDefs({ + SamplerNameSource: { + input: { required: {} }, + output: [['euler', 'ddim']], + output_name: ['COMBO'], + python_module: 'nodes' + }, + SamplerNameSink: { + input: { required: { sampler_name: [['euler', 'ddim'], {}] } }, + output: [], + python_module: 'nodes' + }, + ...DEFS + }) + const plan = planPairs(nodes, ['SamplerNameSource', 'SamplerNameSink']) + expect( + plan.pairs.map( + (p) => + `${p.producer.nodeType}.${p.producer.slotName}->${p.consumer.nodeType}.${p.consumer.slotName}` + ) + ).toEqual(['SamplerNameSource.COMBO->SamplerNameSink.sampler_name']) + expect(plan.combos).toEqual([]) + }) + + // Census-derived: transformed (V2-schema) defs carry combo inputs as the + // string 'COMBO' with options in the opts object. Same vocabulary must + // pair across forms, and a combo with no static options (remote/lazy) + // must never blind-match. + test('V2-form combos pair across forms by vocabulary; unknown options never pair', () => { + const nodes = normalizeNodeDefs({ + ListFormSource: { + input: { required: {} }, + output: [['x', 'y']], + output_name: ['COMBO'], + python_module: 'nodes' + }, + V2FormSink: { + input: { + required: { + dim: ['COMBO', { multiselect: false, options: ['y', 'x'] }] + } + }, + output: [], + python_module: 'nodes' + }, + RemoteComboSink: { + input: { + required: { + image: ['COMBO', { remote: { route: '/internal/files/output' } }] + } + }, + output: [], + python_module: 'nodes' + }, + ...DEFS + }) + const plan = planPairs(nodes, [ + 'ListFormSource', + 'V2FormSink', + 'RemoteComboSink' + ]) + expect( + plan.pairs.map( + (p) => + `${p.producer.nodeType}.${p.producer.slotName}->${p.consumer.nodeType}.${p.consumer.slotName}` + ) + ).toEqual(['ListFormSource.COMBO->V2FormSink.dim']) + expect(plan.combos.map((s) => `${s.nodeType}.${s.slotName}`)).toEqual([ + 'RemoteComboSink.image' + ]) + }) + + test('COMBO vocabulary matching ignores option order', () => { + // A wired input bypasses its own widget, so menu order and the + // options[0] default are not part of the wire contract - membership is. + const nodes = normalizeNodeDefs({ + ShuffledSource: { + input: { required: {} }, + output: [['ddim', 'euler']], + output_name: ['COMBO'], + python_module: 'nodes' + }, + SamplerNameSink: { + input: { required: { sampler_name: [['euler', 'ddim'], {}] } }, + output: [], + python_module: 'nodes' + }, + ...DEFS + }) + const plan = planPairs(nodes, ['ShuffledSource', 'SamplerNameSink']) + expect( + plan.pairs.map( + (p) => + `${p.producer.nodeType}.${p.producer.slotName}->${p.consumer.nodeType}.${p.consumer.slotName}` + ) + ).toEqual(['ShuffledSource.COMBO->SamplerNameSink.sampler_name']) + expect(plan.combos).toEqual([]) + }) + + test('wildcard slots are excluded, orphan types recorded not failed', () => { + const nodes = normalizeNodeDefs(DEFS) + const plan = planPairs(nodes, ['WildcardNode', 'OrphanNode']) + expect(plan.wildcards.map((w) => w.nodeType)).toEqual([ + 'WildcardNode', + 'WildcardNode' + ]) + expect(plan.orphans).toEqual([ + { + nodeType: 'OrphanNode', + pack: 'OrphanPack', + slotName: 'orphan', + slotType: 'NOBODY_CONSUMES_THIS', + dir: 'out' + } + ]) + expect(plan.pairs).toEqual([]) + }) + + test('an output whose output_name is not a string is recorded, never renamed', () => { + const nodes = normalizeNodeDefs({ + TwoCombos: { + input: { required: {} }, + output: [ + ['a', 'b'], + ['x', 'y'] + ], + output_name: [ + ['a', 'b'], + ['x', 'y'] + ], + python_module: 'custom_nodes.two-combos' + }, + ComboSinkXY: { + input: { required: { pick: [['x', 'y'], {}] } }, + output: [], + python_module: 'nodes' + } + }) + const twoCombos = nodes.find((node) => node.type === 'TwoCombos')! + expect(twoCombos.outputs).toEqual([]) + expect(twoCombos.unknownSlots).toEqual(['output[0].name', 'output[1].name']) + const plan = planPairs(nodes, ['TwoCombos', 'ComboSinkXY']) + expect(plan.pairs).toEqual([]) + expect(plan.unknownShapes).toEqual([ + 'TwoCombos.output[0].name', + 'TwoCombos.output[1].name' + ]) + expect( + plan.combos.map((slot) => `${slot.nodeType}.${slot.slotName}`) + ).toEqual(['ComboSinkXY.pick']) + }) +}) diff --git a/browser_tests/tests/customNodes/valueDrift.pure.spec.ts b/browser_tests/tests/customNodes/valueDrift.pure.spec.ts new file mode 100644 index 00000000000..4684fa7de02 --- /dev/null +++ b/browser_tests/tests/customNodes/valueDrift.pure.spec.ts @@ -0,0 +1,281 @@ +import { + comfyExpect as expect, + comfyPageFixture as test +} from '@e2e/fixtures/ComfyPage' +import { + initializationSignalsForTypes, + isCanvasPreviewImagePath, + matchesTopologyExpectation, + OUTPUT_TOPOLOGY_EXPECTATIONS_LITEGRAPH, + OUTPUT_TOPOLOGY_EXPECTATIONS_VUE, + pendingRestoredPreviewWidgets, + pendingRoundtripInitializations, + rendererLedgerFor, + ROUNDTRIP_NODE_LOSS_EXPECTATIONS_LITEGRAPH, + ROUNDTRIP_NODE_LOSS_EXPECTATIONS_VUE, + ROUNDTRIP_VALUE_ALLOWED_INDICES_LITEGRAPH, + ROUNDTRIP_VALUE_ALLOWED_INDICES_VUE, + ROUNDTRIP_VALUE_ALLOWED_KEYS_LITEGRAPH, + ROUNDTRIP_VALUE_ALLOWED_KEYS_VUE, + staleValueDriftIndices, + staleValueDriftKeys +} from '@e2e/fixtures/customNode/valueDrift' + +test.describe('rendererLedgerFor', () => { + test('selects only the active renderer ledger', () => { + const litegraph: Record = { LitegraphNode: '3' } + const vue: Record = { VueNode: '5' } + + expect(rendererLedgerFor(false, litegraph, vue)).toBe(litegraph) + expect(rendererLedgerFor(true, litegraph, vue)).toBe(vue) + }) +}) + +test.describe('output topology', () => { + test('accepts only the exact expected output transition', () => { + const expectation = { + before: 20, + after: 4, + reason: 'pack JS exposes only the default 4 of 20 declared outputs' + } + + expect(matchesTopologyExpectation(expectation, 20, 4)).toBe(true) + expect(matchesTopologyExpectation(expectation, 20, 3)).toBe(false) + expect(matchesTopologyExpectation(expectation, 19, 4)).toBe(false) + }) + + test('rejects an unledgered transition as a roundtrip exception', () => { + expect(matchesTopologyExpectation(undefined, 154, 8)).toBe(false) + }) + + test('pins pack-owned output trimming under both renderers', () => { + for (const ledger of [ + OUTPUT_TOPOLOGY_EXPECTATIONS_LITEGRAPH, + OUTPUT_TOPOLOGY_EXPECTATIONS_VUE + ]) { + expect(ledger['ComfyUI_Fill-Nodes'].FL_VideoBatchSplitter).toMatchObject({ + before: 20, + after: 4 + }) + expect(ledger['WhatDreamsCost-ComfyUI'].MultiImageLoader).toMatchObject({ + before: 51, + after: 1 + }) + } + }) +}) + +test('roundtrip initialization waits for pack-owned ready values', () => { + const signals = { + LoadAudioUI: { + property: '_initializing', + predicate: 'equals' as const, + value: false + }, + SAM3VideoSegmentation: { + property: '_hiddenInputs', + predicate: 'defined' as const + }, + iToolsPaintNode: { + predicate: 'widget-count' as const, + value: 33 + }, + ImageTransformKJ: { + predicate: 'widget-value' as const, + value: '{"fillColor":"#000000"}', + widget: 'bboxes' + } + } + expect( + pendingRoundtripInitializations( + signals, + { + LoadAudioUI: true, + SAM3VideoSegmentation: undefined, + iToolsPaintNode: 32, + ImageTransformKJ: '' + }, + false + ) + ).toEqual([ + 'LoadAudioUI (litegraph: expected false, observed true)', + 'SAM3VideoSegmentation (litegraph: expected defined, observed undefined)', + 'iToolsPaintNode (litegraph: expected 33 widgets, observed 32)', + 'ImageTransformKJ (litegraph: expected bboxes = "{\\"fillColor\\":\\"#000000\\"}", observed "")' + ]) + expect( + pendingRoundtripInitializations( + signals, + { + LoadAudioUI: false, + SAM3VideoSegmentation: {}, + iToolsPaintNode: 33, + ImageTransformKJ: '{"fillColor":"#000000"}' + }, + false + ) + ).toEqual([]) +}) + +test('roundtrip initialization signals apply only to their node batch', () => { + const signals = { + ImageTransformKJ: { + predicate: 'widget-value' as const, + value: '{"fillColor":"#000000"}', + widget: 'bboxes' + }, + SplineEditor: { + property: '_initialized', + predicate: 'defined' as const + } + } + + expect( + initializationSignalsForTypes(signals, [ + 'ImageTransformKJ', + 'UnrelatedNode' + ]) + ).toEqual({ ImageTransformKJ: signals.ImageTransformKJ }) + expect(initializationSignalsForTypes(signals, ['UnrelatedNode'])).toEqual({}) +}) + +test('roundtrip waits for required canvas previews after reload', () => { + const required = { + iToolsLoadImagePlus: ['$$canvas-image-preview'], + LoadImageWithExif: ['$$canvas-image-preview'] + } + + expect( + pendingRestoredPreviewWidgets(required, { + iToolsLoadImagePlus: ['image', 'upload'], + LoadImageWithExif: [ + 'image', + 'default_focal_mm', + 'upload', + '$$canvas-image-preview' + ] + }) + ).toEqual([ + 'iToolsLoadImagePlus: expected $$canvas-image-preview after reload, observed [image,upload]' + ]) + expect( + pendingRestoredPreviewWidgets(required, { + iToolsLoadImagePlus: ['image', 'upload', '$$canvas-image-preview'], + LoadImageWithExif: ['image', 'upload', '$$canvas-image-preview'] + }) + ).toEqual([]) +}) + +test('requires canvas previews only for supported image upload paths', () => { + for (const value of [ + 'input/example.png', + 'example.JPG', + 'example.jpeg [output]', + 'nested/example.webp[temp]' + ]) { + expect(isCanvasPreviewImagePath(value)).toBe(true) + } + + for (const value of [ + '(upload a mesh file)', + 'example.glb', + '', + null, + undefined + ]) { + expect(isCanvasPreviewImagePath(value)).toBe(false) + } + expect(isCanvasPreviewImagePath('input/example.png')).toBe(true) +}) + +test.describe('staleValueDriftIndices', () => { + test('requires every allowed node/index pair to be observed', () => { + expect( + staleValueDriftIndices({ ExampleNode: [3, 4] }, { ExampleNode: [3] }) + ).toEqual(['ExampleNode[4]']) + }) + + test('returns every missing index and ignores unrelated observations', () => { + expect( + staleValueDriftIndices( + { ExampleNode: [1, 5], MissingNode: [0] }, + { ExampleNode: [1, 5], OtherNode: [0] } + ) + ).toEqual(['MissingNode[0]']) + }) +}) + +test.describe('staleValueDriftKeys', () => { + test('requires every allowed node/key pair to be observed', () => { + expect( + staleValueDriftKeys( + { ExampleNode: ['button', 'preview'] }, + { ExampleNode: ['button'] } + ) + ).toEqual(['ExampleNode.preview']) + }) + + test('ignores unrelated observations', () => { + expect( + staleValueDriftKeys( + { ExampleNode: ['button'] }, + { ExampleNode: ['button'], OtherNode: ['preview'] } + ) + ).toEqual([]) + }) +}) + +test.describe('cloud roundtrip expectations', () => { + test('pins only the observed widget indices', () => { + expect(ROUNDTRIP_VALUE_ALLOWED_INDICES_LITEGRAPH).toMatchObject({ + 'ComfyUI_Fill-Nodes': { + FL_ColorPicker: '3,4,5,6', + FL_ReplaceColor: '5,6,7,8,9,10,11,12' + }, + 'ComfyUI-KJNodes': { SplineEditor: '1' }, + 'ComfyUI-LTXVideo': { LTXVSparseTrackEditor: '1' }, + 'WhatDreamsCost-ComfyUI': { + LoadAudioUI: '2,3,5', + LTXDirector: '3,4,5,7' + }, + 'comfyui-itools': { iToolsRegexNode: '0' } + }) + expect(ROUNDTRIP_VALUE_ALLOWED_INDICES_VUE).toMatchObject({ + 'ComfyUI_Fill-Nodes': + ROUNDTRIP_VALUE_ALLOWED_INDICES_LITEGRAPH['ComfyUI_Fill-Nodes'], + 'ComfyUI-KJNodes': { SplineEditor: '1' }, + 'ComfyUI-LTXVideo': { LTXVSparseTrackEditor: '1' }, + 'WhatDreamsCost-ComfyUI': { + LoadAudioUI: '2,3,5', + LTXDirector: '3,4,5,7' + }, + 'comfyui-itools': { iToolsRegexNode: '0' } + }) + expect(ROUNDTRIP_VALUE_ALLOWED_KEYS_LITEGRAPH).toEqual({ + 'ComfyUI-VideoHelperSuite': { + VHS_LoadAudioUpload: 'choose audio to upload', + VHS_LoadImages: 'choose folder to upload', + VHS_LoadVideo: 'choose video to upload', + VHS_LoadVideoFFmpeg: 'choose video to upload', + VHS_VAEDecodeBatched: 'per_batch', + VHS_VAEEncodeBatched: 'per_batch' + } + }) + expect(ROUNDTRIP_VALUE_ALLOWED_KEYS_VUE).toEqual( + ROUNDTRIP_VALUE_ALLOWED_KEYS_LITEGRAPH + ) + }) + + test('keeps the FL_TimeLine loss temporary and renderer-explicit', () => { + expect( + ROUNDTRIP_NODE_LOSS_EXPECTATIONS_LITEGRAPH['ComfyUI_Fill-Nodes'] + .FL_TimeLine + ).toMatchObject({ + reason: expect.stringContaining('node.serialize'), + restore: expect.stringContaining('remove this entry') + }) + expect(ROUNDTRIP_NODE_LOSS_EXPECTATIONS_VUE).toEqual( + ROUNDTRIP_NODE_LOSS_EXPECTATIONS_LITEGRAPH + ) + }) +}) diff --git a/browser_tests/tests/widget.spec.ts b/browser_tests/tests/widget.spec.ts index ed70f5b4f86..d5d08ff9760 100644 --- a/browser_tests/tests/widget.spec.ts +++ b/browser_tests/tests/widget.spec.ts @@ -1,6 +1,7 @@ import { expect } from '@playwright/test' import { comfyPageFixture as test } from '@e2e/fixtures/ComfyPage' +import { getWav } from '@e2e/fixtures/components/AudioPreview' import { DefaultGraphPositions } from '@e2e/fixtures/constants/defaultGraphPositions' test.beforeEach(async ({ comfyPage }) => { @@ -400,9 +401,45 @@ test.describe( ) test.describe('Load audio widget', { tag: ['@screenshot', '@widget'] }, () => { - test('Can load audio', async ({ comfyPage }) => { + test('Hides an empty preview and loads uploaded audio', async ({ + comfyPage, + comfyFiles + }) => { await comfyPage.workflow.loadWorkflow('widgets/load_audio_widget') - await expect(comfyPage.page.locator('.comfy-audio')).toBeVisible() + const audioPreview = comfyPage.page.locator('.comfy-audio') + await expect(audioPreview).toBeHidden() + + const [loadAudioNode] = + await comfyPage.nodeOps.getNodeRefsByType('LoadAudio') + if (!loadAudioNode) throw new Error('LoadAudio node not found') + const audioWidget = await loadAudioNode.getWidgetByName('audio') + const uploadWidget = await loadAudioNode.getWidgetByName('upload') + const filename = 'test-audio.wav' + const uploadResponse = comfyPage.page.waitForResponse((response) => + response.url().includes('/upload/image') + ) + const fileChooser = comfyPage.page.waitForEvent('filechooser') + + await uploadWidget.click() + await ( + await fileChooser + ).setFiles({ + name: filename, + buffer: getWav(), + mimeType: 'audio/x-wav' + }) + comfyFiles.deleteAfterTest({ filename, type: 'input' }) + expect((await uploadResponse).status()).toBe(200) + + await expect.poll(() => audioWidget.getValue()).toBe(filename) + await expect(audioPreview).toBeVisible() + await expect + .poll(() => + audioPreview.evaluate((audio: HTMLAudioElement) => + new URL(audio.src).searchParams.get('filename') + ) + ) + .toBe(filename) await expect(comfyPage.canvas).toHaveScreenshot('load_audio_widget.png') }) }) diff --git a/browser_tests/tests/widget.spec.ts-snapshots/load-audio-widget-chromium-linux.png b/browser_tests/tests/widget.spec.ts-snapshots/load-audio-widget-chromium-linux.png index 2980d85812d..2fcdc14d349 100644 Binary files a/browser_tests/tests/widget.spec.ts-snapshots/load-audio-widget-chromium-linux.png and b/browser_tests/tests/widget.spec.ts-snapshots/load-audio-widget-chromium-linux.png differ diff --git a/docs/custom-node-regression-suite.md b/docs/custom-node-regression-suite.md new file mode 100644 index 00000000000..0612183d1bb --- /dev/null +++ b/docs/custom-node-regression-suite.md @@ -0,0 +1,16 @@ +# Custom-Node Regression Suite - Technical Design Doc + +The source of truth for this document lives in Notion (commentable, +edited by the team): + +**https://app.notion.com/p/3ac6d73d365081d19be4da63fad17589** + +CI failure messages cite this path with step names (e.g. "Step 5c") - +find those steps in the Notion page's "Onboarding a pack" section. +Step headings and quoted assertion messages there are pinned by CI +failure strings and pure specs: renaming them requires a frontend PR, +not just a Notion edit. + +Spec files: `browser_tests/tests/customNodes/`. Fixtures: +`browser_tests/fixtures/customNode/`. Quick start: +`pnpm test:custom-nodes:local`. diff --git a/package.json b/package.json index 1ce6816fc2b..cf29afe280b 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,15 @@ "test:browser": "pnpm exec playwright test", "test:browser:coverage": "cross-env COLLECT_COVERAGE=true pnpm test:browser", "test:browser:local": "cross-env PLAYWRIGHT_LOCAL=1 PLAYWRIGHT_TEST_URL=http://localhost:5173 pnpm test:browser", + "custom-node-quarantine": "tsx scripts/customNodeQuarantine.ts", + "custom-node-shard": "tsx scripts/customNodeShard.ts", + "gen:cloud-manifest": "tsx scripts/cloud-manifest.ts", + "test:custom-nodes": "cross-env PLAYWRIGHT_TEST_URL=http://localhost:5173 pnpm exec playwright test browser_tests/tests/customNodes/ --config playwright.chrome.config.ts --workers=1", + "test:custom-nodes:ci": "cross-env PLAYWRIGHT_TEST_URL=http://localhost:8188 pnpm exec playwright test browser_tests/tests/customNodes/ --config playwright.chrome.config.ts --workers=1", + "test:custom-nodes:watch": "cross-env PLAYWRIGHT_TEST_URL=http://localhost:5173 PLAYWRIGHT_LOCAL=1 SLOW_MO=300 pnpm exec playwright test browser_tests/tests/customNodes/customNode.regression.spec.ts browser_tests/tests/customNodes/connectivity.spec.ts --config playwright.chrome.config.ts --workers=1 --headed", + "test:custom-nodes:debug": "cross-env PLAYWRIGHT_TEST_URL=http://localhost:5173 pnpm exec playwright test browser_tests/tests/customNodes/customNode.regression.spec.ts browser_tests/tests/customNodes/connectivity.spec.ts --config playwright.chrome.config.ts --workers=1 --debug", + "test:custom-nodes:connectivity": "pnpm test:custom-nodes:debug -g \"connectivity\"", + "test:custom-nodes:self-check": "pnpm test:custom-nodes:watch -g \"self-check\"", "test:coverage": "vitest run --coverage", "test:unit": "vitest run", "typecheck": "vue-tsc --noEmit", @@ -66,7 +75,10 @@ "typecheck:scripts": "vue-tsc --project scripts/tsconfig.json", "typecheck:tools": "vue-tsc --project tools/tsconfig.json", "typecheck:website": "pnpm --filter @comfyorg/website run typecheck", - "zipdist": "node scripts/zipdist.js" + "zipdist": "node scripts/zipdist.js", + "test:custom-nodes:local": "start-server-and-test backend:custom-nodes http-get://127.0.0.1:8288/system_stats dev:custom-nodes http-get://localhost:5173 test:custom-nodes", + "backend:custom-nodes": "sh -c 'cd \"${TEST_COMFYUI_DIR:?point at your ComfyUI checkout (README Prerequisites)}\"; cp \"$OLDPWD/browser_tests/assets/plain_video.mp4\" input/ || { echo \"could not stage plain_video.mp4 into $PWD/input - the curated VHS workflow cannot run without it\" >&2; exit 1; }; PY=python3; [ -x venv/bin/python ] && PY=venv/bin/python; [ -x .venv/bin/python ] && PY=.venv/bin/python; \"$PY\" main.py --port 8288 --multi-user --cache-none'", + "dev:custom-nodes": "cross-env DEV_SERVER_COMFYUI_URL=http://127.0.0.1:8288 vite --config vite.config.mts --strictPort" }, "dependencies": { "@alloc/quick-lru": "catalog:", @@ -200,6 +212,7 @@ "postcss-html": "catalog:", "pretty-bytes": "catalog:", "rollup-plugin-visualizer": "catalog:", + "start-server-and-test": "catalog:", "storybook": "catalog:", "stylelint": "catalog:", "tailwindcss": "catalog:", @@ -218,6 +231,7 @@ "vue-component-type-helpers": "catalog:", "vue-eslint-parser": "catalog:", "vue-tsc": "catalog:", + "yaml": "catalog:", "zip-dir": "^2.0.0", "zod-to-json-schema": "catalog:" }, diff --git a/playwright.chrome.config.ts b/playwright.chrome.config.ts new file mode 100644 index 00000000000..4739840521d --- /dev/null +++ b/playwright.chrome.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from '@playwright/test' + +import base from './playwright.config' + +// Local runs against the system-installed Google Chrome (no bundled-chromium +// download). trace is kept on failure so a failed local run leaves a viewable +// Playwright trace (the primary reason to reach for this config); video stays +// off since the trace already carries screenshots + DOM snapshots and video is +// the heavier artifact. +export default defineConfig(base, { + use: { channel: 'chrome', video: 'off', trace: 'retain-on-failure' } +}) diff --git a/playwright.config.ts b/playwright.config.ts index db1e49094e4..b2454183470 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,6 +1,19 @@ import type { PlaywrightTestConfig } from '@playwright/test' import { defineConfig, devices } from '@playwright/test' +const distribution = process.env.DISTRIBUTION +Object.assign(globalThis, { + __DISTRIBUTION__: + distribution === 'desktop' || + distribution === 'localhost' || + distribution === 'cloud' + ? distribution + : process.env.DEV_SERVER_COMFYUI_URL?.includes('.comfy.org') + ? 'cloud' + : 'localhost', + __IS_NIGHTLY__: process.env.IS_NIGHTLY === 'true' +}) + const maybeLocalOptions: PlaywrightTestConfig = process.env.PLAYWRIGHT_LOCAL ? { timeout: 30_000, @@ -41,7 +54,25 @@ export default defineConfig({ name: 'chromium', use: { ...devices['Desktop Chrome'] }, timeout: 15000, - grepInvert: /@mobile|@perf|@audit|@cloud/ + grepInvert: /@mobile|@perf|@audit|@cloud|@custom-nodes/ + }, + + // The custom-node suite needs the manifest packs installed and a quiet + // backend queue, so it runs in its own gating job + // (ci-tests-custom-nodes.yaml) with --workers=1, not alongside the + // parallel main e2e shards: the auto-run tier waits on the WHOLE queue + // to go quiet before it measures, which a parallel shard cannot provide. + // retain-on-failure trace: ~40 serial tests, negligible overhead, and a + // red run needs 6 installed packs + a live backend to reproduce. + { + name: 'custom-nodes', + use: { + ...devices['Desktop Chrome'], + trace: 'retain-on-failure' + }, + timeout: 15000, + grep: /@custom-nodes/, + fullyParallel: false }, { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9d8b152d89c..8189a950ca7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -330,6 +330,9 @@ catalogs: rollup-plugin-visualizer: specifier: ^6.0.4 version: 6.0.11 + start-server-and-test: + specifier: ^3.0.11 + version: 3.0.12 storybook: specifier: ^10.5.6 version: 10.5.6 @@ -546,7 +549,7 @@ importers: version: 14.4.0(vue@3.5.41(typescript@6.0.3)) '@vueuse/integrations': specifier: 'catalog:' - version: 14.4.0(axios@1.18.1)(fuse.js@7.0.0)(vue@3.5.41(typescript@6.0.3)) + version: 14.4.0(axios@1.18.1(debug@4.4.3))(fuse.js@7.0.0)(vue@3.5.41(typescript@6.0.3)) '@vueuse/router': specifier: ^14.4.0 version: 14.4.0(vue-router@5.2.0(@vue/compiler-sfc@3.5.41)(esbuild@0.28.1)(pinia@4.0.2(@vue/devtools-api@8.1.5)(typescript@6.0.3)(vue@3.5.41(typescript@6.0.3)))(rolldown@1.2.3)(vite@8.2.1(@types/node@25.9.5)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0))(vue@3.5.41(typescript@6.0.3)))(vue@3.5.41(typescript@6.0.3)) @@ -567,7 +570,7 @@ importers: version: 5.56.0 axios: specifier: 'catalog:' - version: 1.18.1 + version: 1.18.1(debug@4.4.3) chart.js: specifier: ^4.5.0 version: 4.5.1 @@ -836,6 +839,9 @@ importers: rollup-plugin-visualizer: specifier: 'catalog:' version: 6.0.11(rolldown@1.2.3) + start-server-and-test: + specifier: 'catalog:' + version: 3.0.12 storybook: specifier: 'catalog:' version: 10.5.6(@types/react@19.2.17)(prettier@3.9.5)(react@19.2.8) @@ -890,6 +896,9 @@ importers: vue-tsc: specifier: 'catalog:' version: 3.3.9(typescript@6.0.3) + yaml: + specifier: ^2.8.3 + version: 2.9.0 zip-dir: specifier: ^2.0.0 version: 2.0.0 @@ -1135,7 +1144,7 @@ importers: dependencies: axios: specifier: 'catalog:' - version: 1.18.1 + version: 1.18.1(debug@4.4.3) devDependencies: typescript: specifier: 'catalog:' @@ -2253,6 +2262,26 @@ packages: engines: {node: '>=6'} hasBin: true + '@hapi/address@5.1.1': + resolution: {integrity: sha512-A+po2d/dVoY7cYajycYI43ZbYMXukuopIsqCjh5QzsBCipDtdofHntljDlpccMjIfTy6UOkg+5KPriwYch2bXA==} + engines: {node: '>=14.0.0'} + + '@hapi/formula@3.0.2': + resolution: {integrity: sha512-hY5YPNXzw1He7s0iqkRQi+uMGh383CGdyyIGYtB+W5N3KHPXoqychklvHhKCC9M3Xtv0OCs/IHw+r4dcHtBYWw==} + + '@hapi/hoek@11.0.7': + resolution: {integrity: sha512-HV5undWkKzcB4RZUusqOpcgxOaq6VOAH7zhhIr2g3G8NF/MlFO75SjOr2NfuSx0Mh40+1FqCkagKLJRykUWoFQ==} + + '@hapi/pinpoint@2.0.1': + resolution: {integrity: sha512-EKQmr16tM8s16vTT3cA5L0kZZcTMU5DUOZTuvpnY738m+jyP3JIUj+Mm1xc1rsLkGBQ/gVnfKYPwOmPg1tUR4Q==} + + '@hapi/tlds@1.1.7': + resolution: {integrity: sha512-MgNjRwy9Ti92yVAixLmDc8dd1bJIKwO9qlWCfFQRwRmUEDPQHYn4G6hwPFvFGUTzAa0FsS+inMjLin7GnyBRhA==} + engines: {node: '>=14.0.0'} + + '@hapi/topo@6.0.2': + resolution: {integrity: sha512-KR3rD5inZbGMrHmgPxsJ9dbi6zEK+C3ZwUwTa+eMwWLz7oijWUTWD2pMSNNYJAU6Qq+65NkxXjqHr/7LM2Xkqg==} + '@hey-api/codegen-core@0.8.2': resolution: {integrity: sha512-R2NMf3wq97rh1mjz33WJQU8svz3F0RYUjvx/QzXucjpSqQ3O5huTdDjErG4fMxSr1X+X56NuDrqtGfHmo1TRUQ==} engines: {node: '>=22.13.0'} @@ -5218,6 +5247,10 @@ packages: resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} engines: {node: '>= 16'} + check-more-types@2.24.0: + resolution: {integrity: sha512-Pj779qHxV2tuapviy1bSZNEL1maXr13bPYpsvSDB68HlYcYuhlDrmGd63i0JHMCLKzc7rUSNIrpdJlhVlNwrxA==} + engines: {node: '>= 0.8.0'} + chokidar@3.6.0: resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==} engines: {node: '>= 8.10.0'} @@ -5893,6 +5926,10 @@ packages: eventemitter3@5.0.4: resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==} + execa@5.1.1: + resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} + engines: {node: '>=10'} + expect-type@1.4.0: resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} engines: {node: '>=12.0.0'} @@ -6102,6 +6139,10 @@ packages: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} + get-stream@6.0.1: + resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} + engines: {node: '>=10'} + get-tsconfig@4.14.0: resolution: {integrity: sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==} @@ -6295,6 +6336,10 @@ packages: resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} engines: {node: '>= 6'} + human-signals@2.1.0: + resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} + engines: {node: '>=10.17.0'} + husky@9.1.7: resolution: {integrity: sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==} engines: {node: '>=18'} @@ -6487,6 +6532,10 @@ packages: resolution: {integrity: sha512-ISWac8drv4ZGfwKl5slpHG9OwPNty4jOWPRIhBpxOoD+hqITiwuipOQ2bNthAzwA3B4fIjO4Nln74N0S9byq8A==} engines: {node: '>= 0.4'} + is-stream@2.0.1: + resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} + engines: {node: '>=8'} + is-string@1.1.1: resolution: {integrity: sha512-BtEeSsoaQjlSPBemMQIrY1MY0uM6vnS1g5fmufYOtnxLGUZM2178PKbhsk7Ffv58IX+ZtcvoGwccYsh0PglkAA==} engines: {node: '>= 0.4'} @@ -6554,6 +6603,10 @@ packages: jju@1.4.0: resolution: {integrity: sha512-8wb9Yw966OSxApiCt0K3yNJL8pnNeIv+OEq2YMidz4FKP6nonSRoOXc80iXY4JaN2FC11B9qsNmDsm+ZOfMROA==} + joi@18.2.3: + resolution: {integrity: sha512-N5A3KTWQpPWT4ExxxPlUx7WmykGXRzhNidWhV41d6Abu9YfI2NyWCJuxdPnslJCPWtbRpSVOWSnSS6GakLM/Rg==} + engines: {node: '>= 20'} + js-beautify@1.15.4: resolution: {integrity: sha512-9/KXeZUKKJwqCXUdBxFJ3vPh467OCckSBmYDwSK/EtV090K+iMJ7zx2S3HLVDIWFQdqMIsZWbnaGiba18aWhaA==} engines: {node: '>=14'} @@ -6663,6 +6716,10 @@ packages: kolorist@1.8.0: resolution: {integrity: sha512-Y+60/zizpJ3HRH8DCss+q95yr6145JXZo46OTpFvDZWLfRCE4qChOyk1b26nMaNpfHHgxagk9dXT5OP0Tfe+dQ==} + lazy-ass@2.0.3: + resolution: {integrity: sha512-/O3/DoQmI1XAhklDvF1dAjFf/epE8u3lzOZegQfLZ8G7Ud5bTRSZiFOpukHCu6jODrCA4gtIdwUCC7htxcDACA==} + engines: {node: '> 0.8'} + lenis@1.3.25: resolution: {integrity: sha512-mOKxayErlaONK8fm4LN3XNd99Qu4plTpn9h9qf8wxzjGrJDzuD84FYzZ81HCd6ZsWp++VWVwOzL286Pf2s2u4A==} peerDependencies: @@ -6869,6 +6926,9 @@ packages: lodash.truncate@4.4.2: resolution: {integrity: sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==} + lodash@4.18.1: + resolution: {integrity: sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==} + log-update@6.1.0: resolution: {integrity: sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==} engines: {node: '>=18'} @@ -7036,6 +7096,9 @@ packages: resolution: {integrity: sha512-EDYo6VlmtnumlcBCbh1gLJ//9jvM/ndXHfVXIFrZVr6fGcwTUyCTFNTLCKuY3ffbK8L/+3Mzqnd58RojiZqHVw==} engines: {node: '>=20'} + merge-stream@2.0.0: + resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} + merge2@1.4.1: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} @@ -7160,6 +7223,10 @@ packages: resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} engines: {node: '>= 0.6'} + mimic-fn@2.1.0: + resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} + engines: {node: '>=6'} + mimic-function@5.0.1: resolution: {integrity: sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==} engines: {node: '>=18'} @@ -7287,6 +7354,10 @@ packages: nostics@1.1.4: resolution: {integrity: sha512-U4FApICSLCQ0dYiN59pxUCEZC053palQXi57yLaNdMaL6TBY4C4q3qZrJKUGcor2dGv8EhEwt8y5KvU2bo2kFg==} + npm-run-path@4.0.1: + resolution: {integrity: sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==} + engines: {node: '>=8'} + nth-check@2.1.1: resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} @@ -7323,6 +7394,10 @@ packages: ohash@2.0.11: resolution: {integrity: sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ==} + onetime@5.1.2: + resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} + engines: {node: '>=6'} + onetime@7.0.0: resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==} engines: {node: '>=18'} @@ -7973,6 +8048,9 @@ packages: rxjs-interop@2.0.0: resolution: {integrity: sha512-ASEq9atUw7lualXB+knvgtvwkCEvGWV2gDD/8qnASzBkzEARZck9JAyxmY8OS6Nc1pCPEgDTKNcx+YqqYfzArw==} + rxjs@7.8.2: + resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} + safe-buffer@5.1.2: resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} @@ -8065,6 +8143,9 @@ packages: siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + signal-exit@3.0.7: + resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + signal-exit@4.1.0: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} @@ -8142,6 +8223,11 @@ packages: standardized-audio-context@25.3.77: resolution: {integrity: sha512-Ki9zNz6pKcC5Pi+QPjPyVsD9GwJIJWgryji0XL9cAJXMGyn+dPOf6Qik1AHei0+UNVcc4BOCa0hWLBzlwqsW/A==} + start-server-and-test@3.0.12: + resolution: {integrity: sha512-JhW9LiYXqJxyzmgMvBQ6JxVcz16wV+c5IE6rTHofqLBKjSE075Yr9e6sTr8QgrkHMHlHqkDegzKpQfVKxwauJw==} + engines: {node: ^22 || >=24} + hasBin: true + std-env@4.2.0: resolution: {integrity: sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==} @@ -8205,6 +8291,10 @@ packages: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} + strip-final-newline@2.0.0: + resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} + engines: {node: '>=6'} + strip-indent@3.0.0: resolution: {integrity: sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==} engines: {node: '>=8'} @@ -8392,6 +8482,10 @@ packages: resolution: {integrity: sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==} engines: {node: '>=20'} + tree-kill@1.2.2: + resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} + hasBin: true + trim-lines@3.0.1: resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} @@ -9129,6 +9223,11 @@ packages: resolution: {integrity: sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==} engines: {node: '>=18'} + wait-on@9.1.0: + resolution: {integrity: sha512-PymrLXHLBM1Ju/Xspb2ADUhbPSMvbnuNvy/mN2hWtpbJ3da0h3Ky1LqwKPG5QSVR57liyO0iUpfipYl/s5qNvA==} + engines: {node: '>=20.0.0'} + hasBin: true + walk-up-path@4.0.0: resolution: {integrity: sha512-3hu+tD8YzSLGuFYtPRb48vdhKMi0KQV5sn+uWr8+7dMEq/2G/dtLrdDinkLjqq5TIbIBjYJ4Ax/n3YiaW7QM8A==} engines: {node: 20 || >=22} @@ -10639,6 +10738,22 @@ snapshots: protobufjs: 7.6.5 yargs: 17.7.3 + '@hapi/address@5.1.1': + dependencies: + '@hapi/hoek': 11.0.7 + + '@hapi/formula@3.0.2': {} + + '@hapi/hoek@11.0.7': {} + + '@hapi/pinpoint@2.0.1': {} + + '@hapi/tlds@1.1.7': {} + + '@hapi/topo@6.0.2': + dependencies: + '@hapi/hoek': 11.0.7 + '@hey-api/codegen-core@0.8.2(magicast@0.5.3)': dependencies: '@hey-api/types': 0.1.4 @@ -13052,13 +13167,13 @@ snapshots: '@vueuse/shared': 14.4.0(vue@3.5.41(typescript@6.0.3)) vue: 3.5.41(typescript@6.0.3) - '@vueuse/integrations@14.4.0(axios@1.18.1)(fuse.js@7.0.0)(vue@3.5.41(typescript@6.0.3))': + '@vueuse/integrations@14.4.0(axios@1.18.1(debug@4.4.3))(fuse.js@7.0.0)(vue@3.5.41(typescript@6.0.3))': dependencies: '@vueuse/core': 14.4.0(vue@3.5.41(typescript@6.0.3)) '@vueuse/shared': 14.4.0(vue@3.5.41(typescript@6.0.3)) vue: 3.5.41(typescript@6.0.3) optionalDependencies: - axios: 1.18.1 + axios: 1.18.1(debug@4.4.3) fuse.js: 7.0.0 '@vueuse/metadata@12.8.2': {} @@ -13366,9 +13481,9 @@ snapshots: dependencies: possible-typed-array-names: 1.1.0 - axios@1.18.1: + axios@1.18.1(debug@4.4.3): dependencies: - follow-redirects: 1.16.0 + follow-redirects: 1.16.0(debug@4.4.3) form-data: 4.0.6 https-proxy-agent: 5.0.1 proxy-from-env: 2.1.0 @@ -13537,6 +13652,8 @@ snapshots: check-error@2.1.3: {} + check-more-types@2.24.0: {} + chokidar@3.6.0: dependencies: anymatch: 3.1.3 @@ -14257,6 +14374,18 @@ snapshots: eventemitter3@5.0.4: {} + execa@5.1.1: + dependencies: + cross-spawn: 7.0.6 + get-stream: 6.0.1 + human-signals: 2.1.0 + is-stream: 2.0.1 + merge-stream: 2.0.0 + npm-run-path: 4.0.1 + onetime: 5.1.2 + signal-exit: 3.0.7 + strip-final-newline: 2.0.0 + expect-type@1.4.0: {} exsolve@1.1.0: {} @@ -14432,7 +14561,9 @@ snapshots: flattie@1.1.1: {} - follow-redirects@1.16.0: {} + follow-redirects@1.16.0(debug@4.4.3): + optionalDependencies: + debug: 4.4.3 fontace@0.4.1: dependencies: @@ -14515,6 +14646,8 @@ snapshots: dunder-proto: 1.0.1 es-object-atoms: 1.1.2 + get-stream@6.0.1: {} + get-tsconfig@4.14.0: dependencies: resolve-pkg-maps: 1.0.0 @@ -14823,6 +14956,8 @@ snapshots: transitivePeerDependencies: - supports-color + human-signals@2.1.0: {} + husky@9.1.7: {} idb@7.1.1: {} @@ -14978,6 +15113,8 @@ snapshots: dependencies: call-bound: 1.0.4 + is-stream@2.0.1: {} + is-string@1.1.1: dependencies: call-bound: 1.0.4 @@ -15044,6 +15181,16 @@ snapshots: jju@1.4.0: optional: true + joi@18.2.3: + dependencies: + '@hapi/address': 5.1.1 + '@hapi/formula': 3.0.2 + '@hapi/hoek': 11.0.7 + '@hapi/pinpoint': 2.0.1 + '@hapi/tlds': 1.1.7 + '@hapi/topo': 6.0.2 + '@standard-schema/spec': 1.1.0 + js-beautify@1.15.4: dependencies: config-chain: 1.1.13 @@ -15170,6 +15317,8 @@ snapshots: kolorist@1.8.0: {} + lazy-ass@2.0.3: {} + lenis@1.3.25(react@19.2.8)(vue@3.5.41(typescript@6.0.3)): optionalDependencies: react: 19.2.8 @@ -15326,6 +15475,8 @@ snapshots: lodash.truncate@4.4.2: {} + lodash@4.18.1: {} + log-update@6.1.0: dependencies: ansi-escapes: 7.3.0 @@ -15613,6 +15764,8 @@ snapshots: meow@14.1.0: {} + merge-stream@2.0.0: {} + merge2@1.4.1: {} meshoptimizer@1.1.1: {} @@ -15892,6 +16045,8 @@ snapshots: dependencies: mime-db: 1.52.0 + mimic-fn@2.1.0: {} + mimic-function@5.0.1: {} min-indent@1.0.1: {} @@ -16009,6 +16164,10 @@ snapshots: nostics@1.1.4: {} + npm-run-path@4.0.1: + dependencies: + path-key: 3.1.1 + nth-check@2.1.1: dependencies: boolbase: 1.0.0 @@ -16045,6 +16204,10 @@ snapshots: ohash@2.0.11: {} + onetime@5.1.2: + dependencies: + mimic-fn: 2.1.0 + onetime@7.0.0: dependencies: mimic-function: 5.0.1 @@ -16968,6 +17131,10 @@ snapshots: rxjs-interop@2.0.0: {} + rxjs@7.8.2: + dependencies: + tslib: 2.8.1 + safe-buffer@5.1.2: {} safe-buffer@5.2.1: {} @@ -17110,6 +17277,8 @@ snapshots: siginfo@2.0.0: {} + signal-exit@3.0.7: {} + signal-exit@4.1.0: {} sirv@3.0.2: @@ -17179,6 +17348,18 @@ snapshots: automation-events: 7.1.19 tslib: 2.8.1 + start-server-and-test@3.0.12: + dependencies: + arg: 5.0.2 + check-more-types: 2.24.0 + debug: 4.4.3 + execa: 5.1.1 + lazy-ass: 2.0.3 + tree-kill: 1.2.2 + wait-on: 9.1.0(debug@4.4.3) + transitivePeerDependencies: + - supports-color + std-env@4.2.0: {} stop-iteration-iterator@1.1.0: @@ -17259,6 +17440,8 @@ snapshots: strip-bom@3.0.0: {} + strip-final-newline@2.0.0: {} + strip-indent@3.0.0: dependencies: min-indent: 1.0.1 @@ -17464,6 +17647,8 @@ snapshots: dependencies: punycode: 2.3.1 + tree-kill@1.2.2: {} + trim-lines@3.0.1: {} trough@2.2.0: {} @@ -18205,6 +18390,17 @@ snapshots: dependencies: xml-name-validator: 5.0.0 + wait-on@9.1.0(debug@4.4.3): + dependencies: + axios: 1.18.1(debug@4.4.3) + joi: 18.2.3 + lodash: 4.18.1 + minimist: 1.2.8 + rxjs: 7.8.2 + transitivePeerDependencies: + - debug + - supports-color + walk-up-path@4.0.0: {} web-namespaces@2.0.1: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index e92c26d80f9..202519366ee 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -119,6 +119,7 @@ catalog: primevue: 4.2.5 reka-ui: 2.6.2 rollup-plugin-visualizer: ^6.0.4 + start-server-and-test: ^3.0.11 storybook: ^10.5.6 stylelint: ^17.14.0 tailwindcss: ^4.3.2 diff --git a/scripts/cicd/custom-nodes-summary.py b/scripts/cicd/custom-nodes-summary.py new file mode 100644 index 00000000000..03abb90fdae --- /dev/null +++ b/scripts/cicd/custom-nodes-summary.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Render the custom-node core gate's results table. + +Reads custom-nodes-results.json from the working directory and renders the +run's context (branch tested, head SHA, ComfyUI ref, filters) plus a +per-pack x tier verdict table - as an aligned table on stdout for the job +log, and as markdown appended to GITHUB_STEP_SUMMARY. Tolerates a missing +results file (the suite died before Playwright reported) by emitting the +context alone. Inputs arrive via env: BRANCH_TESTED, COMFYUI_REF_USED, +GREP_FILTER, plus the standard GITHUB_* vars. +""" + +import json, os, re, subprocess + +RESULT_FILE = 'custom-nodes-results.json' +LOG_FILE = 'custom-nodes.log' +MANIFEST = os.environ.get('CUSTOM_NODES_MANIFEST', 'core') +SUITE_TITLE = f"Custom-node {'Core depth' if MANIFEST == 'core' else 'Cloud breadth'} suite" +CURATED_TIER = 'curated run + S15' if MANIFEST == 'core' else 'curated run' + +def out(md, log=None): + with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as f: + f.write(md + '\n') + print(log if log is not None else md) + +head = subprocess.run(['git', 'rev-parse', 'HEAD'], + capture_output=True, text=True).stdout.strip() +ctx = [ + ('Branch tested', os.environ.get('BRANCH_TESTED', '?')), + ('Head SHA', head[:12]), + ('ComfyUI ref', os.environ.get('COMFYUI_REF_USED', '?')[:12]), + ('Event / actor', f"{os.environ.get('GITHUB_EVENT_NAME','?')} / {os.environ.get('GITHUB_ACTOR','?')}"), + ('Grep filter', os.environ.get('GREP_FILTER') or '(full suite)'), +] +if not os.path.exists(RESULT_FILE): + out(f'## {SUITE_TITLE}\n\n**No results json was written** - the suite step died before Playwright could report.') + for k, v in ctx: + out(f'- **{k}**: {v}', f'{k}: {v}') + raise SystemExit(0) + +data = json.load(open(RESULT_FILE)) +stats = data.get('stats', {}) + +TIERS = ['startup/load', 'S1', 'S2', 'S3', 'S9', CURATED_TIER, 'S12', 'S13'] +packs, wide = {}, {} +auto_run_exclusions = [] + +def bucket(file, path): + title = ' > '.join(path) + pack = None + m = re.search(r'(?:custom node|all nodes|dynamic inputs): ([^@>]+?)(?:\s*@|\s*>|$)', title) + if m: + pack = m.group(1).strip() + if 'allNodes' in file: + tier = re.search(r'all nodes by tier @custom-nodes > (S(?:1|2|3|9)):', title) + if tier: + return (None, tier.group(1)) + return (pack, 'all nodes') if pack else (None, 'manifest coverage') + if 'customNode.regression' in file: + if pack is None: + return (None, 'harness self-checks') + tier = 'startup/load' if 'startup/load' in title else CURATED_TIER + return (pack, tier) + if 'dynamicInputs' in file: + return (pack, 'S12') + if 'interactionProfiles' in file: + return (pack, 'S13') + if 'connectivity' in file: + return (None, 'connectivity') + if 'coreSmoke' in file: + return (None, 'core smoke') + return (None, 'pure specs') + +RANK = {'skipped': 1, 'flaky': 2, 'unexpected': 3} +SYM = {0: 'PASS', 1: 'SKIP', 2: 'FLAKY', 3: 'FAIL'} +failed = [] + +def visit(suite, file, path): + file = suite.get('file') or file + path = path + ([suite['title']] if suite.get('title') else []) + for spec in suite.get('specs', []): + worst = max((RANK.get(t.get('status'), 0) for t in spec.get('tests', [])), default=0) + pack, tier = bucket(file or '', path + [spec.get('title', '')]) + cell = packs.setdefault(pack, {}) if pack else wide + key = tier + prev = cell.get(key, (0, 0, 0)) + cell[key] = (max(prev[0], worst), prev[1] + 1, prev[2] + (1 if worst == 3 else 0)) + if worst == 3: + err = '' + for t in spec.get('tests', []): + for r in t.get('results', []): + msg = (r.get('error') or {}).get('message') or '' + if msg: + err = re.sub(r'\x1b\[[0-9;]*m', '', msg).split('\n')[0][:140] + break + if err: + break + failed.append((f"{' > '.join(p for p in path[1:] if p)} > {spec.get('title','')}"[:120], err)) + for child in suite.get('suites', []): + visit(child, file, path) + +for s in data.get('suites', []): + visit(s, s.get('file'), []) + +if os.path.exists(LOG_FILE): + for line in open(LOG_FILE): + excluded = re.fullmatch(r'(.+): (.+) excluded from auto-run \((.+)\)\n?', line) + if excluded: + auto_run_exclusions.append(excluded.groups()) + continue + m = re.fullmatch(r'\[tier-pack\] tier=(S(?:1|2|3|9)) pack=(.+) result=(pass|fail)\n?', line) + if not m: + continue + tier, pack, result = m.groups() + prev = packs.setdefault(pack, {}).get(tier, (0, 0, 0)) + failed_pack = result == 'fail' + packs[pack][tier] = ( + max(prev[0], 3 if failed_pack else 0), + prev[1] + 1, + prev[2] + failed_pack, + ) + +def cell(v): + if v is None: + return 'not enrolled' + worst, total, fails = v + return SYM[worst] if fails == 0 else f'FAIL {fails}/{total}' + +out(f'## {SUITE_TITLE}') +out('') +out('| | |\n|---|---|') +for k, v in ctx: + out(f'| **{k}** | {v} |', f'{k:16} {v}') +mins = stats.get('duration', 0) / 60000 +totals = f"{stats.get('expected',0)} passed / {stats.get('unexpected',0)} failed / {stats.get('flaky',0)} flaky / {stats.get('skipped',0)} skipped in {mins:.1f}m" +out(f'| **Result** | {totals} |', f'{"Result":16} {totals}') +out('') +out('| Pack | ' + ' | '.join(TIERS) + ' |', + f'{"Pack":34}' + ''.join(f'{t:>16}' for t in TIERS)) +out('|---|' + '---|' * len(TIERS), '-' * 100) +for pack in sorted(packs): + row = [cell(packs[pack].get(t)) for t in TIERS] + out(f'| {pack} | ' + ' | '.join(row) + ' |', f'{pack:34}' + ''.join(f'{c:>16}' for c in row)) +out('') +out('`PASS` the tier ran for this pack and passed. `FAIL n/m` the tier ran and n of m checks failed. `not enrolled` this pack\'s manifest row does not declare this tier, so it never ran for this pack - that is population scope, not a skipped test.') +out('') +for name in sorted(wide): + out(f'- **{name}**: {cell(wide[name])}', f'{name:34}{cell(wide[name])}') +if failed: + out('\n### Failed tests') + for title, err in failed[:20]: + out(f'- `{title}`\n - {err}', f'FAILED: {title}\n {err}') +if auto_run_exclusions: + out('\n### S9 node execution skips') + out('These nodes were registered, but S9 did not queue them. Other applicable tiers still ran for their pack; this is node-level execution coverage debt, not a whole-pack skip.') + for pack, node, reason in sorted(set(auto_run_exclusions)): + out(f'- **S9 - SKIP - NODE NOT EXECUTED - {pack} / {node}**\n - reason: {reason}\n - to remove: make this node deterministic and safe on the bare CPU backend, then remove its S9 exclusion') + print(f'::warning title=S9 node execution skipped::{pack} / {node} - {reason}') diff --git a/scripts/cicd/custom-nodes-summary.test.ts b/scripts/cicd/custom-nodes-summary.test.ts new file mode 100644 index 00000000000..7cd7d8d7a35 --- /dev/null +++ b/scripts/cicd/custom-nodes-summary.test.ts @@ -0,0 +1,101 @@ +import { spawnSync } from 'node:child_process' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { expect, it } from 'vitest' + +const SCRIPT = path.join(import.meta.dirname, 'custom-nodes-summary.py') + +it('reports each all-node tier result separately for each pack', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'custom-nodes-summary-')) + try { + fs.writeFileSync( + path.join(dir, 'custom-nodes-results.json'), + JSON.stringify({ stats: { expected: 1, unexpected: 1 } }) + ) + fs.writeFileSync( + path.join(dir, 'custom-nodes.log'), + [ + '[tier-pack] tier=S1 pack=Pack-A result=pass', + '[tier-pack] tier=S2 pack=Pack-A result=fail', + '[tier-pack] tier=S1 pack=Pack-B result=pass', + '[tier-pack] tier=S2 pack=Pack-B result=pass', + 'Pack-B: ExampleNode excluded from auto-run (requires a model)' + ].join('\n') + ) + + const result = spawnSync('python3', [SCRIPT], { + cwd: dir, + encoding: 'utf8', + env: { + ...process.env, + CUSTOM_NODES_MANIFEST: 'cloud', + GITHUB_STEP_SUMMARY: path.join(dir, 'summary.md') + } + }) + + expect(result.status).toBe(0) + const summary = fs.readFileSync(path.join(dir, 'summary.md'), 'utf8') + expect(summary).toContain('## Custom-node Cloud breadth suite') + expect(summary).toContain('| Pack | startup/load | S1 | S2 | S3 | S9 |') + expect(summary).toContain('### S9 node execution skips') + expect(summary).toContain( + '**S9 - SKIP - NODE NOT EXECUTED - Pack-B / ExampleNode**' + ) + expect(summary).toContain( + 'this is node-level execution coverage debt, not a whole-pack skip' + ) + expect(summary).toContain( + "`not enrolled` this pack's manifest row does not declare this tier" + ) + expect(result.stdout).toMatch( + /^Pack-A\s+not enrolled\s+PASS\s+FAIL 1\/1\s+not enrolled/m + ) + expect(result.stdout).toMatch( + /^Pack-B\s+not enrolled\s+PASS\s+PASS\s+not enrolled/m + ) + } finally { + fs.rmSync(dir, { recursive: true, force: true }) + } +}) + +it('labels an isolated all-node failure with its S-tier', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'custom-nodes-summary-')) + try { + fs.writeFileSync( + path.join(dir, 'custom-nodes-results.json'), + JSON.stringify({ + stats: { expected: 0, unexpected: 1 }, + suites: [ + { + file: 'browser_tests/tests/customNodes/allNodes.spec.ts', + title: 'all nodes by tier @custom-nodes', + specs: [ + { + title: 'S3: every registered node survives save and reload', + tests: [{ status: 'unexpected', results: [] }] + } + ] + } + ] + }) + ) + fs.writeFileSync(path.join(dir, 'custom-nodes.log'), '') + + const result = spawnSync('python3', [SCRIPT], { + cwd: dir, + encoding: 'utf8', + env: { + ...process.env, + GITHUB_STEP_SUMMARY: path.join(dir, 'summary.md') + } + }) + + expect(result.status).toBe(0) + const summary = fs.readFileSync(path.join(dir, 'summary.md'), 'utf8') + expect(summary).toContain('- **S3**: FAIL 1/1') + expect(summary).not.toContain('- **manifest coverage**: FAIL 1/1') + } finally { + fs.rmSync(dir, { recursive: true, force: true }) + } +}) diff --git a/scripts/cloud-manifest.test.ts b/scripts/cloud-manifest.test.ts new file mode 100644 index 00000000000..d4dfb42022d --- /dev/null +++ b/scripts/cloud-manifest.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from 'vitest' + +import { + buildCloudManifest, + sourceFromSupportedNodesHeader, + validateCuratedCloudOverlay, + validateSupportedNodesDoc +} from './cloud-manifest' + +const sourceHeader = `# Source: https://github.com/Comfy-Org/cloud/blob/${'a'.repeat(40)}/comfy-complete/supported_nodes.yaml +# Imported: 2026-08-14 +` + +describe('cloud manifest provenance', () => { + it('binds the generated manifest to the vendored yaml revision', () => { + const source = sourceFromSupportedNodesHeader(sourceHeader) + const manifest = buildCloudManifest( + { + labels: [], + node_packs: [ + { name: 'example-pack', version: '1.2.3', web_directory: 'web' } + ] + }, + { + ExampleNode: { python_module: 'custom_nodes.example-pack' } + }, + source + ) + + expect(manifest.source).toEqual(source) + expect(manifest.packs).toHaveLength(1) + expect(manifest.packs[0].deployRef).toBe('example-pack@1.2.3') + expect(manifest.packs[0].webDirectory).toBe('web') + }) + + it('rejects a snapshot without an immutable source header', () => { + expect(() => + sourceFromSupportedNodesHeader('# Imported: 2026-08-14') + ).toThrow(/Source/) + expect(() => + sourceFromSupportedNodesHeader( + `# Source: https://github.com/Comfy-Org/cloud/blob/${'a'.repeat(40)}/supported_nodes.yaml\n# Imported: 2026-02-31` + ) + ).toThrow(/real ISO date/) + }) + + it('rejects a web directory that can escape the pack root', () => { + expect(() => + validateSupportedNodesDoc({ + labels: [], + node_packs: [ + { + name: 'example-pack', + version: '1.2.3', + web_directory: '../web' + } + ] + }) + ).toThrow(/safe relative path/) + }) +}) + +describe('cloud run overlay', () => { + it('requires a calibrated runnable corpus for every run-tier row', () => { + expect(() => + validateCuratedCloudOverlay({ + pack: { + workflow: 'assets/customNodes/example.json', + tiers: ['load', 'run'] + } + }) + ).toThrow(/expectedRunnableCount/) + }) + + it.each([ + '/tmp/workflow.json', + 'C:\\tmp\\workflow.json', + '../workflow.json', + 'assets/../../workflow.json', + 'assets\\..\\workflow.json' + ])('rejects workflow paths outside browser_tests: %s', (workflow) => { + expect(() => + validateCuratedCloudOverlay({ + pack: { + workflow, + tiers: ['load', 'run'], + expectedRunnableCount: 1 + } + }) + ).toThrow(/inside browser_tests/) + }) +}) diff --git a/scripts/cloud-manifest.ts b/scripts/cloud-manifest.ts new file mode 100644 index 00000000000..4b264611532 --- /dev/null +++ b/scripts/cloud-manifest.ts @@ -0,0 +1,532 @@ +import { readFileSync, writeFileSync } from 'node:fs' +import { posix, win32 } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' + +import { parse } from 'yaml' + +import type { + CloudManifest, + CloudManifestEntry, + CloudManifestSource +} from '../browser_tests/fixtures/customNode/manifest' +import { + assertCloudEntry, + assertCloudManifestShape +} from '../browser_tests/fixtures/customNode/manifest' +import type { RawNodeDef } from '../browser_tests/fixtures/customNode/typePairing' +import { packOf } from '../browser_tests/fixtures/customNode/typePairing' + +export interface SupportedNodesPack { + name: string + version?: string + node_labels?: Record + web_directory?: string +} + +export interface SupportedNodesDoc { + labels: string[] + node_packs: SupportedNodesPack[] +} + +export type ObjectInfoSnapshot = Record + +const URL_PIN = /@[0-9a-f]{40}$/ +const SOURCE_LINE = + /^# Source: (https:\/\/github\.com\/[^/]+\/[^/]+)\/blob\/([0-9a-f]{40})\/(\S+)$/m +const IMPORTED_LINE = /^# Imported: (\d{4}-\d{2}-\d{2})$/m + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function isStringArray(value: unknown): value is string[] { + return ( + Array.isArray(value) && + value.every((item) => typeof item === 'string' && item.length > 0) + ) +} + +export function validateSupportedNodesDoc(value: unknown): SupportedNodesDoc { + if (!isRecord(value) || !isStringArray(value.labels)) + throw new Error( + 'supported_nodes.yaml: expected a top-level { labels, node_packs } document' + ) + const declared = new Set(value.labels) + if (!Array.isArray(value.node_packs)) + throw new Error('supported_nodes.yaml: node_packs must be a list') + const packs = value.node_packs.map((pack, index): SupportedNodesPack => { + if (!isRecord(pack) || typeof pack.name !== 'string' || pack.name === '') + throw new Error(`supported_nodes.yaml: pack ${index} has no name`) + const { name } = pack + if (name.startsWith('http') && !URL_PIN.test(name)) + throw new Error( + `supported_nodes.yaml: ${name} is URL-pinned but carries no @<40-hex-sha> suffix` + ) + if (pack.version !== undefined && typeof pack.version !== 'string') + throw new Error(`supported_nodes.yaml: ${name} version must be a string`) + if ( + pack.web_directory !== undefined && + (typeof pack.web_directory !== 'string' || + !/^(?!.*(?:^|\/)\.\.(?:\/|$))[^/][A-Za-z0-9._/-]*$/.test( + pack.web_directory + )) + ) + throw new Error( + `supported_nodes.yaml: ${name} web_directory must be a safe relative path` + ) + if ( + name !== 'core' && + !name.startsWith('http') && + (pack.version === undefined || pack.version === '') + ) + throw new Error( + `supported_nodes.yaml: registry pack ${name} has no version - deployRef needs one` + ) + let nodeLabels: Record | undefined + if (pack.node_labels !== undefined) { + if (!isRecord(pack.node_labels)) + throw new Error( + `supported_nodes.yaml: ${name} node_labels must map node -> labels` + ) + nodeLabels = {} + for (const [node, labels] of Object.entries(pack.node_labels)) { + if (!isStringArray(labels) || labels.length === 0) + throw new Error( + `supported_nodes.yaml: ${name} node ${node} must carry a non-empty label list` + ) + if (new Set(labels).size !== labels.length) + throw new Error( + `supported_nodes.yaml: ${name} node ${node} carries a duplicate label` + ) + for (const label of labels) + if (!declared.has(label)) + throw new Error( + `supported_nodes.yaml: ${name} node ${node} uses undeclared label ${label}` + ) + nodeLabels[node] = labels + } + } + return { + name, + version: typeof pack.version === 'string' ? pack.version : undefined, + node_labels: nodeLabels, + web_directory: + typeof pack.web_directory === 'string' ? pack.web_directory : undefined + } + }) + if (packs.filter((pack) => pack.name === 'core').length > 1) + throw new Error('supported_nodes.yaml: more than one core entry') + return { labels: value.labels, node_packs: packs } +} + +export function sourceFromSupportedNodesHeader( + contents: string +): CloudManifestSource { + const source = SOURCE_LINE.exec(contents) + const imported = IMPORTED_LINE.exec(contents) + if (!source || !imported) + throw new Error( + 'supported_nodes.yaml: expected pinned # Source and # Imported headers' + ) + const importedAt = imported[1] + const timestamp = Date.parse(`${importedAt}T00:00:00Z`) + if ( + Number.isNaN(timestamp) || + new Date(timestamp).toISOString().slice(0, 10) !== importedAt + ) + throw new Error('supported_nodes.yaml: # Imported must be a real ISO date') + return { + repository: source[1], + ref: source[2], + path: source[3], + importedAt + } +} + +export function validateObjectInfoSnapshot(value: unknown): ObjectInfoSnapshot { + if (!isRecord(value)) + throw new Error( + 'object_info snapshot: expected the raw /object_info shape (nodes keyed by class name)' + ) + for (const [node, def] of Object.entries(value)) { + if (!isRecord(def)) + throw new Error(`object_info snapshot: node ${node} is not an object`) + if ( + def.python_module !== undefined && + typeof def.python_module !== 'string' + ) + throw new Error( + `object_info snapshot: node ${node} python_module is not a string` + ) + } + return value as ObjectInfoSnapshot +} + +interface CuratedCloudWorkflow { + workflow: string + tiers: CloudManifestEntry['tiers'] + expectedNodes?: string[] + expectedRunnableCount?: number + timeoutMs?: number +} + +export type CuratedCloudOverlay = Record + +const OVERLAY_KEYS = [ + 'workflow', + 'tiers', + 'expectedNodes', + 'expectedRunnableCount', + 'timeoutMs' +] + +export function validateCuratedCloudOverlay( + value: unknown +): CuratedCloudOverlay { + if (!isRecord(value)) + throw new Error( + 'curated overlay: expected { "": { workflow, tiers[, timeoutMs] } }' + ) + const overlay: CuratedCloudOverlay = {} + for (const [pack, entry] of Object.entries(value)) { + if (!isRecord(entry)) + throw new Error(`curated overlay: ${pack} must be an object`) + const unknown = Object.keys(entry).filter( + (key) => !OVERLAY_KEYS.includes(key) + ) + if (unknown.length > 0) + throw new Error( + `curated overlay: ${pack} carries unknown key(s) ${unknown.join(', ')} (allowed: ${OVERLAY_KEYS.join(', ')})` + ) + if (typeof entry.workflow !== 'string' || entry.workflow === '') + throw new Error( + `curated overlay: ${pack} workflow must be a non-empty path relative to browser_tests/` + ) + if ( + posix.isAbsolute(entry.workflow) || + win32.isAbsolute(entry.workflow) || + entry.workflow.split(/[\\/]/).includes('..') + ) + throw new Error( + `curated overlay: ${pack} workflow must stay inside browser_tests/` + ) + if (!isStringArray(entry.tiers) || entry.tiers.length === 0) + throw new Error( + `curated overlay: ${pack} tiers must be the row's full non-empty tier list` + ) + if (!entry.tiers.includes('run')) + throw new Error(`curated overlay: ${pack} tiers must include 'run'`) + if ( + entry.expectedNodes !== undefined && + (!isStringArray(entry.expectedNodes) || + entry.expectedNodes.length === 0 || + new Set(entry.expectedNodes).size !== entry.expectedNodes.length) + ) + throw new Error( + `curated overlay: ${pack} expectedNodes must be a non-empty array of unique non-empty node keys` + ) + if ( + !Number.isInteger(entry.expectedRunnableCount) || + (entry.expectedRunnableCount as number) <= 0 + ) + throw new Error( + `curated overlay: ${pack} expectedRunnableCount must be a positive integer` + ) + if ( + entry.timeoutMs !== undefined && + (typeof entry.timeoutMs !== 'number' || + !Number.isFinite(entry.timeoutMs) || + entry.timeoutMs <= 0) + ) + throw new Error( + `curated overlay: ${pack} timeoutMs must be a positive number` + ) + overlay[pack] = { + workflow: entry.workflow, + tiers: entry.tiers as CloudManifestEntry['tiers'], + ...(entry.expectedNodes !== undefined + ? { expectedNodes: entry.expectedNodes } + : {}), + expectedRunnableCount: entry.expectedRunnableCount as number, + ...(entry.timeoutMs !== undefined ? { timeoutMs: entry.timeoutMs } : {}) + } + } + return overlay +} + +// yaml pack ids and snapshot dirnames agree only up to case and separators. +function joinKeyOf(name: string): string { + return name.toLowerCase().replace(/[^a-z0-9]/g, '') +} + +function yamlJoinKeyOf(pack: SupportedNodesPack): string { + if (!pack.name.startsWith('http')) return joinKeyOf(pack.name) + const repo = pack.name + .replace(URL_PIN, '') + .replace(/\/+$/, '') + .replace(/\.git$/, '') + return joinKeyOf(repo.slice(repo.lastIndexOf('/') + 1)) +} + +function deployRefOf(pack: SupportedNodesPack): string { + if (pack.name.startsWith('http')) return pack.name + return `${pack.name}@${pack.version}` +} + +function sortedRecordOf(labelsByNode: Record) { + return Object.fromEntries( + Object.keys(labelsByNode) + .sort() + .map((node) => [node, [...labelsByNode[node]].sort()]) + ) +} + +function snapshotPacksOf(snapshot: ObjectInfoSnapshot): Map { + const nodesByPack = new Map() + for (const [node, def] of Object.entries(snapshot)) { + const pack = packOf(def.python_module) + nodesByPack.set(pack, [...(nodesByPack.get(pack) ?? []), node]) + } + return nodesByPack +} + +// One boot-registered extension name per pack, read off the JS Cloud actually +// serves at /extensions/. One sentinel, not the full set: the assert only has +// to prove the pack's JS loaded, and a conditionally-registered extra would +// red a healthy run. +export type CloudExtensionSentinels = Record + +// Per-pack auto-run calibration (nodes that cannot execute on pure defaults +// against the cloud backend), carried as a sidecar so regeneration preserves +// it. Calibrated from gate-run failure details, not authored by hand. +export type CloudCannotRunAlone = Record + +export function validateCloudCannotRunAlone( + value: unknown +): CloudCannotRunAlone { + if (!isRecord(value)) + throw new Error( + 'cloudCannotRunAlone: expected { "": ["", ...] }' + ) + const sidecar: CloudCannotRunAlone = {} + for (const [pack, keys] of Object.entries(value)) { + if ( + !isStringArray(keys) || + keys.length === 0 || + keys.some((key) => key === '') || + new Set(keys).size !== keys.length + ) + throw new Error( + `cloudCannotRunAlone: ${pack} must be a non-empty array of unique non-empty node keys` + ) + sidecar[pack] = [...keys].sort() + } + return sidecar +} + +export function buildCloudManifest( + doc: SupportedNodesDoc, + snapshot: ObjectInfoSnapshot, + source: CloudManifestSource, + overlay: CuratedCloudOverlay = {}, + sentinels: CloudExtensionSentinels = {}, + cannotRunAlone: CloudCannotRunAlone = {} +): CloudManifest { + const nodesByPack = snapshotPacksOf(snapshot) + const dirnameByJoinKey = new Map() + for (const dirname of nodesByPack.keys()) { + if (dirname === 'core') continue + const key = joinKeyOf(dirname) + const collision = dirnameByJoinKey.get(key) + if (collision !== undefined) + throw new Error( + `snapshot packs ${collision} and ${dirname} collide on join key ${key}` + ) + dirnameByJoinKey.set(key, dirname) + } + + const unmatched: string[] = [] + const packs: CloudManifestEntry[] = [] + const seenYamlPacks = new Map() + for (const pack of doc.node_packs) { + if (pack.name === 'core') continue + const key = yamlJoinKeyOf(pack) + const prior = seenYamlPacks.get(key) + if (prior !== undefined) + throw new Error( + `yaml packs ${prior} and ${pack.name} collide on join key ${key}` + ) + seenYamlPacks.set(key, pack.name) + const dirname = dirnameByJoinKey.get(key) + if (dirname === undefined) { + unmatched.push(pack.name) + continue + } + // Assumes 'vanish' semantics - flip with CLOUD_DISABLED_SEMANTICS (cloudExclusions.ts). + const disabled = new Set(Object.keys(pack.node_labels ?? {})) + const enabled = (nodesByPack.get(dirname) ?? []) + .filter((node) => !disabled.has(node)) + .sort() + if (enabled.length === 0) + throw new Error( + `pack ${dirname}: every snapshot node is label-disabled - nothing left to expect` + ) + const curated = overlay[dirname] + // The default two sentinels are alphabetical, so a run-tier pack's load + // tier can end up asserting nodes its curated workflow never opens. The + // overlay may name the workflow's own nodes instead; they must still be + // nodes this env registers. + const offEnabled = (curated?.expectedNodes ?? []).filter( + (node) => !enabled.includes(node) + ) + if (offEnabled.length > 0) + throw new Error( + `curated overlay: ${dirname} expectedNodes ${offEnabled.sort().join(', ')} ` + + `are not enabled nodes of the pack - they must be snapshot nodes the ` + + `deployment leaves label-enabled` + ) + packs.push({ + pack: dirname, + deployRef: deployRefOf(pack), + tiers: curated?.tiers ?? ['load', 'connectivity'], + workflow: curated?.workflow ?? '', + expectedNodes: curated?.expectedNodes ?? enabled.slice(0, 2), + ...(curated + ? { expectedRunnableCount: curated.expectedRunnableCount } + : {}), + expectedNodeCount: enabled.length, + expectedExtensions: sentinels[dirname] ?? [], + disabledNodes: sortedRecordOf(pack.node_labels ?? {}), + ...(pack.web_directory !== undefined + ? { webDirectory: pack.web_directory } + : {}), + timeoutMs: curated?.timeoutMs ?? 30_000, + ...(curated && cannotRunAlone[dirname] + ? { cannotRunAlone: cannotRunAlone[dirname] } + : {}) + }) + } + // A yaml pack with no snapshot nodes gets recorded, not thrown on: it would + // otherwise block every other pack's row. The snapshot cannot say WHY it is + // empty (the pack registers nothing, or the dirname rule broke), so the list + // ships in the manifest for review instead of being silently dropped. + if (unmatched.length > packs.length) + throw new Error( + `${unmatched.length} yaml packs have no /object_info pack to join but only ` + + `${packs.length} joined - the dirname mapping rule has broken, not a few ` + + `empty packs: ${unmatched.slice(0, 10).join(', ')}` + ) + if (packs.length === 0) + throw new Error( + 'no pack rows generated - the yaml contains no joinable non-core packs' + ) + const orphaned = Object.keys(overlay).filter( + (pack) => !packs.some((row) => row.pack === pack) + ) + if (orphaned.length > 0) + throw new Error( + `curated overlay pack(s) with no generated row to attach to: ` + + `${orphaned.sort().join(', ')} - overlay keys must be snapshot pack ` + + `dirnames; known packs: ${packs.map((row) => row.pack).join(', ')}` + ) + const orphanedCalibration = Object.keys(cannotRunAlone).filter( + (pack) => !packs.some((row) => row.pack === pack) + ) + if (orphanedCalibration.length > 0) + throw new Error( + `cloudCannotRunAlone pack(s) with no generated row to attach to: ` + + `${orphanedCalibration.sort().join(', ')} - keys must be snapshot pack dirnames` + ) + + packs.sort((a, b) => (a.pack < b.pack ? -1 : a.pack > b.pack ? 1 : 0)) + packs.forEach(assertCloudEntry) + + const core = doc.node_packs.find((pack) => pack.name === 'core') + return { + source, + coreDisabledNodes: sortedRecordOf(core?.node_labels ?? {}), + packs, + unjoinedYamlPacks: unmatched.sort() + } +} + +export function renderCloudManifest(manifest: CloudManifest): string { + return `${JSON.stringify(manifest, null, 2)}\n` +} + +function dataPath(name: string): string { + return fileURLToPath( + new URL(`../browser_tests/fixtures/data/${name}`, import.meta.url) + ) +} + +function regenerationSidecars(manifest: CloudManifest): { + overlay: CuratedCloudOverlay + sentinels: CloudExtensionSentinels + cannotRunAlone: CloudCannotRunAlone +} { + const runEntries = manifest.packs.filter((entry) => + entry.tiers.includes('run') + ) + return { + overlay: Object.fromEntries( + runEntries.map((entry) => [ + entry.pack, + { + workflow: entry.workflow, + tiers: entry.tiers, + expectedNodes: entry.expectedNodes, + expectedRunnableCount: entry.expectedRunnableCount, + timeoutMs: entry.timeoutMs + } + ]) + ), + sentinels: Object.fromEntries( + manifest.packs + .filter((entry) => entry.expectedExtensions.length > 0) + .map((entry) => [entry.pack, entry.expectedExtensions]) + ), + cannotRunAlone: Object.fromEntries( + runEntries.flatMap((entry) => + entry.cannotRunAlone === undefined + ? [] + : [[entry.pack, entry.cannotRunAlone]] + ) + ) + } +} + +function generate(snapshotPath: string): void { + const yamlPath = dataPath('cloud/supported_nodes.yaml') + const manifestPath = dataPath('customNodeManifest.cloud.json') + const yamlContents = readFileSync(yamlPath, 'utf8') + const currentValue: unknown = JSON.parse(readFileSync(manifestPath, 'utf8')) + const current = assertCloudManifestShape(currentValue, manifestPath) + const snapshotValue: unknown = JSON.parse(readFileSync(snapshotPath, 'utf8')) + const { overlay, sentinels, cannotRunAlone } = regenerationSidecars(current) + const manifest = buildCloudManifest( + validateSupportedNodesDoc(parse(yamlContents)), + validateObjectInfoSnapshot(snapshotValue), + sourceFromSupportedNodesHeader(yamlContents), + validateCuratedCloudOverlay(overlay), + sentinels, + validateCloudCannotRunAlone(cannotRunAlone) + ) + writeFileSync(manifestPath, renderCloudManifest(manifest)) +} + +const invokedDirectly = + process.argv[1] !== undefined && + import.meta.url === pathToFileURL(process.argv[1]).href + +if (invokedDirectly) { + const snapshotPath = process.argv[2] + if (snapshotPath === undefined) { + process.stderr.write( + 'usage: pnpm gen:cloud-manifest \n' + ) + process.exitCode = 1 + } else { + generate(snapshotPath) + } +} diff --git a/scripts/customNodeQuarantine.ts b/scripts/customNodeQuarantine.ts new file mode 100644 index 00000000000..d025b76a724 --- /dev/null +++ b/scripts/customNodeQuarantine.ts @@ -0,0 +1,387 @@ +/** + * Reports every excluded pack and rechecks externally verifiable failures. + * + * Quarantine removes a pack from the population, so it is lost coverage that + * looks like a green run. The only thing that makes it safe is this: each + * source failure is rechecked against the live world, and a fixed entry fails. + * Hardware exclusions cannot be rechecked on this CPU runner; they stay bold + * in the Actions summary with the GPU-shard work needed to restore coverage. + * + * pnpm custom-node-quarantine + */ +import { execFile } from 'node:child_process' +import { appendFileSync, mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { promisify } from 'node:util' + +import type { QuarantinedPack } from '../browser_tests/fixtures/customNode/manifest' +import { connectivityExpectations } from '../browser_tests/fixtures/customNode/connectivityExpectations' +import { + FRONTEND_ASSET_EXCLUSIONS, + loadFullManifest, + loadPackQuarantine, + loadUnjoinedYamlPacks, + packIdentity, + staleLocalExpectations +} from '../browser_tests/fixtures/customNode/manifest' +import { consoleErrorExclusionsForPacks } from '../browser_tests/fixtures/customNode/consoleErrorLedger' +import { + CUSTOM_NODE_TIER_NODE_EXCLUSIONS, + tierNodeExclusionProblems +} from '../browser_tests/fixtures/customNode/tierNodeExclusions' +import { ROUNDTRIP_NODE_LOSS_EXPECTATIONS_LITEGRAPH } from '../browser_tests/fixtures/customNode/valueDrift' +import { + provesRefIsMissing, + provesRequirementIsUnsatisfiable +} from './customNodeQuarantineProbe' + +const run = promisify(execFile) + +const say = (line: string) => process.stdout.write(`${line}\n`) + +async function dryRunPython(): Promise { + const candidates = [ + process.env.CUSTOM_NODE_PYTHON, + 'python3', + 'python3.13', + 'python3.12', + 'python3.11', + 'python' + ].filter((candidate): candidate is string => candidate !== undefined) + for (const candidate of new Set(candidates)) { + try { + const { stdout } = await run( + candidate, + ['-m', 'pip', 'install', '--help'], + { timeout: 30_000 } + ) + if (stdout.includes('--dry-run')) return candidate + } catch { + continue + } + } + throw new Error('no Python interpreter provides pip install --dry-run') +} + +async function refStillMissing(deployRef: string): Promise { + const [url, sha] = deployRef.split(/@(?=[^@]*$)/) + if (!url || !sha) throw new Error(`invalid git deployRef: ${deployRef}`) + const dir = mkdtempSync(join(tmpdir(), 'cnq-')) + await run('git', ['init', '-q', dir], { timeout: 30_000 }) + await run('git', ['-C', dir, 'remote', 'add', 'origin', url]) + let lastError: unknown + for (let attempt = 0; attempt < 3; attempt++) { + try { + await run('git', ['-C', dir, 'fetch', '--depth', '1', 'origin', sha], { + timeout: 90_000 + }) + return false + } catch (error) { + if (provesRefIsMissing(error)) return true + lastError = error + if (attempt < 2) + await new Promise((resolve) => + setTimeout(resolve, (attempt + 1) * 1000) + ) + } + } + throw new Error(`could not conclusively recheck ${deployRef}`, { + cause: lastError + }) +} + +async function requirementsStillUnsatisfiable( + deployRef: string, + failurePattern: string +): Promise { + const [url, sha] = deployRef.split(/@(?=[^@]*$)/) + const slug = url?.split('github.com/')[1] + if (!slug || !sha) throw new Error(`invalid git deployRef: ${deployRef}`) + let body: string + try { + const { stdout } = await run('curl', [ + '-sfL', + '--retry', + '3', + '--retry-delay', + '1', + '--retry-connrefused', + '--max-time', + '30', + `https://raw.githubusercontent.com/${slug}/${sha}/requirements.txt` + ]) + body = stdout + } catch (error) { + throw new Error(`could not fetch requirements for ${deployRef}`, { + cause: error + }) + } + const dir = mkdtempSync(join(tmpdir(), 'cnq-')) + const file = join(dir, 'requirements.txt') + writeFileSync(file, body) + const python = await dryRunPython() + try { + await run( + python, + [ + '-m', + 'pip', + 'install', + '--dry-run', + '--quiet', + '--target', + join(dir, 'target'), + '-r', + file + ], + { timeout: 300_000 } + ) + return false + } catch (error) { + if (provesRequirementIsUnsatisfiable(error, failurePattern)) return true + throw new Error( + `could not conclusively resolve requirements for ${deployRef}`, + { cause: error } + ) + } +} + +async function stillBroken( + pack: string, + entry: QuarantinedPack, + deployRef: string +): Promise { + switch (entry.class) { + case 'unfetchable-ref': + return refStillMissing(deployRef) + case 'unsatisfiable-requirement': + return requirementsStillUnsatisfiable(deployRef, entry.failurePattern) + case 'requires-gpu-runner': + return true + default: + throw new Error(`${pack}: unknown quarantine class`) + } +} + +const quarantine = loadPackQuarantine() +const manifest = new Map(loadFullManifest().map((e) => [e.pack, e])) +const unjoinedYamlPacks = loadUnjoinedYamlPacks() +const manifestPackByFoldedName = new Map( + [...manifest.keys()].map((pack) => [pack.toLowerCase(), pack]) +) +const entries = Object.entries(quarantine) +const connectivityExclusions = [ + ...connectivityExpectations.connectRejected, + ...connectivityExpectations.deterministicSlotContractMismatch, + ...connectivityExpectations.dynamicSlotCleanupStalled, + ...connectivityExpectations.roundtripLost +].flatMap((group) => { + const pack = manifestPackByFoldedName.get(group.pack.toLowerCase()) + return pack + ? [ + { + label: `${pack} connectivity ${group.id} (${group.pairs.length} pair${group.pairs.length === 1 ? '' : 's'})`, + pack, + reason: group.reason, + restore: group.restore, + scope: 'connectivity breadth sweep', + tier: 'S4' as const, + mode: 'expected-failure' as const + } + ] + : [] +}) + +const summary: string[] = [] +const note = (line: string) => { + say(line) + summary.push(line) +} + +note( + `## Pack-level coverage exclusions - **${entries.length + unjoinedYamlPacks.length} of ${manifest.size + unjoinedYamlPacks.length} packs**` +) +note('') +note( + 'These are whole-pack skips: the pack was not installed, so **no S-tier ran** for it.' +) +note('') + +const stale: string[] = [] +const unknown: string[] = [] +const inconclusive: string[] = [] + +for (const pack of unjoinedYamlPacks) { + note(`- **PACK - SKIP - NO S-TIER RAN - ${pack}** \`unjoined-object-info\``) + note(' - meaning: no test surface exercised this pack') + note( + ' - the pinned Cloud YAML names the pack, but its object_info snapshot has no nodes attributable to it' + ) + note( + ' - to remove: fix the pack attribution or registration, then regenerate the cloud manifest' + ) + process.stdout.write( + `::warning title=Pack excluded from the suite::${pack} - no nodes joined from the pinned object_info snapshot\n` + ) +} + +for (const [pack, entry] of entries) { + const row = manifest.get(pack) + if (!row) { + unknown.push(pack) + note(`- **${pack}** - NOT IN MANIFEST, the exclusion outlived the pack`) + continue + } + const ref = packIdentity(row) + let broken: boolean + try { + broken = await stillBroken(pack, entry, ref) + } catch (error) { + const detail = error instanceof Error ? error.message : String(error) + inconclusive.push(`${pack}: ${detail}`) + note( + `- **PACK - SKIP - NO S-TIER RAN - ${pack} - PROBE INCONCLUSIVE** \`${entry.class}\`` + ) + note(' - meaning: no test surface exercised this pack') + note(` - ${entry.reason}`) + note(` - gate failure: ${detail}`) + process.stdout.write( + `::error title=Pack exclusion probe inconclusive::${pack} - ${detail}\n` + ) + continue + } + const status = + entry.class === 'requires-gpu-runner' + ? 'requires GPU-backed coverage' + : broken + ? 'still excluded' + : '**FIXED UPSTREAM**' + note( + `- **PACK - SKIP - NO S-TIER RAN - ${pack}** \`${entry.class}\` - ${status}` + ) + note(' - meaning: no test surface exercised this pack') + note(` - ${entry.reason}`) + note(` - to remove: ${entry.upstreamFix}`) + process.stdout.write( + `::warning title=Pack excluded from the suite::${pack} - ${entry.reason}\n` + ) + if (entry.class !== 'requires-gpu-runner' && !broken) stale.push(pack) +} + +const tierNodeProblems = tierNodeExclusionProblems( + [...manifest.values()].map((entry) => ({ + identity: packIdentity(entry), + pack: entry.pack + })) +) +const excludedTierSurfaces = CUSTOM_NODE_TIER_NODE_EXCLUSIONS.reduce( + (total, exclusion) => total + exclusion.tiers.length, + 0 +) +note('') +note( + `## Tier-scoped node coverage exclusions - **${CUSTOM_NODE_TIER_NODE_EXCLUSIONS.length} node, ${excludedTierSurfaces} S-tier surfaces**` +) +note('') +note( + 'The pack still has an exact registration-count sentinel and its other nodes run. Only the named node is excluded from the named tiers.' +) +note('') +for (const exclusion of CUSTOM_NODE_TIER_NODE_EXCLUSIONS) { + note( + `- **${exclusion.tiers.join('/')} - SKIP - NODE NOT EXERCISED - ${exclusion.pack} / ${exclusion.nodeType}**` + ) + note(` - pinned artifact: ${exclusion.identity}`) + note(` - reason: ${exclusion.reason}`) + note(` - to remove: ${exclusion.restore}`) + note(` - ticket: ${exclusion.ticket}`) + process.stdout.write( + `::warning title=${exclusion.tiers.join('/')} node coverage skipped::${exclusion.pack} / ${exclusion.nodeType} - ${exclusion.reason}\n` + ) +} + +const nodeExclusions = [ + ...Object.entries(ROUNDTRIP_NODE_LOSS_EXPECTATIONS_LITEGRAPH).flatMap( + ([pack, nodes]) => + Object.entries(nodes).map(([nodeType, exclusion]) => ({ + label: `${nodeType} save/reload`, + pack, + ...exclusion, + scope: 'save/reload sweep', + tier: 'S3' as const, + mode: 'expected-failure' as const + })) + ), + ...Object.entries(FRONTEND_ASSET_EXCLUSIONS).map(([pack, exclusion]) => ({ + label: `${pack} frontend assets`, + pack, + reason: exclusion.reason, + restore: exclusion.restore, + scope: 'frontend asset registration', + tier: 'S11' as const, + mode: 'expected-failure' as const + })), + ...consoleErrorExclusionsForPacks([...manifest.keys()]), + ...connectivityExclusions +].sort( + (left, right) => + left.tier.localeCompare(right.tier, undefined, { numeric: true }) || + left.label.localeCompare(right.label) +) +const unknownNodeExclusions: string[] = [] +const exactDefects = nodeExclusions.filter( + (exclusion) => exclusion.mode === 'expected-failure' +).length +const conditionalAllowances = nodeExclusions.length - exactDefects +note('') +note( + `## Tier-scoped acceptance ledger - **${exactDefects} exact defects, ${conditionalAllowances} conditional allowances**` +) +note('') +note( + 'These are assertion exceptions, not skipped tests. The named S-tier still ran. An exact defect must reproduce its recorded outcome; a conditional S8 allowance accepts only the named error signature if environmental state triggers it.' +) +note('') +for (const exclusion of nodeExclusions) { + const status = + exclusion.mode === 'conditional-console' + ? 'CONDITIONAL ERROR ALLOWANCE - TEST RAN' + : 'EXACT KNOWN DEFECT - TEST RAN' + note(`- **${exclusion.tier} - ${status} - ${exclusion.label}**`) + note(` - exercised in: ${exclusion.scope}`) + note(` - ${exclusion.reason}`) + note(` - to remove: ${exclusion.restore}`) + process.stdout.write( + `::warning title=${exclusion.tier} accepted defect::${exclusion.label} - ${exclusion.reason}\n` + ) + if (!manifest.has(exclusion.pack)) unknownNodeExclusions.push(exclusion.label) +} + +say('') +const problems: string[] = [...inconclusive, ...tierNodeProblems] +if (stale.length) + problems.push( + `${stale.join(', ')} now install cleanly - remove them from packQuarantine.json and let them back into the population` + ) +if (unknown.length) + problems.push( + `${unknown.join(', ')} are quarantined but not in the manifest - drop the entries` + ) +if (unknownNodeExclusions.length) + problems.push( + `${unknownNodeExclusions.join(', ')} exclude node checks for packs outside the manifest` + ) +const staleBaselines = staleLocalExpectations() +if (staleBaselines.length) + problems.push( + `${staleBaselines.join(', ')} now register the count the manifest expects - drop them from localExpectations.json` + ) +for (const problem of problems) { + note(`- **PROBLEM** ${problem}`) + process.stdout.write(`::error::${problem}\n`) +} + +const stepSummary = process.env.GITHUB_STEP_SUMMARY +if (stepSummary) appendFileSync(stepSummary, `${summary.join('\n')}\n`) +process.exitCode = problems.length ? 1 : 0 diff --git a/scripts/customNodeQuarantineProbe.test.ts b/scripts/customNodeQuarantineProbe.test.ts new file mode 100644 index 00000000000..01a2fa2d652 --- /dev/null +++ b/scripts/customNodeQuarantineProbe.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, test } from 'vitest' + +import { + provesRefIsMissing, + provesRequirementIsUnsatisfiable +} from './customNodeQuarantineProbe' + +describe('custom-node quarantine probes', () => { + test('accepts an explicit missing-ref response but not a network failure', () => { + expect( + provesRefIsMissing({ + stderr: 'fatal: remote error: upload-pack: not our ref abc123' + }) + ).toBe(true) + expect( + provesRefIsMissing({ stderr: 'Could not resolve host: github.com' }) + ).toBe(false) + }) + + test('requires both the declared requirement and a resolver verdict', () => { + const error = { + stderr: + 'Could not find a version that satisfies the requirement Imath>=3.1.0\nNo matching distribution found for Imath>=3.1.0' + } + expect(provesRequirementIsUnsatisfiable(error, 'Imath>=3.1.0')).toBe(true) + expect(provesRequirementIsUnsatisfiable(error, 'OpenEXR>=3.2.0')).toBe( + false + ) + expect( + provesRequirementIsUnsatisfiable( + { stderr: 'Temporary failure in name resolution: pypi.org' }, + 'Imath>=3.1.0' + ) + ).toBe(false) + expect( + provesRequirementIsUnsatisfiable( + { + stderr: + 'Collecting Imath>=3.1.0\nCould not find a version that satisfies the requirement Other>=1.0' + }, + 'Imath>=3.1.0' + ) + ).toBe(false) + }) +}) diff --git a/scripts/customNodeQuarantineProbe.ts b/scripts/customNodeQuarantineProbe.ts new file mode 100644 index 00000000000..e70efa294fd --- /dev/null +++ b/scripts/customNodeQuarantineProbe.ts @@ -0,0 +1,26 @@ +function diagnosticText(error: unknown): string { + if (typeof error === 'string') return error + if (typeof error !== 'object' || error === null) return String(error) + const record = error as Record + return [record.message, record.stderr, record.stdout] + .filter((value): value is string => typeof value === 'string') + .join('\n') +} + +export function provesRefIsMissing(error: unknown): boolean { + return /(?:not our ref|couldn't find remote ref|unadvertised object)/i.test( + diagnosticText(error) + ) +} + +export function provesRequirementIsUnsatisfiable( + error: unknown, + expectedRequirement: string +): boolean { + const diagnostic = diagnosticText(error) + const escaped = expectedRequirement.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + return new RegExp( + `(?:Could not find a version that satisfies the requirement|No matching distribution found for)\\s+${escaped}(?:\\s|\\(|$)`, + 'i' + ).test(diagnostic) +} diff --git a/scripts/customNodeShard.ts b/scripts/customNodeShard.ts new file mode 100644 index 00000000000..e2d48f3b14d --- /dev/null +++ b/scripts/customNodeShard.ts @@ -0,0 +1,76 @@ +/** + * Which packs this shard owns, for the CI install loop. + * + * The workflow installs packs and the specs register tests against them. If + * the two ever disagreed about the slice, the shard would install packs no + * test exercises and register tests for packs the backend does not have - so + * both read it from loadManifest(), and the workflow never re-derives the + * stripe itself. + * + * pnpm custom-node-shard packdeployRef, one per line + * pnpm custom-node-shard --expected-tests how many tests that slice registers + * pnpm custom-node-shard --expected-tier-tests + * how many S1/S2/S3/S9 tests register + * + * Reads CUSTOM_NODES_MANIFEST / CUSTOM_NODES_BACKEND / CUSTOM_NODES_SHARD, + * exactly as the specs do. + */ +import { + customNodesManifest, + loadApplicableAutogrowCases, + loadManifest, + packIdentity +} from '../browser_tests/fixtures/customNode/manifest' +import { hasCommittedProfile } from '../browser_tests/fixtures/customNode/interactionProfiles' + +// Tests every run registers whatever the slice holds: allNodes's +// manifest-coverage test, connectivity (three), coreSmoke (two), and the +// regression spec's three self-checks. The all-nodes TIERS are counted +// separately because they now register only when the slice contains a pack +// that asks for them. +const SLICE_INDEPENDENT_TESTS = 10 +// S1/S2/S3 need a 'load' pack; S9 needs a 'run' pack. A slice with neither +// registers neither. +const LOAD_TIERS = 3 +const RUN_TIERS = 1 +// Per pack in the slice: the regression spec's load test. +const TESTS_PER_PACK = 1 + +function expectedTestCount(): number { + const entries = loadManifest() + return ( + SLICE_INDEPENDENT_TESTS + + expectedTierTestCount(entries) + + TESTS_PER_PACK * entries.length + + (customNodesManifest() === 'core' + ? entries.filter((entry) => + hasCommittedProfile(entry.pack, packIdentity(entry)) + ).length + : 0) + + entries.filter((entry) => entry.tiers.includes('run')).length + + loadApplicableAutogrowCases().length + ) +} + +function expectedTierTestCount(entries = loadManifest()): number { + return ( + (entries.some((entry) => entry.tiers.includes('load')) ? LOAD_TIERS : 0) + + (entries.some((entry) => entry.tiers.includes('run')) ? RUN_TIERS : 0) + ) +} + +function packRows(): string[] { + return loadManifest().map((entry) => { + const installRef = + 'repo' in entry ? `${entry.repo}@${entry.pin}` : entry.deployRef + return `${entry.pack}\t${installRef}` + }) +} + +const lines = process.argv.includes('--expected-tests') + ? [String(expectedTestCount())] + : process.argv.includes('--expected-tier-tests') + ? [String(expectedTierTestCount())] + : packRows() + +process.stdout.write(`${lines.join('\n')}\n`) diff --git a/src/extensions/core/uploadAudio.test.ts b/src/extensions/core/uploadAudio.test.ts index 735cf21814d..144af7697ea 100644 --- a/src/extensions/core/uploadAudio.test.ts +++ b/src/extensions/core/uploadAudio.test.ts @@ -157,6 +157,29 @@ describe('Comfy.UploadAudio AUDIOUPLOAD widget', () => { capturedPaste = undefined }) + it('does not preview an empty audio option', async () => { + const AUDIOUPLOAD = await loadAudioUploadWidget() + const { audioWidget, node } = createAudioNode() + audioWidget.value = 'none' + audioWidget.options.values = ['none'] + + AUDIOUPLOAD(node, 'upload') + + expect(mockApiURL).not.toHaveBeenCalled() + + audioWidget.value = '' + audioWidget.options.values = [] + audioWidget.callback() + expect(mockApiURL).not.toHaveBeenCalled() + + audioWidget.value = 'none' + audioWidget.options.values = ['none', 'other.mp3'] + audioWidget.callback() + expect(mockApiURL).toHaveBeenCalledWith( + '/view?filename=none&subfolder=&type=input' + ) + }) + it('sets isUploading while upload is in progress and clears it after success', async () => { const AUDIOUPLOAD = await loadAudioUploadWidget() const { audioWidget, node } = createAudioNode() diff --git a/src/extensions/core/uploadAudio.ts b/src/extensions/core/uploadAudio.ts index 17e895e13eb..159b5b9eaf2 100644 --- a/src/extensions/core/uploadAudio.ts +++ b/src/extensions/core/uploadAudio.ts @@ -219,11 +219,15 @@ app.registerExtension({ ) as unknown as DOMWidget const onAudioWidgetUpdate = () => { + const value = audioWidget.value + const values = audioWidget.options.values + const isEmptySentinel = + value === 'none' && values?.length === 1 && values[0] === value updateUIWidget( audioUIWidget, - api.apiURL( - getResourceURL(...splitFilePath(audioWidget.value ?? '')) - ) + value && !isEmptySentinel + ? api.apiURL(getResourceURL(...splitFilePath(value))) + : '' ) } // Initially load default audio file to audioUIWidget. diff --git a/src/renderer/extensions/vueNodes/widgets/composables/useImageUploadWidget.test.ts b/src/renderer/extensions/vueNodes/widgets/composables/useImageUploadWidget.test.ts index 0fa94f896ad..2ffa359ac33 100644 --- a/src/renderer/extensions/vueNodes/widgets/composables/useImageUploadWidget.test.ts +++ b/src/renderer/extensions/vueNodes/widgets/composables/useImageUploadWidget.test.ts @@ -65,7 +65,7 @@ vi.mock('@/utils/litegraphUtil', () => ({ } })) -function createUploadNode() { +function createUploadNode(initialValue: string = 'missing.png') { const onWidgetChanged = vi.fn() const node = new LGraphNode('LoadImage') node.type = 'LoadImage' @@ -73,7 +73,7 @@ function createUploadNode() { const fileComboWidget = node.addWidget( 'combo', 'image', - 'missing.png', + initialValue, () => undefined, { values: ['missing.png'] } ) as IComboWidget @@ -81,6 +81,18 @@ function createUploadNode() { return { fileComboWidget, node, onWidgetChanged } } +function construct(node: LGraphNode) { + useImageUploadWidget()( + node, + 'upload', + [ + 'IMAGEUPLOAD', + { imageInputName: 'image', image_upload: true } + ] as InputSpec, + fromPartial({}) + ) +} + const outputFolderCases: { name: string value: string | ResultItem @@ -111,17 +123,8 @@ describe('useImageUploadWidget', () => { it('emits onWidgetChanged after upload changes the combo widget value', () => { const { fileComboWidget, node, onWidgetChanged } = createUploadNode() - const constructor = useImageUploadWidget() - constructor( - node, - 'upload', - [ - 'IMAGEUPLOAD', - { imageInputName: 'image', image_upload: true } - ] as InputSpec, - fromPartial({}) - ) + construct(node) mocks.capturedUploadOptions?.onUploadComplete(['uploaded.png']) @@ -137,6 +140,32 @@ describe('useImageUploadWidget', () => { ) }) + it('previews the combo value once the initial frame runs', () => { + const { node } = createUploadNode('beach.jpg') + const frame = vi.fn() + vi.stubGlobal('requestAnimationFrame', frame) + + construct(node) + frame.mock.calls[0][0]() + + expect(mocks.setNodeOutputs).toHaveBeenCalledWith(node, 'beach.jpg', { + isAnimated: false + }) + }) + + it('does not preview a combo whose value is still unset', () => { + const { fileComboWidget, node } = createUploadNode() + Object.assign(fileComboWidget, { value: undefined }) + const frame = vi.fn() + vi.stubGlobal('requestAnimationFrame', frame) + + construct(node) + frame.mock.calls[0][0]() + + expect(mocks.setNodeOutputs).not.toHaveBeenCalled() + expect(mocks.showPreview).toHaveBeenCalled() + }) + it.for(outputFolderCases)('$name', ({ value, expected }) => { const { fileComboWidget, node } = createUploadNode() const constructor = useImageUploadWidget() diff --git a/src/renderer/extensions/vueNodes/widgets/composables/useImageUploadWidget.ts b/src/renderer/extensions/vueNodes/widgets/composables/useImageUploadWidget.ts index 75a456acc01..0a0ff88ba79 100644 --- a/src/renderer/extensions/vueNodes/widgets/composables/useImageUploadWidget.ts +++ b/src/renderer/extensions/vueNodes/widgets/composables/useImageUploadWidget.ts @@ -128,9 +128,10 @@ export const useImageUploadWidget = () => { // The value isn't set immediately so we need to wait a moment // No change callbacks seem to be fired on initial setting of the value requestAnimationFrame(() => { - nodeOutputStore.setNodeOutputs(node, String(fileComboWidget.value), { - isAnimated - }) + if (fileComboWidget.value != null) + nodeOutputStore.setNodeOutputs(node, String(fileComboWidget.value), { + isAnimated + }) showPreview({ block: false }) })