From 6f4e510518ef2de4a650fa5259e8f538a4ed3aee Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" Date: Tue, 17 Nov 2026 09:29:58 +0900 Subject: [PATCH] v4: Fix git_exe, start-up without git, .git/config reading and snapshot restore Five fixes. The CHANGELOG has an entry for each. - Give the scheduled snapshot restore the directories of the server - Start with a node pack whose .git/config cannot be read - Use the git executable set as git_exe - Report the failed steps of a snapshot restore, restore the pinned commit and run the install step of the nodepacks it installs - Update nested submodules with pygit2 and fail when submodules could not be updated --- CHANGELOG.md | 74 ++ cm_cli/__main__.py | 40 +- comfyui_manager/common/git_compat.py | 119 +++- comfyui_manager/common/git_utils.py | 61 +- comfyui_manager/glob/manager_core.py | 169 +++-- comfyui_manager/glob/manager_server.py | 2 + .../glob/utils/environment_utils.py | 2 +- comfyui_manager/legacy/manager_core.py | 167 +++-- comfyui_manager/legacy/manager_server.py | 9 +- comfyui_manager/prestartup_script.py | 16 +- tests/_install_flags_testutil.py | 6 + tests/test_git_config_url.py | 112 +++ tests/test_git_exe_setting.py | 378 ++++++++++ tests/test_restore_summary.py | 656 ++++++++++++++++++ tests/test_scheduled_restore_command.py | 71 ++ 15 files changed, 1729 insertions(+), 153 deletions(-) create mode 100644 tests/test_git_config_url.py create mode 100644 tests/test_git_exe_setting.py create mode 100644 tests/test_restore_summary.py create mode 100644 tests/test_scheduled_restore_command.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b350d2707a..e0cc2fdbf9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,80 @@ and this project adheres to [Semantic Versioning 2.0.0](https://semver.org/spec/ middle of the operation that reported it. Numbers and quoted strings keep their colors. +- Carry out a scheduled snapshot restore in the directories of the server. + With `--base-directory`, the restore that runs at the next start worked on + the Manager's files and the `custom_nodes` directory inside the ComfyUI + directory instead of those of the base directory: it did not see the node + packs of the server, and ended with `Restore snapshot failed`. + +- Start with a node pack whose `.git/config` Manager could not read. A url + with a `%`, such as one with `%20`, a key without a value, such as `bare` + alone, a byte order mark at the start of the file, or on Windows text that + the code page of the system cannot decode, such as a Korean user name + written in UTF-8, raised an error while Manager was loading: ComfyUI did not + start, or with `--enable-manager-legacy-ui` started without Manager. These + files are read now, and for a file that cannot be read the url of that pack + is unknown and a warning names the file once. + +- Run the git executable set as `git_exe` in `config.ini`. The path was read + but not used: with the GitPython backend, git commands kept running the + `git` found on `PATH`, and without a `git` on `PATH` the pygit2 backend was + selected, or with the legacy UI Manager did not load. Both servers, the git + helper and `cm-cli` now run the configured executable. For the servers and + what they start it no longer has to be on `PATH`; `cm-cli` run by hand still + needs a git on `PATH`. A value that is not a working git is reported, and + the default git is used. The value is read as it is written: with a `%` in + it, as in a path of Windows, Manager read every setting of `config.ini` as + its default. The configured executable is also run when pygit2 + is asked for with `CM_USE_PYGIT2=1` but is not installed. +- Start without a git. With neither a `git` on `PATH` nor pygit2, ComfyUI did + not start, or with the legacy UI started without Manager; Manager now loads + and reports that git is missing, and what needs git fails when it is used. + With pygit2 and no `git`, the legacy UI started without Manager; it now + loads on the pygit2 backend, but what the legacy server does with GitPython + itself still needs a git. +- Run the start-up setup of the default server. It was never called, so the + server did not read the ComfyUI revision, did not log the ComfyUI version, + did not call the handlers that custom nodes register for the revision, and + did not check for nodepacks in an invalid state. + +- Report the steps of a snapshot restore that failed. The summary listed a + nodepack as `INSTALLED`, `DISABLED` or `ENABLED` without looking at the + result of the step: a clone that failed, for example because the directory + already existed or the url could not be reached, was `INSTALLED`. A step + that fails is listed as `FAILED`, and Manager itself is no longer listed as + `DISABLED`. A Registry that cannot be reached ended the restore at the first + nodepack to install from it, and so did a nodepack whose directory could not + be moved to disable or enable it; such a nodepack is listed as `FAILED`, and + the restore goes on. +- Restore the commit of the snapshot for every git nodepack. A git nodepack + that the restore had to clone stayed at the newest commit; it is checked out + at the commit of the snapshot, and its install script runs after that. An + installed git nodepack that the Registry does not know was checked out only + when `cm-cli` was run from `custom_nodes`, as a scheduled restore does; it + is checked out from any directory. The submodules of a git nodepack are + brought to the commits that the commit of the snapshot names; a submodule + that was never fetched is fetched only when the nodepack itself is checked + out. +- Install the requirements and run the install script of a Registry nodepack + that a snapshot restore installs. The nodepack was unpacked and nothing + more. It is listed as `FAILED` when they fail. +- Try a requirement or an install script that failed again. It was taken as + done until the next start: the next nodepack that names the requirement + skipped it, and installing or fixing the nodepack again did not run the + script. +- Update the submodules of a submodule with the pygit2 backend, and fail when + the submodules could not be updated. pygit2 updated the submodules of the + nodepack and not those below them, which git does. When it could not update + them and there was no `git` to do it in its place, or that `git` did not + answer in time, a message was printed and the install or the update went on + as if they were updated. On Windows an update still ends as done: the helper + that runs it prints the failure, and its result is not read. +- Record the commit of a git nodepack whose refs are packed, as after + `git gc`, in a snapshot. It was recorded as `unknown`. A `HEAD` that is not + UTF-8 raised an error where the commit of a nodepack is read; the commit of + such a nodepack is now unknown. + - Refuse a `db_mode` or an `update_policy` that Manager does not know in the legacy server. With `--enable-manager-legacy-ui`, the two setting routes stored whatever value a request carried and wrote it to `config.ini`. They diff --git a/cm_cli/__main__.py b/cm_cli/__main__.py index 9d15d308ec..56d18d6baa 100644 --- a/cm_cli/__main__.py +++ b/cm_cli/__main__.py @@ -32,6 +32,7 @@ import utils.extra_config +from comfyui_manager.common.git_compat import setup_git_environment # before the modules that import GitPython from comfyui_manager.common import manager_util from comfyui_manager.common import cm_global from comfyui_manager.legacy import manager_core as core @@ -135,25 +136,26 @@ def set_no_deps(self, no_deps): self.no_deps = no_deps def set_user_directory(self, user_directory): - if user_directory is None: - return - - extra_model_paths_yaml = os.path.join(user_directory, 'extra_model_paths.yaml') - if os.path.exists(extra_model_paths_yaml): - utils.extra_config.load_extra_path_config(extra_model_paths_yaml) - - context.update_user_directory(user_directory) - - if os.path.exists(context.manager_pip_overrides_path): - with open(context.manager_pip_overrides_path, 'r', encoding="UTF-8", errors="ignore") as json_file: - cm_global.pip_overrides = json.load(json_file) - - if os.path.exists(context.manager_pip_blacklist_path): - with open(context.manager_pip_blacklist_path, 'r', encoding="UTF-8", errors="ignore") as f: - for x in f.readlines(): - y = x.strip() - if y != '': - cm_global.pip_blacklist.add(y) + if user_directory is not None: + extra_model_paths_yaml = os.path.join(user_directory, 'extra_model_paths.yaml') + if os.path.exists(extra_model_paths_yaml): + utils.extra_config.load_extra_path_config(extra_model_paths_yaml) + + context.update_user_directory(user_directory) + + if os.path.exists(context.manager_pip_overrides_path): + with open(context.manager_pip_overrides_path, 'r', encoding="UTF-8", errors="ignore") as json_file: + cm_global.pip_overrides = json.load(json_file) + + if os.path.exists(context.manager_pip_blacklist_path): + with open(context.manager_pip_blacklist_path, 'r', encoding="UTF-8", errors="ignore") as f: + for x in f.readlines(): + y = x.strip() + if y != '': + cm_global.pip_blacklist.add(y) + + # the configuration of the user directory is known from here on + setup_git_environment(core.get_config()['git_exe']) def update_custom_nodes_dir(self, target_dir): import folder_paths diff --git a/comfyui_manager/common/git_compat.py b/comfyui_manager/common/git_compat.py index 42dc2940d6..f8f02c0dcd 100644 --- a/comfyui_manager/common/git_compat.py +++ b/comfyui_manager/common/git_compat.py @@ -15,6 +15,8 @@ import os import re +import shutil +import subprocess import sys from abc import ABC, abstractmethod from collections import deque @@ -52,25 +54,71 @@ def _to_https_url(url): # Backend selection # --------------------------------------------------------------------------- +def find_git_executable(git_exe, timeout=10): + """The full path of the program `git_exe` if it answers `version` as git does; None otherwise.""" + # A name without a directory is looked up on PATH; GitPython would take it + # as a file of the current directory. + path = shutil.which(git_exe) + if path is None: + return None + + try: + res = subprocess.run([path, 'version'], capture_output=True, stdin=subprocess.DEVNULL, timeout=timeout) + except (OSError, subprocess.SubprocessError): + return None + + if res.returncode != 0 or not res.stdout.startswith(b'git version'): + return None + + return os.path.abspath(path) + + _PYGIT2_REQUESTED = os.environ.get('CM_USE_PYGIT2', '').strip() == '1' USE_PYGIT2 = _PYGIT2_REQUESTED +# The git set as `git_exe`, from prestartup or from the process that runs git_helper. +# GitPython is told before it is imported, so that a git that is not on PATH can be named. +_GIT_EXE = os.environ.get('GIT_EXE_PATH') +_GIT_EXE_FOUND = None + + +def _find_configured_git(): + found = find_git_executable(_GIT_EXE) + if found is not None: + os.environ['GIT_PYTHON_GIT_EXECUTABLE'] = found + else: + print(f"[ComfyUI-Manager] `git_exe = {_GIT_EXE}` is not a working git executable. The default git is used.", file=sys.stderr) + return found + + +if not USE_PYGIT2 and _GIT_EXE: + _GIT_EXE_FOUND = _find_configured_git() + if not USE_PYGIT2: try: import git as _git - _git.Git().execute(['git', '--version']) + _git.Git().execute([_git.Git.GIT_PYTHON_GIT_EXECUTABLE, '--version']) except Exception: USE_PYGIT2 = True if USE_PYGIT2: + # Without a git, GitPython raises already when it is imported, unless it + # is told to keep quiet. The legacy modules import it themselves. + os.environ.setdefault('GIT_PYTHON_REFRESH', 'quiet') + try: import pygit2 as _pygit2 except ImportError: # pygit2 not available either — fall back to GitPython and let it - # fail at the point of use, preserving pre-existing behavior. + # fail at the point of use. USE_PYGIT2 = False + if _PYGIT2_REQUESTED and _GIT_EXE: + # pygit2 was asked for, so the setting was not looked at above + _GIT_EXE_FOUND = _find_configured_git() _PYGIT2_REQUESTED = False import git as _git + if not _git.GIT_OK: + print("[ComfyUI-Manager] No git executable was found and pygit2 is not installed. What needs git will fail.", file=sys.stderr) else: # Disable owner validation once at import time. # Required for Desktop 2.0 standalone installs where repo directories @@ -237,7 +285,7 @@ def checkout(self, ref): ... def checkout_new_branch(self, branch_name: str, start_point: str): ... @abstractmethod - def submodule_update(self): ... + def submodule_update(self, init=True): ... @abstractmethod def clear_cache(self): ... @@ -424,8 +472,8 @@ def checkout(self, ref): def checkout_new_branch(self, branch_name, start_point): self._repo.git.checkout('-b', branch_name, start_point) - def submodule_update(self): - self._repo.git.submodule('update', '--init', '--recursive') + def submodule_update(self, init=True): + self._repo.git.submodule('update', *(['--init'] if init else []), '--recursive') def clear_cache(self): self._repo.git.clear_cache() @@ -845,28 +893,45 @@ def checkout_new_branch(self, branch_name, start_point): self._repo.checkout(branch) self._repo.set_head(branch.name) - def submodule_update(self): + def _update_submodules(self, repo): + # One that `.gitmodules` alone names has no commit, and a commit that `.gitmodules` does not name has no url: + # neither can be fetched, and pygit2 refuses the second. + names = [submodule.name for submodule in repo.submodules + if submodule.head_id is not None and submodule.url is not None] + if names: + repo.submodules.init(names) + repo.submodules.update(names) + + # as `--recursive`: the submodules of a submodule + for name in names: + self._update_submodules(repo.submodules[name].open()) + + def submodule_update(self, init=True): try: - self._repo.submodules.init() - self._repo.submodules.update() - except Exception: + if init: + self._update_submodules(self._repo) + else: + # one that was never fetched is left alone; pygit2 refuses it and would not come to the others + fetched = [submodule.name for submodule in self._repo.submodules + if os.path.exists(os.path.join(self._working_dir, submodule.path, '.git'))] + if fetched: + self._repo.submodules.update(fetched) + except Exception as e: import subprocess try: result = subprocess.run( - ['git', 'submodule', 'update', '--init', '--recursive'], + ['git', 'submodule', 'update', *(['--init'] if init else []), '--recursive'], cwd=self._working_dir, capture_output=True, timeout=120, ) - if result.returncode != 0: - raise GitCommandError( - f"submodule update failed (exit {result.returncode}): " - f"{result.stderr.decode(errors='replace')}") except FileNotFoundError: - print("[ComfyUI-Manager] pygit2: submodule update requires system git (not installed)", file=sys.stderr) - except GitCommandError: - raise - except Exception as sub_e: - print(f"[ComfyUI-Manager] pygit2: submodule update failed: {sub_e}", file=sys.stderr) + # no git to do it in the place of pygit2: the caller must not take the update as done + raise GitCommandError(f"submodule update failed: {e}") from e + + if result.returncode != 0: + raise GitCommandError( + f"submodule update failed (exit {result.returncode}): " + f"{result.stderr.decode(errors='replace')}") def clear_cache(self): pass @@ -938,7 +1003,17 @@ def clone_repo(url, dest, progress=None): def setup_git_environment(git_exe): """Configure the git executable path (GitPython only).""" - if USE_PYGIT2: + if USE_PYGIT2 or not git_exe: return - if git_exe: - _git.Git().update_environment(GIT_PYTHON_GIT_EXECUTABLE=git_exe) + + if git_exe == _GIT_EXE: + # checked, and reported if it is no git, when this module was imported + found = _GIT_EXE_FOUND + else: + found = find_git_executable(git_exe) + if found is None: + print(f"[ComfyUI-Manager] `git_exe = {git_exe}` is not a working git executable. The default git is used.", file=sys.stderr) + + # GitPython may have been imported before this module, with the git on PATH + if found is not None and _git.Git.GIT_PYTHON_GIT_EXECUTABLE != found: + _git.refresh(found) diff --git a/comfyui_manager/common/git_utils.py b/comfyui_manager/common/git_utils.py index 9e8ca5bbfd..32a16d0bf0 100644 --- a/comfyui_manager/common/git_utils.py +++ b/comfyui_manager/common/git_utils.py @@ -1,5 +1,6 @@ import os import configparser +import logging GITHUB_ENDPOINT = os.getenv('GITHUB_ENDPOINT') @@ -15,23 +16,59 @@ def is_git_repo(path: str) -> bool: def get_commit_hash(fullpath): git_head = os.path.join(fullpath, '.git', 'HEAD') if os.path.exists(git_head): - with open(git_head) as f: + # a ref can be named in any bytes, so the files are read as bytes and the names compared as they are + with open(git_head, 'rb') as f: line = f.readline() - if line.startswith("ref: "): - ref = os.path.join(fullpath, '.git', line[5:].strip()) + if line.startswith(b"ref: "): + ref_name = line[5:].strip() + ref = os.path.join(os.fsencode(fullpath), b'.git', ref_name) if os.path.exists(ref): - with open(ref) as f2: - return f2.readline().strip() - else: - return "unknown" + with open(ref, 'rb') as f2: + return f2.readline().strip().decode('ascii', errors='replace') + + # after `git pack-refs` or `git gc` the ref is a line of packed-refs + packed_refs = os.path.join(fullpath, '.git', 'packed-refs') + if os.path.exists(packed_refs): + with open(packed_refs, 'rb') as f2: + for packed in f2: + fields = packed.split() + if len(fields) == 2 and fields[1] == ref_name: + return fields[0].decode('ascii', errors='replace') + + return "unknown" else: # Detached HEAD: the file holds the commit hash and a newline. - return line.strip() + return line.strip().decode('ascii', errors='replace') return "unknown" +unreadable_git_configs = set() + + +def read_git_config(git_config_path): + """ + parse a .git/config; None if it cannot be read + """ + # Set `strict=False` to allow duplicate `vscode-merge-base` sections, addressing + # a url may hold a `%`, git accepts a key without a value, the file may start with a BOM, + # and a value that is not UTF-8 does not make the others unreadable + config = configparser.ConfigParser(strict=False, interpolation=None, allow_no_value=True) + try: + with open(git_config_path, encoding='utf-8-sig', errors='replace') as f: + config.read_file(f) + except (configparser.Error, AttributeError, OSError) as e: + # AttributeError: a line more indented than the key without a value before it + # the files are read again at every reload of the node packs + if git_config_path not in unreadable_git_configs: + unreadable_git_configs.add(git_config_path) + logging.warning(f"[ComfyUI-Manager] Cannot read '{git_config_path}': {e}") + return None + + return config + + def git_url(fullpath): """ resolve version of unclassified custom node based on remote url in .git/config @@ -41,12 +78,12 @@ def git_url(fullpath): if not os.path.exists(git_config_path): return None - # Set `strict=False` to allow duplicate `vscode-merge-base` sections, addressing - config = configparser.ConfigParser(strict=False) - config.read(git_config_path) + config = read_git_config(git_config_path) + if config is None: + return None for k, v in config.items(): - if k.startswith('remote ') and 'url' in v: + if k.startswith('remote ') and v.get('url'): if 'Comfy-Org/ComfyUI-Manager' in v['url']: return "https://github.com/ltdrdata/ComfyUI-Manager" return v['url'] diff --git a/comfyui_manager/glob/manager_core.py b/comfyui_manager/glob/manager_core.py index d73733810a..b6d7c3196e 100644 --- a/comfyui_manager/glob/manager_core.py +++ b/comfyui_manager/glob/manager_core.py @@ -801,19 +801,23 @@ def execute_install_script(self, url, repo_path, instant_execution=False, lazy_m for line in lines: package_name = remap_pip_package(line.strip()) if package_name and not package_name.startswith('#') and package_name not in self.processed_install: - self.processed_install.add(package_name) clean_package_name = package_name.split('#')[0].strip() install_cmd = manager_util.make_pip_cmd(["install", clean_package_name]) if clean_package_name != "" and not clean_package_name.startswith('#'): res = res and try_install_script(url, repo_path, install_cmd, instant_execution=instant_execution) + # one that was not installed is tried again for the next node pack that names it + if res: + self.processed_install.add(package_name) pip_fixer.fix_broken() if os.path.exists(install_script_path) and install_script_path not in self.processed_install: - self.processed_install.add(install_script_path) print("Install: install script") install_cmd = [sys.executable, "install.py"] - return res and try_install_script(url, repo_path, install_cmd, instant_execution=instant_execution) + res = res and try_install_script(url, repo_path, install_cmd, instant_execution=instant_execution) + # one that did not run, or failed, is run when the node pack is installed or fixed again + if res: + self.processed_install.add(install_script_path) return res @@ -1715,7 +1719,8 @@ def get_bool(key, default_value): return { 'http_channel_enabled': get_bool('http_channel_enabled', False), - 'git_exe': default_conf.get('git_exe', ''), + # read as written: a path of Windows may hold a `%`, which the parser takes as the start of a reference + 'git_exe': default_conf.get('git_exe', '', raw=True), 'use_uv': get_bool('use_uv', True), 'use_unified_resolver': get_bool('use_unified_resolver', False), 'channel_url': default_conf.get('channel_url', DEFAULT_CHANNEL), @@ -3132,6 +3137,22 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): installed_node_packs = [] failed = [] + def run_postinstall(res): + # an install script that fails can raise; the restore goes on with the other packs + try: + return res.postinstall() + except Exception: + traceback.print_exc() + return False + + def move(step, *args): + # disabling and enabling move the directory of the nodepack; one that cannot be moved fails alone + try: + return step(*args) + except Exception as e: + logging.error(f"[ComfyUI-Manager] {e}") + return ManagedResult('fail').fail(str(e)) + await unified_manager.reload('cache') await unified_manager.get_custom_nodes('default', 'cache') @@ -3141,8 +3162,6 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): print("Restore snapshot.") - postinstalls = [] - with open(snapshot_path, 'r', encoding="UTF-8") as snapshot_file: if snapshot_path.endswith('.json'): info = json.load(snapshot_file) @@ -3177,11 +3196,17 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): skip_node_packs.append(k) for x in todo_disable: - unified_manager.unified_disable(x, False) - disabled_repos.append(x) + if move(unified_manager.unified_disable, x, False).result: + disabled_repos.append(x) + else: + failed.append(x) for x in todo_checkout: - ps = unified_manager.cnr_switch_version(x[0], x[1], instant_execution=True, no_deps=True, return_postinstall=False) + try: + ps = unified_manager.cnr_switch_version(x[0], x[1], instant_execution=True, no_deps=True, return_postinstall=False) + except Exception as e: + # the request to the Registry raises when it cannot be reached; the restore goes on with the other packs + ps = ManagedResult('switch-cnr').fail(str(e)) if ps.action == 'switch-cnr' and ps.result: switched_node_packs.append(f"{x[0]}@{x[1]}") elif ps.action == 'skip': @@ -3196,18 +3221,17 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): if 'comfyui-manager' in k or k in checked_cnr: continue - ps = await unified_manager.install_by_id(k, version_spec=v, instant_execution=True, return_postinstall=True) - if ps.action == 'install-cnr' and ps.result: - installed_node_packs.append(f"{k}@{v}") - elif not ps.result: + try: + ps = await unified_manager.install_by_id(k, version_spec=v, instant_execution=True, return_postinstall=True) + except Exception as e: + ps = ManagedResult('install-cnr').fail(str(e)) + if not ps.result: failed.append(f"{k}@{v}") logging.error(ps.msg) - - if ps is not None and ps.result: - if hasattr(ps, 'postinstall'): - postinstalls.append(ps.postinstall) - else: - print("cm-cli: unexpected [0001]") + elif not run_postinstall(ps): + failed.append(f"{k}@{v}") + elif ps.action == 'install-cnr': + installed_node_packs.append(f"{k}@{v}") # for nightly restore _git_info = info.get('git_custom_nodes') @@ -3260,17 +3284,23 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): processed_urls.append(normalized_url) for x in todo_disable: - unified_manager.unified_disable(x, False) - disabled_repos.append(x) + if move(unified_manager.unified_disable, x, False).result: + disabled_repos.append(x) + else: + failed.append(x) for x in todo_enable: - res = unified_manager.unified_enable(x[0], 'nightly') + res = move(unified_manager.unified_enable, x[0], 'nightly') is_switched = False - if res and res.target: + if res.result and res.target: is_switched = repo_switch_commit(res.target, x[1]) - if is_switched: + if not res.result: + failed.append(x[0]) + elif is_switched is None: + failed.append(f"{x[0]}@{x[1]}") + elif is_switched: checkout_repos.append(f"{x[0]}@{x[1]}") else: enabled_repos.append(x[0]) @@ -3280,15 +3310,30 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): if is_switched: checkout_repos.append(f"{x[0]}@{x[1]}") + elif is_switched is None: + failed.append(f"{x[0]}@{x[1]}") for x in git_info.keys(): normalized_url = git_utils.normalize_url(x) cnr = unified_manager.repo_cnr_map.get(normalized_url) if cnr is not None: pack_id = cnr['id'] - res = await unified_manager.install_by_id(pack_id, 'nightly', instant_execution=True, no_deps=False, return_postinstall=False) + # the install script runs after the checkout, for the files of the pinned commit; + # a clone that cannot be checked out stays, so it runs for that one as well + try: + res = await unified_manager.install_by_id(pack_id, 'nightly', instant_execution=True, no_deps=False, return_postinstall=True) + except Exception as e: + logging.error(f"[ComfyUI-Manager] {e}") + res = ManagedResult('fail').fail(str(e)) if res.action == 'install-git' and res.result: - cloned_repos.append(pack_id) + is_switched = repo_switch_commit(unified_manager.active_nodes[pack_id][1], git_info[x]['hash']) + is_installed = run_postinstall(res) + if is_switched is None: + failed.append(f"{pack_id}@{git_info[x]['hash']}") + elif not is_installed: + failed.append(pack_id) + else: + cloned_repos.append(pack_id) elif res.action == 'skip': skip_node_packs.append(pack_id) elif not res.result: @@ -3306,6 +3351,9 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): processed_urls = [] for k2, v2 in unified_manager.unknown_active_nodes.items(): + if 'comfyui-manager' in k2.lower(): + continue + repo_url = resolve_giturl_from_path(v2[1]) if repo_url is None: @@ -3317,7 +3365,7 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): todo_disable.append(k2) else: commit_hash = git_info[normalized_url]['hash'] - todo_checkout.append((k2, commit_hash)) + todo_checkout.append((k2, v2[1], commit_hash)) processed_urls.append(normalized_url) for k2, v2 in unified_manager.unknown_inactive_nodes.items(): @@ -3334,28 +3382,36 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): processed_urls.append(normalized_url) for x in todo_disable: - unified_manager.unified_disable(x, True) - disabled_repos.append(x) + if move(unified_manager.unified_disable, x, True).result: + disabled_repos.append(x) + else: + failed.append(x) for x in todo_enable: - res = unified_manager.unified_enable(x[0], 'unknown') + res = move(unified_manager.unified_enable, x[0], 'unknown') is_switched = False - if res and res.target: + if res.result and res.target: is_switched = repo_switch_commit(res.target, x[1]) - if is_switched: + if not res.result: + failed.append(x[0]) + elif is_switched is None: + failed.append(f"{x[0]}@{x[1]}") + elif is_switched: checkout_repos.append(f"{x[0]}@{x[1]}") else: enabled_repos.append(x[0]) - for x in todo_checkout: - is_switched = repo_switch_commit(x[0], x[1]) + for name, path, commit_hash in todo_checkout: + is_switched = repo_switch_commit(path, commit_hash) if is_switched: - checkout_repos.append(f"{x[0]}@{x[1]}") + checkout_repos.append(f"{name}@{commit_hash}") + elif is_switched is None: + failed.append(f"{name}@{commit_hash}") else: - skip_node_packs.append(x[0]) + skip_node_packs.append(name) for x in processed_urls: if x in git_info: @@ -3367,8 +3423,19 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): repo_name = repo_name[:-4] to_path = os.path.join(get_default_custom_nodes_path(), repo_name) - unified_manager.repo_install(repo_url, to_path, instant_execution=True, no_deps=False, return_postinstall=False) - cloned_repos.append(repo_name) + res = unified_manager.repo_install(repo_url, to_path, instant_execution=True, no_deps=False, return_postinstall=True) + if not res.result: + failed.append(repo_name) + continue + + is_switched = repo_switch_commit(to_path, git_info[repo_url]['hash']) + is_installed = run_postinstall(res) + if is_switched is None: + failed.append(f"{repo_name}@{git_info[repo_url]['hash']}") + elif not is_installed: + failed.append(repo_name) + else: + cloned_repos.append(repo_name) manager_util.restore_pip_snapshot(pips, git_helper_extras) @@ -3387,7 +3454,7 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): print(f"[ DISABLED ] {x}") for x in skip_node_packs: print(f"[ SKIPPED ] {x}") - for x in failed: + for x in dict.fromkeys(failed): # a pack that failed at two steps is listed once print(f"[ FAILED ] {x}") # if is_failed: @@ -3498,23 +3565,31 @@ def resolve_giturl_from_path(fullpath): if not os.path.exists(git_config_path): return "unknown" - config = configparser.ConfigParser(strict=False) - config.read(git_config_path) + config = git_utils.read_git_config(git_config_path) + if config is None: + return None for k, v in config.items(): - if k.startswith('remote ') and 'url' in v: + if k.startswith('remote ') and v.get('url'): return v['url'].replace("git@github.com:", "https://github.com/") return None def repo_switch_commit(repo_path, commit_hash): + if commit_hash == 'unknown': + return False + try: with open_repo(repo_path) as repo: - if repo.head_commit_hexsha == commit_hash: - return False - - repo.checkout(commit_hash) - return True + if repo.head_commit_hexsha != commit_hash: + repo.checkout(commit_hash) + repo.submodule_update() + return True + + # an update of the submodules that failed is done again; one that was never fetched is left alone + if os.path.exists(os.path.join(repo_path, '.gitmodules')): + repo.submodule_update(init=False) + return False except Exception: return None diff --git a/comfyui_manager/glob/manager_server.py b/comfyui_manager/glob/manager_server.py index 06dd5d422a..7917f8f666 100644 --- a/comfyui_manager/glob/manager_server.py +++ b/comfyui_manager/glob/manager_server.py @@ -99,6 +99,8 @@ network_mode_description = core.get_config()["network_mode"] logging.info("[ComfyUI-Manager] network_mode: " + network_mode_description) +environment_utils.initialize_environment() + MAXIMUM_HISTORY_SIZE = 10000 routes = PromptServer.instance.routes diff --git a/comfyui_manager/glob/utils/environment_utils.py b/comfyui_manager/glob/utils/environment_utils.py index 60ea20af4c..0c4b4e3fcf 100644 --- a/comfyui_manager/glob/utils/environment_utils.py +++ b/comfyui_manager/glob/utils/environment_utils.py @@ -132,7 +132,7 @@ def initialize_environment(): # manager_util.comfyui_manager_path, "extension-node-map.json" # ) - print_comfyui_version() setup_environment() + print_comfyui_version() core.check_invalid_nodes() diff --git a/comfyui_manager/legacy/manager_core.py b/comfyui_manager/legacy/manager_core.py index e1ac47dd28..b7d0e18a4c 100644 --- a/comfyui_manager/legacy/manager_core.py +++ b/comfyui_manager/legacy/manager_core.py @@ -788,18 +788,22 @@ def execute_install_script(self, url, repo_path, instant_execution=False, lazy_m for line in lines: package_name = remap_pip_package(line.strip()) if package_name and not package_name.startswith('#') and package_name not in self.processed_install: - self.processed_install.add(package_name) install_cmd = manager_util.make_pip_cmd(["install", package_name]) if package_name.strip() != "" and not package_name.startswith('#'): res = res and try_install_script(url, repo_path, install_cmd, instant_execution=instant_execution) + # one that was not installed is tried again for the next node pack that names it + if res: + self.processed_install.add(package_name) pip_fixer.fix_broken() if os.path.exists(install_script_path) and install_script_path not in self.processed_install: - self.processed_install.add(install_script_path) print("Install: install script") install_cmd = [sys.executable, "install.py"] - return res and try_install_script(url, repo_path, install_cmd, instant_execution=instant_execution) + res = res and try_install_script(url, repo_path, install_cmd, instant_execution=instant_execution) + # one that did not run, or failed, is run when the node pack is installed or fixed again + if res: + self.processed_install.add(install_script_path) return res @@ -1726,7 +1730,8 @@ def get_bool(key, default_value): return { 'http_channel_enabled': get_bool('http_channel_enabled', False), - 'git_exe': default_conf.get('git_exe', ''), + # read as written: a path of Windows may hold a `%`, which the parser takes as the start of a reference + 'git_exe': default_conf.get('git_exe', '', raw=True), 'use_uv': get_bool('use_uv', True), 'channel_url': default_conf.get('channel_url', DEFAULT_CHANNEL), 'default_cache_as_channel_url': get_bool('default_cache_as_channel_url', False), @@ -3145,6 +3150,22 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): installed_node_packs = [] failed = [] + def run_postinstall(res): + # an install script that fails can raise; the restore goes on with the other packs + try: + return res.postinstall() + except Exception: + traceback.print_exc() + return False + + def move(step, *args): + # disabling and enabling move the directory of the nodepack; one that cannot be moved fails alone + try: + return step(*args) + except Exception as e: + logging.error(f"[ComfyUI-Manager] {e}") + return ManagedResult('fail').fail(str(e)) + await unified_manager.reload('cache') await unified_manager.get_custom_nodes('default', 'cache') @@ -3154,8 +3175,6 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): print("Restore snapshot.") - postinstalls = [] - with open(snapshot_path, 'r', encoding="UTF-8") as snapshot_file: if snapshot_path.endswith('.json'): info = json.load(snapshot_file) @@ -3190,11 +3209,17 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): skip_node_packs.append(k) for x in todo_disable: - unified_manager.unified_disable(x, False) - disabled_repos.append(x) + if move(unified_manager.unified_disable, x, False).result: + disabled_repos.append(x) + else: + failed.append(x) for x in todo_checkout: - ps = unified_manager.cnr_switch_version(x[0], x[1], instant_execution=True, no_deps=True, return_postinstall=False) + try: + ps = unified_manager.cnr_switch_version(x[0], x[1], instant_execution=True, no_deps=True, return_postinstall=False) + except Exception as e: + # the request to the Registry raises when it cannot be reached; the restore goes on with the other packs + ps = ManagedResult('switch-cnr').fail(str(e)) if ps.action == 'switch-cnr' and ps.result: switched_node_packs.append(f"{x[0]}@{x[1]}") elif ps.action == 'skip': @@ -3209,18 +3234,17 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): if 'comfyui-manager' in k or k in checked_cnr: continue - ps = await unified_manager.install_by_id(k, version_spec=v, instant_execution=True, return_postinstall=True) - if ps.action == 'install-cnr' and ps.result: - installed_node_packs.append(f"{k}@{v}") - elif not ps.result: + try: + ps = await unified_manager.install_by_id(k, version_spec=v, instant_execution=True, return_postinstall=True) + except Exception as e: + ps = ManagedResult('install-cnr').fail(str(e)) + if not ps.result: failed.append(f"{k}@{v}") logging.error(ps.msg) - - if ps is not None and ps.result: - if hasattr(ps, 'postinstall'): - postinstalls.append(ps.postinstall) - else: - print("cm-cli: unexpected [0001]") + elif not run_postinstall(ps): + failed.append(f"{k}@{v}") + elif ps.action == 'install-cnr': + installed_node_packs.append(f"{k}@{v}") # for nightly restore _git_info = info.get('git_custom_nodes') @@ -3273,17 +3297,23 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): processed_urls.append(normalized_url) for x in todo_disable: - unified_manager.unified_disable(x, False) - disabled_repos.append(x) + if move(unified_manager.unified_disable, x, False).result: + disabled_repos.append(x) + else: + failed.append(x) for x in todo_enable: - res = unified_manager.unified_enable(x[0], 'nightly') + res = move(unified_manager.unified_enable, x[0], 'nightly') is_switched = False - if res and res.target: + if res.result and res.target: is_switched = repo_switch_commit(res.target, x[1]) - if is_switched: + if not res.result: + failed.append(x[0]) + elif is_switched is None: + failed.append(f"{x[0]}@{x[1]}") + elif is_switched: checkout_repos.append(f"{x[0]}@{x[1]}") else: enabled_repos.append(x[0]) @@ -3293,15 +3323,30 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): if is_switched: checkout_repos.append(f"{x[0]}@{x[1]}") + elif is_switched is None: + failed.append(f"{x[0]}@{x[1]}") for x in git_info.keys(): normalized_url = git_utils.normalize_url(x) cnr = unified_manager.repo_cnr_map.get(normalized_url) if cnr is not None: pack_id = cnr['id'] - res = await unified_manager.install_by_id(pack_id, 'nightly', instant_execution=True, no_deps=False, return_postinstall=False) + # the install script runs after the checkout, for the files of the pinned commit; + # a clone that cannot be checked out stays, so it runs for that one as well + try: + res = await unified_manager.install_by_id(pack_id, 'nightly', instant_execution=True, no_deps=False, return_postinstall=True) + except Exception as e: + logging.error(f"[ComfyUI-Manager] {e}") + res = ManagedResult('fail').fail(str(e)) if res.action == 'install-git' and res.result: - cloned_repos.append(pack_id) + is_switched = repo_switch_commit(unified_manager.active_nodes[pack_id][1], git_info[x]['hash']) + is_installed = run_postinstall(res) + if is_switched is None: + failed.append(f"{pack_id}@{git_info[x]['hash']}") + elif not is_installed: + failed.append(pack_id) + else: + cloned_repos.append(pack_id) elif res.action == 'skip': skip_node_packs.append(pack_id) elif not res.result: @@ -3319,6 +3364,9 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): processed_urls = [] for k2, v2 in unified_manager.unknown_active_nodes.items(): + if 'comfyui-manager' in k2.lower(): + continue + repo_url = resolve_giturl_from_path(v2[1]) if repo_url is None: @@ -3330,7 +3378,7 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): todo_disable.append(k2) else: commit_hash = git_info[normalized_url]['hash'] - todo_checkout.append((k2, commit_hash)) + todo_checkout.append((k2, v2[1], commit_hash)) processed_urls.append(normalized_url) for k2, v2 in unified_manager.unknown_inactive_nodes.items(): @@ -3347,28 +3395,36 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): processed_urls.append(normalized_url) for x in todo_disable: - unified_manager.unified_disable(x, True) - disabled_repos.append(x) + if move(unified_manager.unified_disable, x, True).result: + disabled_repos.append(x) + else: + failed.append(x) for x in todo_enable: - res = unified_manager.unified_enable(x[0], 'unknown') + res = move(unified_manager.unified_enable, x[0], 'unknown') is_switched = False - if res and res.target: + if res.result and res.target: is_switched = repo_switch_commit(res.target, x[1]) - if is_switched: + if not res.result: + failed.append(x[0]) + elif is_switched is None: + failed.append(f"{x[0]}@{x[1]}") + elif is_switched: checkout_repos.append(f"{x[0]}@{x[1]}") else: enabled_repos.append(x[0]) - for x in todo_checkout: - is_switched = repo_switch_commit(x[0], x[1]) + for name, path, commit_hash in todo_checkout: + is_switched = repo_switch_commit(path, commit_hash) if is_switched: - checkout_repos.append(f"{x[0]}@{x[1]}") + checkout_repos.append(f"{name}@{commit_hash}") + elif is_switched is None: + failed.append(f"{name}@{commit_hash}") else: - skip_node_packs.append(x[0]) + skip_node_packs.append(name) for x in processed_urls: if x in git_info: @@ -3380,8 +3436,19 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): repo_name = repo_name[:-4] to_path = os.path.join(get_default_custom_nodes_path(), repo_name) - unified_manager.repo_install(repo_url, to_path, instant_execution=True, no_deps=False, return_postinstall=False) - cloned_repos.append(repo_name) + res = unified_manager.repo_install(repo_url, to_path, instant_execution=True, no_deps=False, return_postinstall=True) + if not res.result: + failed.append(repo_name) + continue + + is_switched = repo_switch_commit(to_path, git_info[repo_url]['hash']) + is_installed = run_postinstall(res) + if is_switched is None: + failed.append(f"{repo_name}@{git_info[repo_url]['hash']}") + elif not is_installed: + failed.append(repo_name) + else: + cloned_repos.append(repo_name) manager_util.restore_pip_snapshot(pips, git_helper_extras) @@ -3400,7 +3467,7 @@ async def restore_snapshot(snapshot_path, git_helper_extras=None): print(f"[ DISABLED ] {x}") for x in skip_node_packs: print(f"[ SKIPPED ] {x}") - for x in failed: + for x in dict.fromkeys(failed): # a pack that failed at two steps is listed once print(f"[ FAILED ] {x}") # if is_failed: @@ -3514,23 +3581,31 @@ def resolve_giturl_from_path(fullpath): if not os.path.exists(git_config_path): return "unknown" - config = configparser.ConfigParser(strict=False) - config.read(git_config_path) + config = git_utils.read_git_config(git_config_path) + if config is None: + return None for k, v in config.items(): - if k.startswith('remote ') and 'url' in v: + if k.startswith('remote ') and v.get('url'): return v['url'].replace("git@github.com:", "https://github.com/") return None def repo_switch_commit(repo_path, commit_hash): + if commit_hash == 'unknown': + return False + try: repo = git.Repo(repo_path) - if repo.head.commit.hexsha == commit_hash: - return False + if repo.head.commit.hexsha != commit_hash: + repo.git.checkout(commit_hash) + repo.git.submodule('update', '--init', '--recursive') + return True - repo.git.checkout(commit_hash) - return True + # an update of the submodules that failed is done again; one that was never fetched is left alone + if os.path.exists(os.path.join(repo_path, '.gitmodules')): + repo.git.submodule('update', '--recursive') + return False except Exception: return None diff --git a/comfyui_manager/legacy/manager_server.py b/comfyui_manager/legacy/manager_server.py index 0b917a261b..cd23aa67c2 100644 --- a/comfyui_manager/legacy/manager_server.py +++ b/comfyui_manager/legacy/manager_server.py @@ -25,6 +25,7 @@ from ..common import manager_downloader from ..common import context from ..common import manager_security +from ..common.git_compat import setup_git_environment logging.info(f"### Loading: ComfyUI-Manager ({core.version_str})") @@ -270,18 +271,16 @@ def print_comfyui_version(): logging.info("### ComfyUI Revision: UNKNOWN (The currently installed ComfyUI is not a Git repository)") -print_comfyui_version() -core.check_invalid_nodes() - - def setup_environment(): git_exe = core.get_config()['git_exe'] if git_exe != '': - git.Git().update_environment(GIT_PYTHON_GIT_EXECUTABLE=git_exe) + setup_git_environment(git_exe) setup_environment() +print_comfyui_version() +core.check_invalid_nodes() # Expand Server api diff --git a/comfyui_manager/prestartup_script.py b/comfyui_manager/prestartup_script.py index 3c7c24320b..bf54ab64d9 100644 --- a/comfyui_manager/prestartup_script.py +++ b/comfyui_manager/prestartup_script.py @@ -101,7 +101,18 @@ def check_file_logging(): enable_file_logging = False +def set_git_executable(): + # git_compat reads this when it is imported + # read as written: a path of Windows may hold a `%`, which the parser takes as the start of a reference + git_exe = default_conf.get('git_exe', '', raw=True) if 'git_exe' in default_conf else '' + if git_exe != '': + os.environ['GIT_EXE_PATH'] = git_exe + + read_config() +set_git_executable() +# GitPython is imported here, with the git of `git_exe`, before any other module imports it +from .common import git_compat # noqa: E402,F401 read_uv_mode() read_unified_resolver_mode() security_check.security_check() @@ -588,7 +599,10 @@ def msg_capture(stream, prefix): allow_flagged or manager_security.is_loopback_listener(args.listen) ).lower() new_env['_COMFYUI_MANAGER_CNR_WHITELIST_DIR'] = manager_files_path - cmd_str = [sys.executable, '-m', 'cm_cli', 'restore-snapshot', restore_snapshot_path] + # cm_cli does not see the server's arguments, such as --base-directory; + # its --user-directory is the directory of the Manager's files + cmd_str = [sys.executable, '-m', 'cm_cli', 'restore-snapshot', restore_snapshot_path, + '--user-directory', manager_files_path, '--restore-to', custom_nodes_base_path] exit_code = process_wrap(cmd_str, custom_nodes_base_path, handler=msg_capture, env=new_env) if exit_code != 0: diff --git a/tests/_install_flags_testutil.py b/tests/_install_flags_testutil.py index 7f935e23d1..b6d19a414c 100644 --- a/tests/_install_flags_testutil.py +++ b/tests/_install_flags_testutil.py @@ -175,3 +175,9 @@ def import_glob_security_utils(): (the glob copy of the security_level gate matrix — guard rows SC-25C / SC-28).""" return _import_real("comfyui_manager.glob.utils.security_utils") + + +# what `git rev-parse --local-env-vars` lists: with one of them, git does not work on the repository of its directory +GIT_LOCATION = ('GIT_ALTERNATE_OBJECT_DIRECTORIES', 'GIT_CONFIG', 'GIT_CONFIG_PARAMETERS', 'GIT_CONFIG_COUNT', 'GIT_OBJECT_DIRECTORY', + 'GIT_DIR', 'GIT_WORK_TREE', 'GIT_IMPLICIT_WORK_TREE', 'GIT_GRAFT_FILE', 'GIT_INDEX_FILE', 'GIT_NO_REPLACE_OBJECTS', + 'GIT_REPLACE_REF_BASE', 'GIT_PREFIX', 'GIT_SHALLOW_FILE', 'GIT_COMMON_DIR') diff --git a/tests/test_git_config_url.py b/tests/test_git_config_url.py new file mode 100644 index 0000000000..202b424158 --- /dev/null +++ b/tests/test_git_config_url.py @@ -0,0 +1,112 @@ +"""The remote url of a node pack is read from any `.git/config` that git reads. + +The cases run the three real readers on a `.git/config` written to a temporary +directory: `git_url` of `common/git_utils.py`, and `resolve_giturl_from_path` +of the two `manager_core` modules. +""" +import configparser +import importlib.util +import logging +import os + +import pytest + +from legacy_html_testutil import PACKAGE, load_functions + +URL = "https://github.com/someone/some%20pack" +PLAIN = '[core]\n\tbare = false\n[remote "origin"]\n\turl = https://github.com/someone/pack\n' +PERCENT = f'[core]\n\tbare = false\n[remote "origin"]\n\turl = {URL}\n\tfetch = +refs/heads/*:refs/remotes/origin/*\n' +NO_VALUE = '[core]\n\tbare\n[remote "origin"]\n\turl = https://github.com/someone/pack\n' +NO_VALUE_THEN_DEEPER = '[core]\n\tbare\n\t\tfilemode = false\n[remote "origin"]\n\turl = https://github.com/someone/pack\n' +NO_REMOTE = '[core]\n\tbare = false\n' +BOM = "\ufeff" + PLAIN +NOT_A_CONFIG = "ref: refs/heads/master\n" + PLAIN +NOT_ASCII = PLAIN + '[user]\n\tname = 홍길동\n' +NOT_UTF8 = PLAIN.encode() + b'[user]\n\tname = \xff\xfe\n' + + +def git_utils(): + spec = importlib.util.spec_from_file_location("_git_utils", PACKAGE / "common/git_utils.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def git_url(): + return git_utils().git_url + + +def resolve(core): + return lambda: load_functions(PACKAGE / core / "manager_core.py", {"resolve_giturl_from_path"}, { + "os": os, "configparser": configparser, "logging": logging, "git_utils": git_utils(), + })["resolve_giturl_from_path"] + + +@pytest.fixture(params=[git_url, resolve("glob"), resolve("legacy")], ids=["git_url", "glob", "legacy"]) +def read(request, tmp_path): + reader = request.param() + + def read(config, mode=None): + (tmp_path / ".git").mkdir(exist_ok=True) + (tmp_path / ".git" / "config").write_bytes(config if isinstance(config, bytes) else config.encode()) + if mode is not None: + (tmp_path / ".git" / "config").chmod(mode) + return reader(str(tmp_path)) + + return read + + +def test_url_is_read(read): + assert read(PLAIN) == "https://github.com/someone/pack" + + +def test_url_with_a_percent_sign_is_read_as_it_is(read): + assert read(PERCENT) == URL + + +def test_url_is_read_beside_a_key_without_a_value(read): + assert read(NO_VALUE) == "https://github.com/someone/pack" + + +def test_config_without_a_remote_has_no_url(read): + assert read(NO_REMOTE) is None + + +def test_url_without_a_value_is_no_url(read): + assert read('[remote "origin"]\n\turl\n') is None + + +def test_url_is_read_after_a_byte_order_mark(read): + assert read(BOM) == "https://github.com/someone/pack" + + +def test_url_is_read_beside_text_that_is_not_ascii(read): + assert read(NOT_ASCII) == "https://github.com/someone/pack" + + +def test_url_is_read_beside_bytes_that_are_not_utf8(read): + assert read(NOT_UTF8) == "https://github.com/someone/pack" + + +def test_config_that_cannot_be_parsed_has_no_url_and_is_reported(read, caplog, tmp_path): + assert read(NOT_A_CONFIG) is None + assert str(tmp_path / ".git" / "config") in caplog.text + + +def test_key_without_a_value_before_a_deeper_line_has_no_url_and_is_reported(read, caplog, tmp_path): + """The parser takes the deeper line as more of the value, which the key does not have, and fails.""" + assert read(NO_VALUE_THEN_DEEPER) is None + assert str(tmp_path / ".git" / "config") in caplog.text + + +def test_config_that_cannot_be_parsed_is_reported_once(read, caplog, tmp_path): + for _ in range(3): + assert read(NOT_A_CONFIG) is None + + assert caplog.text.count(f"Cannot read '{tmp_path / '.git' / 'config'}'") == 1 + + +@pytest.mark.skipif(os.name == "nt" or os.geteuid() == 0, reason="needs a file the process may not read") +def test_config_that_may_not_be_read_has_no_url_and_is_reported(read, caplog, tmp_path): + assert read(PLAIN, mode=0) is None + assert str(tmp_path / ".git" / "config") in caplog.text diff --git a/tests/test_git_exe_setting.py b/tests/test_git_exe_setting.py new file mode 100644 index 0000000000..4093052966 --- /dev/null +++ b/tests/test_git_exe_setting.py @@ -0,0 +1,378 @@ +"""`git_exe` chooses the git executable that GitPython runs. + +`git_compat` learns the configured executable from `GIT_EXE_PATH` when it is +imported, so the cases about that import run it in a fresh process. The +configured executable is a script that records its arguments and then runs the +git of the test environment. +""" +import ast +import configparser +import importlib.util +import json +import os +import shutil +import subprocess +import sys +from types import SimpleNamespace + +import git +import pytest + +from _install_flags_testutil import GIT_LOCATION, import_context, import_reader +from legacy_html_testutil import PACKAGE, load_functions + +pytestmark = pytest.mark.skipif(os.name == "nt", reason="the executables of the cases are shell scripts") + +COMMON = PACKAGE / "common" +NOT_A_GIT = { + "a path that does not exist": None, + "a program that fails": "#!/bin/sh\nexit 3\n", + "a program that is not git": "#!/bin/sh\necho hello\n", + "a program that waits": "#!/bin/sh\nexec sleep 30\n", + "a program that prints bytes that are not text": "#!/bin/sh\nprintf '\\377\\376'\n", + "a file that cannot be run": "not a program\n", +} +IMPORT = """ +import json, sys +sys.path.insert(0, {common!r}) +import git_compat +git_compat.find_git_executable.__defaults__ = (2,) +used = None +if not git_compat.USE_PYGIT2: + import git + git.Repo.init({repo!r}) + used = git.Git.GIT_PYTHON_GIT_EXECUTABLE +print(json.dumps({{"pygit2": git_compat.USE_PYGIT2, "git": used}})) +""" + + +@pytest.fixture(autouse=True) +def git_finds_its_repository_by_the_directory(monkeypatch): + for name in GIT_LOCATION: + monkeypatch.delenv(name, raising=False) + + +@pytest.fixture +def recording_git(tmp_path): + """A git executable that appends each command line to `calls`.""" + calls = tmp_path / "calls" + executable = tmp_path / "recording-git" + executable.write_text(f'#!/bin/sh\necho "$@" >> "{calls}"\nexec "{shutil.which("git")}" "$@"\n') + executable.chmod(0o755) + return SimpleNamespace(path=str(executable), + calls=lambda: calls.read_text().splitlines() if calls.exists() else [], + forget=lambda: calls.unlink(missing_ok=True)) + + +@pytest.fixture(params=NOT_A_GIT) +def not_a_git(request, tmp_path): + path = tmp_path / "not-a-git" + if NOT_A_GIT[request.param] is not None: + path.write_text(NOT_A_GIT[request.param]) + path.chmod(0o755) + return str(path) + + +@pytest.fixture +def imported(tmp_path): + """Import `git_compat` in a fresh process: the backend, the git GitPython runs, and what was reported.""" + def imported(git_exe=None, git_on_path=True, pygit2=False): + env = {key: value for key, value in os.environ.items() + if key not in ("GIT_EXE_PATH", "GIT_PYTHON_GIT_EXECUTABLE", "GIT_PYTHON_REFRESH", "CM_USE_PYGIT2")} + if git_exe is not None: + env["GIT_EXE_PATH"] = git_exe + if not git_on_path: + (tmp_path / "no-git").mkdir() + env["PATH"] = str(tmp_path / "no-git") + if pygit2: + env["CM_USE_PYGIT2"] = "1" + code = IMPORT.format(common=str(COMMON), repo=str(tmp_path / "repo")) + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, env=env, timeout=60) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout.strip().splitlines()[-1]), result.stderr + + return imported + + +def test_configured_git_is_run(imported, recording_git): + assert imported(recording_git.path)[0] == {"pygit2": False, "git": recording_git.path} + assert any(line.startswith("init") for line in recording_git.calls()) + + +def test_configured_git_is_run_without_a_git_on_path(imported, recording_git): + assert imported(recording_git.path, git_on_path=False)[0] == {"pygit2": False, "git": recording_git.path} + assert any(line.startswith("init") for line in recording_git.calls()) + + +@pytest.mark.parametrize("git_exe", [None, ""]) +def test_no_setting_keeps_the_default_git(imported, git_exe): + state, reported = imported(git_exe) + + assert state == {"pygit2": False, "git": "git"} + assert "git_exe" not in reported + + +def test_setting_that_is_no_git_is_reported_and_the_default_kept(imported, not_a_git): + state, reported = imported(not_a_git) + + assert state == {"pygit2": False, "git": "git"} + assert not_a_git in reported + + +LATE = """ +import json, os, sys +import git # GitPython first, as the legacy modules import it +sys.path.insert(0, {common!r}) +import git_compat +git_compat.setup_git_environment(os.environ["GIT_EXE_PATH"]) +git.Repo.init({repo!r}) +print(json.dumps(git.Git.GIT_PYTHON_GIT_EXECUTABLE)) +""" + + +def test_configured_git_is_run_when_gitpython_was_imported_first(recording_git, tmp_path): + env = {key: value for key, value in os.environ.items() if key not in ("GIT_PYTHON_GIT_EXECUTABLE", "CM_USE_PYGIT2")} + env["GIT_EXE_PATH"] = recording_git.path + code = LATE.format(common=str(COMMON), repo=str(tmp_path / "repo")) + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, env=env, timeout=60) + + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout.strip().splitlines()[-1]) == recording_git.path + assert any(line.startswith("init") for line in recording_git.calls()) + + +def test_what_was_refused_at_the_import_is_not_selected_or_reported_again(tmp_path): + fails = tmp_path / "fails" + fails.write_text("#!/bin/sh\nexit 3\n") + fails.chmod(0o755) + env = {key: value for key, value in os.environ.items() if key not in ("GIT_PYTHON_GIT_EXECUTABLE", "CM_USE_PYGIT2")} + env["GIT_EXE_PATH"] = str(fails) + code = LATE.format(common=str(COMMON), repo=str(tmp_path / "repo")) + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, env=env, timeout=60) + + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout.strip().splitlines()[-1]) == "git" + assert result.stderr.count(str(fails)) == 1 + + +def test_git_compat_is_imported_without_any_git(tmp_path): + """Neither a git on PATH nor pygit2: Manager loads, and git fails where it is used, not at the import.""" + (tmp_path / "no-git").mkdir() + (tmp_path / "no-pygit2").mkdir() + (tmp_path / "no-pygit2" / "pygit2.py").write_text("raise ImportError('pygit2 is not installed')\n") + env = {key: value for key, value in os.environ.items() + if key not in ("GIT_EXE_PATH", "GIT_PYTHON_GIT_EXECUTABLE", "GIT_PYTHON_REFRESH", "CM_USE_PYGIT2")} + env["PATH"] = str(tmp_path / "no-git") + code = (f"import sys; sys.path[:0] = [{str(tmp_path / 'no-pygit2')!r}, {str(COMMON)!r}]\n" + "import git_compat, git\n" + "try:\n git.Git().version()\nexcept git.GitCommandNotFound:\n print('fails at use')\n") + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, env=env, timeout=60) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip().splitlines()[-1] == "fails at use" + assert "no git" in result.stderr.lower() + + +def test_gitpython_can_be_imported_beside_the_pygit2_backend_without_a_git(tmp_path): + """The legacy server imports GitPython itself; without a git that import is not to stop Manager.""" + pytest.importorskip("pygit2") + (tmp_path / "no-git").mkdir() + env = {key: value for key, value in os.environ.items() + if key not in ("GIT_EXE_PATH", "GIT_PYTHON_GIT_EXECUTABLE", "GIT_PYTHON_REFRESH", "CM_USE_PYGIT2")} + env["PATH"] = str(tmp_path / "no-git") + code = f"import sys; sys.path.insert(0, {str(COMMON)!r})\nimport git_compat, git\nprint(git_compat.USE_PYGIT2)\n" + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, env=env, timeout=60) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip().splitlines()[-1] == "True" + + +def test_configured_git_is_run_when_pygit2_is_asked_for_but_not_installed(recording_git, tmp_path): + """GitPython is the backend then, and the setting was not looked at before pygit2 was tried.""" + (tmp_path / "no-git").mkdir() + (tmp_path / "no-pygit2").mkdir() + (tmp_path / "no-pygit2" / "pygit2.py").write_text("raise ImportError('pygit2 is not installed')\n") + env = {key: value for key, value in os.environ.items() if key not in ("GIT_PYTHON_GIT_EXECUTABLE", "GIT_PYTHON_REFRESH")} + env.update(PATH=str(tmp_path / "no-git"), GIT_EXE_PATH=recording_git.path, CM_USE_PYGIT2="1") + code = (f"import json, os, sys; sys.path[:0] = [{str(tmp_path / 'no-pygit2')!r}, {str(COMMON)!r}]\n" + "import git_compat, git\n" + "git_compat.setup_git_environment(os.environ['GIT_EXE_PATH'])\n" + f"git.Repo.init({str(tmp_path / 'repo')!r})\n" + "print(json.dumps([git_compat.USE_PYGIT2, git.Git.GIT_PYTHON_GIT_EXECUTABLE]))\n") + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, env=env, timeout=60) + + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout.strip().splitlines()[-1]) == [False, recording_git.path] + assert any(line.startswith("init") for line in recording_git.calls()) + + +def test_gitpython_is_imported_by_prestartup_after_the_setting_is_handed_on(): + """Whatever imports GitPython later, the legacy server among them, finds it set up with the configured git.""" + body = ast.parse((PACKAGE / "prestartup_script.py").read_text(encoding="utf-8")).body + steps = [] + for node in body: + if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call) and ast.unparse(node.value.func) == "set_git_executable": + steps.append("setting") + elif isinstance(node, (ast.Import, ast.ImportFrom)) and any(name in ast.unparse(node) for name in ("git_compat", "context", "import git")): + steps.append("GitPython") + + assert steps == ["setting", "GitPython"] + + +def test_git_named_by_a_command_on_path_is_run_by_its_full_path(imported, git_compat, tmp_path, monkeypatch): + """GitPython takes a name without a directory as a file of the current directory.""" + assert imported("git")[0] == {"pygit2": False, "git": shutil.which("git")} + + monkeypatch.chdir(tmp_path) + git_compat.setup_git_environment("git") + + assert git.Git.GIT_PYTHON_GIT_EXECUTABLE == shutil.which("git") + + +def test_pygit2_backend_does_not_use_the_setting(imported, recording_git): + pytest.importorskip("pygit2") + + assert imported(recording_git.path, pygit2=True)[0] == {"pygit2": True, "git": None} + assert recording_git.calls() == [] + + +@pytest.fixture +def git_compat(monkeypatch): + """`common/git_compat.py` on the GitPython backend in this process, with the default git restored afterwards.""" + for name in ("CM_USE_PYGIT2", "GIT_EXE_PATH"): + monkeypatch.delenv(name, raising=False) + spec = importlib.util.spec_from_file_location("_git_compat_under_test", COMMON / "git_compat.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + assert not module.USE_PYGIT2 + find = module.find_git_executable + monkeypatch.setattr(module, "find_git_executable", lambda git_exe: find(git_exe, timeout=2)) + yield module + git.refresh() + + +def test_git_given_after_the_import_is_run(git_compat, recording_git, tmp_path): + git_compat.setup_git_environment(recording_git.path) + recording_git.forget() + + git.Repo.init(tmp_path / "repo") + + assert any(line.startswith("init") for line in recording_git.calls()) + + +@pytest.mark.parametrize("git_exe", ["", None]) +def test_nothing_given_after_the_import_keeps_the_default(git_compat, git_exe, capsys): + default = git.Git.GIT_PYTHON_GIT_EXECUTABLE + capsys.readouterr() + + git_compat.setup_git_environment(git_exe) + + assert capsys.readouterr().err == "" and git.Git.GIT_PYTHON_GIT_EXECUTABLE == default + + +def test_what_is_no_git_given_after_the_import_is_reported(git_compat, not_a_git, tmp_path, capsys): + default = git.Git.GIT_PYTHON_GIT_EXECUTABLE + flags = git.remote.FetchInfo._flag_map.copy() + + git_compat.setup_git_environment(not_a_git) + + assert not_a_git in capsys.readouterr().err + assert git.Git.GIT_PYTHON_GIT_EXECUTABLE == default and git.remote.FetchInfo._flag_map == flags + git.Repo.init(tmp_path / "repo") + assert (tmp_path / "repo" / ".git").is_dir() + + +PERCENT = "%LOCALAPPDATA%\\Git\\bin\\git.exe" + + +def set_git_executable(monkeypatch, git_exe): + """Run `set_git_executable` of `prestartup_script.py` on a `config.ini` with this `git_exe`; what it leaves in `GIT_EXE_PATH`.""" + # set and removed, so that the variable the function sets is removed after the case, also when there was none + monkeypatch.setenv("GIT_EXE_PATH", "") + monkeypatch.delenv("GIT_EXE_PATH") + config = configparser.ConfigParser(strict=False) + config.read_string("[default]\n" + ("" if git_exe is None else f"git_exe = {git_exe}\n")) + load_functions(PACKAGE / "prestartup_script.py", {"set_git_executable"}, {"os": os, "default_conf": config["default"]})["set_git_executable"]() + return os.environ.get("GIT_EXE_PATH") + + +@pytest.mark.parametrize("git_exe, given", [("/opt/git/bin/git", "/opt/git/bin/git"), (PERCENT, PERCENT), ("", None), (None, None)]) +def test_prestartup_hands_the_setting_to_git_compat(monkeypatch, git_exe, given): + assert set_git_executable(monkeypatch, git_exe) == given + + +def test_prestartup_without_a_configuration_file_sets_nothing(monkeypatch): + monkeypatch.setenv("GIT_EXE_PATH", "") + monkeypatch.delenv("GIT_EXE_PATH") + load_functions(PACKAGE / "prestartup_script.py", {"set_git_executable"}, {"os": os, "default_conf": {}})["set_git_executable"]() + + assert os.environ.get("GIT_EXE_PATH") is None + + +def test_prestartup_case_leaves_no_variable_behind(): + """Runs after the cases above, which make the function set the variable.""" + assert os.environ.get("GIT_EXE_PATH") is None + + +@pytest.mark.parametrize("core", ["glob", "legacy"]) +def test_setting_with_a_percent_sign_is_read_as_written_and_the_others_kept(core, monkeypatch, tmp_path): + """A path of Windows may hold a `%`, which the parser takes as the start of a reference.""" + (tmp_path / "config.ini").write_text(f"[default]\ngit_exe = {PERCENT}\ndb_mode = local\nnetwork_mode = offline\n") + monkeypatch.setattr(import_context(), "manager_config_path", str(tmp_path / "config.ini")) + + config = import_reader(core).read_config() + + assert (config["git_exe"], config["db_mode"], config["network_mode"]) == (PERCENT, "local", "offline") + + +@pytest.mark.parametrize("core", ["glob", "legacy"]) +def test_setting_saved_beside_a_git_exe_with_a_percent_sign_keeps_it(core, monkeypatch, tmp_path): + path = tmp_path / "config.ini" + path.write_text(f"[default]\ngit_exe = {PERCENT}\ndb_mode = local\n") + monkeypatch.setattr(import_context(), "manager_config_path", str(path)) + module = import_reader(core) + monkeypatch.setattr(module, "cached_config", None) + + module.get_config()["db_mode"] = "remote" + module.write_config() + + assert sorted(line for line in path.read_text().splitlines() if line.startswith(("git_exe", "db_mode"))) == [ + "db_mode = remote", f"git_exe = {PERCENT}"] + + +@pytest.mark.parametrize("where", ["legacy/manager_server.py", "glob/utils/environment_utils.py", "common/git_helper.py"]) +def test_callers_pass_the_setting_on(where): + passed = [] + load_functions(PACKAGE / where, {"setup_environment"}, { + "git_exe_path": "/opt/git/bin/git", "setup_git_environment": passed.append, + "core": SimpleNamespace(get_config=lambda: {"git_exe": "/opt/git/bin/git"}), + })["setup_environment"]() + + assert passed == ["/opt/git/bin/git"] + + +def start_up_calls(path): + """The calls a module makes when it is imported, in the order of the file.""" + body = ast.parse((PACKAGE / path).read_text(encoding="utf-8")).body + return [ast.unparse(node.value.func) for node in body if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call)] + + +def test_git_is_chosen_before_the_first_git_command_of_the_legacy_server(): + calls = [call for call in start_up_calls("legacy/manager_server.py") if call in ("setup_environment", "print_comfyui_version")] + + assert calls == ["setup_environment", "print_comfyui_version"] + + +def test_default_server_runs_its_start_up_setup(): + assert "environment_utils.initialize_environment" in start_up_calls("glob/manager_server.py") + + +def test_start_up_setup_chooses_git_reads_the_version_and_checks_the_nodes(): + steps = [] + load_functions(PACKAGE / "glob/utils/environment_utils.py", {"initialize_environment"}, { + "os": os, "folder_paths": SimpleNamespace(__file__="/comfy/folder_paths.py"), + "context": SimpleNamespace(comfy_path=None), "core": SimpleNamespace(check_invalid_nodes=lambda: steps.append("nodes")), + "setup_environment": lambda: steps.append("git"), "print_comfyui_version": lambda: steps.append("version"), + })["initialize_environment"]() + + assert steps == ["git", "version", "nodes"] diff --git a/tests/test_restore_summary.py b/tests/test_restore_summary.py new file mode 100644 index 0000000000..10fbcd440f --- /dev/null +++ b/tests/test_restore_summary.py @@ -0,0 +1,656 @@ +"""The summary of a snapshot restore lists a step that failed as failed. + +Both `manager_core` modules are run. What the restore asks the manager to do is replaced, +and answers with a success or a failure. +""" + +import os +import shutil +import subprocess +import sys +from unittest.mock import AsyncMock, Mock + +import git +import pytest + +from _install_flags_testutil import GIT_LOCATION +from test_snapshot_restore_pins import COMMIT, REPO_URL, core, manager, restore # noqa: F401 + + +@pytest.fixture(autouse=True) +def git_finds_its_repository_by_the_directory(monkeypatch): + for name in GIT_LOCATION: + monkeypatch.delenv(name, raising=False) + + +def pinned(url, commit=COMMIT): + return {'git_custom_nodes': {url: {'hash': commit, 'disabled': False}}} + + +@pytest.fixture +def summary(core, tmp_path, capsys): # noqa: F811 + def summary(**snapshot): + restore(core, tmp_path, **snapshot) + return [line.strip() for line in capsys.readouterr().out.splitlines() if line.startswith('[ ')] + + return summary + + +@pytest.fixture +def answer(core): # noqa: F811 + def answer(action, ok): + result = core.ManagedResult(action) + return Mock(return_value=result if ok else result.fail('failed')) + + return answer + + +def git_pack(path, url): + """A directory whose .git/config names `url`, as a git nodepack the Registry does not know.""" + (path / '.git').mkdir(parents=True) + (path / '.git/config').write_text('[remote "origin"]\n\turl = %s\n' % url) + return url, str(path) + + +def head(path): + return subprocess.run(['git', 'rev-parse', 'HEAD'], cwd=path, check=True, capture_output=True, text=True).stdout.strip() + + +def repository(path): + """A git repository with two commits at `path`; their hashes.""" + path.mkdir(parents=True) + commits = [] + for version in (1, 2): + (path / '__init__.py').write_text('VERSION = %d' % version) + for args in (['init', '-q'], ['add', '.'], + ['-c', 'user.name=test', '-c', 'user.email=test@example.invalid', 'commit', '-q', '-m', 'version %d' % version]): + subprocess.run(['git', *args], cwd=path, check=True, capture_output=True) + commits.append(head(path)) + return commits + + +@pytest.mark.parametrize('cloned, line', [(True, '[ INSTALLED ] fixture'), (False, '[ FAILED ] fixture')]) +def test_clone_follows_its_result(core, manager, summary, answer, monkeypatch, tmp_path, cloned, line): # noqa: F811 + manager.repo_install = answer('install-git', cloned) + monkeypatch.setattr(core, 'repo_switch_commit', Mock(return_value=True)) + + assert summary(**pinned(REPO_URL + '.git')) == [line] + assert manager.repo_install.call_args.args[1] == str(tmp_path / 'fixture') + + +def test_url_that_is_not_cloned_is_reported_as_failed(manager, summary, tmp_path): # noqa: F811 + assert summary(**pinned('/home/someone/repos/fixture')) == ['[ FAILED ] fixture'] + assert not (tmp_path / 'fixture').exists() + + +def test_cloned_pack_is_checked_out_at_the_pinned_commit(core, manager, summary, tmp_path): # noqa: F811 + commits = repository(tmp_path / 'origin') + + def clone(url, path, **options): + subprocess.run(['git', 'clone', '-q', str(tmp_path / 'origin'), path], check=True, capture_output=True) + return core.ManagedResult('install-git') + + manager.repo_install = clone + + assert summary(**pinned(REPO_URL, commits[0])) == ['[ INSTALLED ] fixture'] + assert head(tmp_path / 'fixture') == commits[0] + assert (tmp_path / 'fixture/__init__.py').read_text() == 'VERSION = 1' + + +def test_cloned_registry_pack_is_checked_out_at_the_pinned_commit(core, manager, summary, tmp_path): # noqa: F811 + commits = repository(tmp_path / 'origin') + + async def install(pack_id, version_spec, **options): + subprocess.run(['git', 'clone', '-q', str(tmp_path / 'origin'), tmp_path / pack_id], check=True, capture_output=True) + manager.active_nodes[pack_id] = (version_spec, str(tmp_path / pack_id)) + return core.ManagedResult('install-git') + + manager.repo_cnr_map[REPO_URL] = {'id': 'fixture'} + manager.install_by_id = AsyncMock(side_effect=install) + + assert summary(**pinned(REPO_URL, commits[0])) == ['[ INSTALLED ] fixture'] + assert head(tmp_path / 'fixture') == commits[0] + + +def test_cloned_pack_that_cannot_be_checked_out_is_reported_as_failed(manager, summary, answer): # noqa: F811 + manager.repo_install = answer('install-git', True) + + assert summary(**pinned(REPO_URL)) == [f'[ FAILED ] fixture@{COMMIT}'] + + +@pytest.mark.parametrize('disabled, line', [(True, '[ DISABLED ] fixture'), (False, '[ FAILED ] fixture')]) +def test_disabling_a_git_pack_that_is_not_in_the_snapshot_follows_its_result(manager, summary, answer, tmp_path, disabled, line): # noqa: F811 + manager.unknown_active_nodes['fixture'] = git_pack(tmp_path / 'fixture', REPO_URL) + manager.unified_disable = answer('disable', disabled) + + assert summary() == [line] + + +@pytest.mark.parametrize('disabled, line', [(True, '[ DISABLED ] fixture'), (False, '[ FAILED ] fixture')]) +def test_disabling_a_registry_pack_that_is_not_in_the_snapshot_follows_its_result(manager, summary, answer, tmp_path, disabled, line): # noqa: F811 + manager.active_nodes['fixture'] = ('1.0.0', str(tmp_path / 'fixture')) + manager.unified_disable = answer('disable', disabled) + + assert summary() == [line] + + +@pytest.mark.parametrize('name', ['ComfyUI-Manager', 'comfyui-manager']) +def test_the_manager_itself_is_not_listed(manager, summary, answer, tmp_path, name): # noqa: F811 + manager.unknown_active_nodes[name] = git_pack(tmp_path / name, 'https://github.com/ltdrdata/ComfyUI-Manager') + manager.unified_disable = answer('disable', True) + + assert summary() == [] + assert manager.unified_disable.call_count == 0 + + +@pytest.mark.parametrize('enabled, line', [(True, '[ ENABLED ] fixture'), (False, '[ FAILED ] fixture')]) +def test_enabling_a_git_pack_of_the_snapshot_follows_its_result(manager, summary, answer, tmp_path, enabled, line): # noqa: F811 + manager.unknown_inactive_nodes['fixture'] = git_pack(tmp_path / '.disabled/fixture', REPO_URL) + manager.unified_enable = answer('enable', enabled) + + assert summary(**pinned(REPO_URL)) == [line] + + +def test_installed_git_pack_the_registry_does_not_know_is_checked_out_at_the_pinned_commit(manager, summary, tmp_path): # noqa: F811 + commits = repository(tmp_path / 'fixture') + subprocess.run(['git', 'remote', 'add', 'origin', REPO_URL], cwd=tmp_path / 'fixture', check=True) + manager.unknown_active_nodes['fixture'] = (REPO_URL, str(tmp_path / 'fixture')) + + assert summary(**pinned(REPO_URL, commits[0])) == [f'[ CHECKOUT ] fixture@{commits[0]}'] + assert head(tmp_path / 'fixture') == commits[0] + + +def test_checkout_that_fails_is_reported_as_failed(manager, summary, tmp_path): # noqa: F811 + manager.unknown_active_nodes['fixture'] = git_pack(tmp_path / 'fixture', REPO_URL) + + assert summary(**pinned(REPO_URL)) == [f'[ FAILED ] fixture@{COMMIT}'] + + +def test_pack_without_a_commit_in_the_snapshot_is_left_where_it_is(manager, summary, tmp_path): # noqa: F811 + commits = repository(tmp_path / 'fixture') + subprocess.run(['git', 'remote', 'add', 'origin', REPO_URL], cwd=tmp_path / 'fixture', check=True) + manager.unknown_active_nodes['fixture'] = (REPO_URL, str(tmp_path / 'fixture')) + + assert summary(**pinned(REPO_URL, 'unknown')) == ['[ SKIPPED ] fixture'] + assert head(tmp_path / 'fixture') == commits[1] + + +def test_cloned_pack_without_a_commit_in_the_snapshot_is_installed(manager, summary, answer): # noqa: F811 + manager.repo_install = answer('install-git', True) + + assert summary(**pinned(REPO_URL, 'unknown')) == ['[ INSTALLED ] fixture'] + + +@pytest.mark.parametrize('installed, line', [(True, '[ INSTALLED ] fixture'), (False, '[ FAILED ] fixture')]) +def test_install_script_of_a_cloned_pack_runs_at_the_pinned_commit(core, manager, summary, tmp_path, installed, line): # noqa: F811 + commits = repository(tmp_path / 'origin') + ran_at = [] + + def clone(url, path, return_postinstall=False, **options): + subprocess.run(['git', 'clone', '-q', str(tmp_path / 'origin'), path], check=True, capture_output=True) + + def postinstall(): + ran_at.append(head(path)) + return installed + + result = core.ManagedResult('install-git') + return result.with_postinstall(postinstall) if return_postinstall else (result if postinstall() else result.fail('install script')) + + manager.repo_install = clone + + assert summary(**pinned(REPO_URL, commits[0])) == [line] + assert ran_at == [commits[0]] + + +def test_commit_of_a_pack_whose_refs_are_packed_is_read(core, tmp_path): # noqa: F811 + commits = repository(tmp_path / 'fixture') + subprocess.run(['git', 'pack-refs', '--all'], cwd=tmp_path / 'fixture', check=True) + with open(tmp_path / 'fixture/.git/packed-refs', 'a') as packed: + packed.write('\n') + branch = (tmp_path / 'fixture/.git/HEAD').read_text().strip()[5:] + + assert not (tmp_path / 'fixture/.git' / branch).exists() + assert core.git_utils.get_commit_hash(str(tmp_path / 'fixture')) == commits[1] + + +def test_commit_is_read_beside_refs_that_are_not_utf8(core, tmp_path): # noqa: F811 + commits = repository(tmp_path / 'fixture') + subprocess.run(['git', 'pack-refs', '--all'], cwd=tmp_path / 'fixture', check=True) + packed = tmp_path / 'fixture/.git/packed-refs' + packed.write_bytes(b'0123456789abcdef0123456789abcdef01234567 refs/heads/\xff\xfe\n' + packed.read_bytes()) + + assert core.git_utils.get_commit_hash(str(tmp_path / 'fixture')) == commits[1] + + +def test_head_that_is_not_utf8_gives_no_commit(core, tmp_path): # noqa: F811 + (tmp_path / 'fixture/.git').mkdir(parents=True) + (tmp_path / 'fixture/.git/HEAD').write_bytes(b'ref: refs/heads/\xff\xfe\n') + + assert core.git_utils.get_commit_hash(str(tmp_path / 'fixture')) == 'unknown' + + +def test_packed_ref_is_matched_by_its_bytes(core, tmp_path): # noqa: F811 + commits = repository(tmp_path / 'fixture') + subprocess.run(['git', 'pack-refs', '--all'], cwd=tmp_path / 'fixture', check=True) + git_dir = tmp_path / 'fixture/.git' + other = b'0123456789abcdef0123456789abcdef01234567 refs/heads/release-\xff\n' + (git_dir / 'packed-refs').write_bytes(other + (git_dir / 'packed-refs').read_bytes()) + (git_dir / 'HEAD').write_bytes(b'ref: refs/heads/release-\xfe\n') + + assert core.git_utils.get_commit_hash(str(tmp_path / 'fixture')) == 'unknown' + + (git_dir / 'packed-refs').write_bytes(other + commits[1].encode() + b' refs/heads/release-\xfe\n') + + assert core.git_utils.get_commit_hash(str(tmp_path / 'fixture')) == commits[1] + + +def test_install_script_that_raises_is_reported_as_failed_and_the_restore_goes_on(core, manager, summary, monkeypatch): # noqa: F811 + def clone(url, path, return_postinstall=False, **options): + def postinstall(): + raise subprocess.CalledProcessError(1, ['python', 'install.py']) + + return core.ManagedResult('install-git').with_postinstall(postinstall) + + manager.repo_install = clone + monkeypatch.setattr(core, 'repo_switch_commit', Mock(return_value=True)) + + assert summary(git_custom_nodes={REPO_URL: {'hash': COMMIT, 'disabled': False}, + REPO_URL + '-other': {'hash': COMMIT, 'disabled': False}}) == ['[ FAILED ] fixture', '[ FAILED ] fixture-other'] + + +def test_install_script_of_a_cloned_pack_runs_also_when_the_commit_cannot_be_checked_out(core, manager, summary, tmp_path): # noqa: F811 + """The clone stays where it is, and a later restore only checks it out.""" + ran = [] + manager.repo_install = lambda url, path, **options: core.ManagedResult('install-git').with_postinstall(lambda: ran.append(path) or True) + + assert summary(**pinned(REPO_URL)) == [f'[ FAILED ] fixture@{COMMIT}'] + assert ran == [str(tmp_path / 'fixture')] + + +def test_install_script_of_a_cloned_registry_pack_runs_also_when_the_commit_cannot_be_checked_out(core, manager, summary, tmp_path): # noqa: F811 + ran = [] + + async def install(pack_id, version_spec, **options): + manager.active_nodes[pack_id] = (version_spec, str(tmp_path / pack_id)) + return core.ManagedResult('install-git').with_postinstall(lambda: ran.append(pack_id) or True) + + manager.repo_cnr_map[REPO_URL] = {'id': 'fixture'} + manager.install_by_id = AsyncMock(side_effect=install) + + assert summary(**pinned(REPO_URL)) == [f'[ FAILED ] fixture@{COMMIT}'] + assert ran == ['fixture'] + + +@pytest.mark.parametrize('outcome, line', [(True, '[ INSTALLED ] fixture@1.0.0'), (False, '[ FAILED ] fixture@1.0.0'), + (subprocess.CalledProcessError(1, ['python', 'install.py']), '[ FAILED ] fixture@1.0.0')]) +def test_install_script_of_a_registry_pack_is_run(core, manager, summary, outcome, line): # noqa: F811 + ran = [] + + def postinstall(): + ran.append('fixture') + if isinstance(outcome, Exception): + raise outcome + return outcome + + manager.install_by_id = AsyncMock(return_value=core.ManagedResult('install-cnr').with_postinstall(postinstall)) + + assert summary(cnr_custom_nodes={'fixture': '1.0.0'}) == [line] + assert ran == ['fixture'] + assert manager.install_by_id.await_args.kwargs['return_postinstall'] is True + + +def test_git_of_a_case_does_not_inherit_a_repository(): + """Run from a git hook, the tests inherit variables such as GIT_DIR, which point git at the repository of the hook.""" + names = subprocess.run(['git', 'rev-parse', '--local-env-vars'], check=True, capture_output=True, text=True).stdout.split() + case = f'{__file__}::test_cloned_pack_is_checked_out_at_the_pinned_commit' + child = subprocess.run([sys.executable, '-m', 'pytest', '-q', '-p', 'no:cacheprovider', case], + env={**os.environ, **dict.fromkeys(names, 'elsewhere')}, capture_output=True, text=True) + + assert child.returncode == 0, child.stdout + + +def git_with_local_submodules(path, *args): + subprocess.run(['git', '-c', 'protocol.file.allow=always', '-c', 'user.name=test', '-c', 'user.email=test@example.invalid', *args], + cwd=path, check=True, capture_output=True) + + +def repository_with_a_submodule(tmp_path): + """A repository `origin` whose two commits pin the two commits of its submodule `library`; the hashes of both.""" + library = repository(tmp_path / 'library') + origin = tmp_path / 'origin' + origin.mkdir() + git_with_local_submodules(origin, 'init', '-q') + git_with_local_submodules(origin, 'submodule', 'add', '-q', str(tmp_path / 'library'), 'library') + pinned_commits = [] + for commit in library: + git_with_local_submodules(origin / 'library', 'checkout', '-q', commit) + git_with_local_submodules(origin, 'commit', '-q', '-a', '-m', 'library at ' + commit) + pinned_commits.append(head(origin)) + return library, pinned_commits + + +def test_submodules_of_a_cloned_pack_follow_the_pinned_commit(core, manager, summary, tmp_path): # noqa: F811 + library, pinned_commits = repository_with_a_submodule(tmp_path) + + def clone(url, path, **options): + git_with_local_submodules(tmp_path, 'clone', '-q', '--recursive', str(tmp_path / 'origin'), path) + return core.ManagedResult('install-git') + + manager.repo_install = clone + + assert summary(**pinned(REPO_URL, pinned_commits[0])) == ['[ INSTALLED ] fixture'] + assert head(tmp_path / 'fixture') == pinned_commits[0] + assert head(tmp_path / 'fixture/library') == library[0] + + +def test_submodules_are_brought_to_the_commit_of_a_pack_that_is_at_it_already(core, tmp_path): # noqa: F811 + """As after a restore whose checkout succeeded and whose update of the submodules failed.""" + library, pinned_commits = repository_with_a_submodule(tmp_path) + git_with_local_submodules(tmp_path, 'clone', '-q', '--recursive', str(tmp_path / 'origin'), 'fixture') + git_with_local_submodules(tmp_path / 'fixture/library', 'checkout', '-q', library[0]) + + assert core.repo_switch_commit(str(tmp_path / 'fixture'), pinned_commits[1]) is False + assert head(tmp_path / 'fixture/library') == library[1] + + +def test_submodule_that_was_never_fetched_is_left_alone_in_a_pack_that_is_at_the_commit(core, tmp_path): # noqa: F811 + """A snapshot does not hold the state of the submodules; one that cannot be reached is not a failure of the pack.""" + library, pinned_commits = repository_with_a_submodule(tmp_path) + git_with_local_submodules(tmp_path, 'clone', '-q', str(tmp_path / 'origin'), 'fixture') + shutil.rmtree(tmp_path / 'library') + + assert core.repo_switch_commit(str(tmp_path / 'fixture'), pinned_commits[1]) is False + assert list((tmp_path / 'fixture/library').iterdir()) == [] + + +def test_registry_pack_whose_install_raises_is_reported_as_failed_and_the_restore_goes_on(core, manager, summary): # noqa: F811 + """Without a connection the request to the Registry raises.""" + async def install(pack_id, **options): + if pack_id == 'fixture': + raise ConnectionError('the Registry cannot be reached') + return core.ManagedResult('install-cnr') + + manager.install_by_id = AsyncMock(side_effect=install) + + assert summary(cnr_custom_nodes={'fixture': '1.0.0', 'other': '1.0.0'}) == ['[ INSTALLED ] other@1.0.0', '[ FAILED ] fixture@1.0.0'] + + +def test_registry_pack_whose_switch_raises_is_reported_as_failed_and_the_restore_goes_on(core, manager, summary, tmp_path): # noqa: F811 + manager.active_nodes['fixture'] = ('1.0.0', str(tmp_path / 'fixture')) + manager.cnr_switch_version = Mock(side_effect=ConnectionError('the Registry cannot be reached')) + manager.install_by_id = AsyncMock(return_value=core.ManagedResult('install-cnr')) + + assert summary(cnr_custom_nodes={'fixture': '2.0.0', 'other': '1.0.0'}) == ['[ INSTALLED ] other@1.0.0', '[ FAILED ] fixture@2.0.0'] + + +def registry_pack(manager, path): # noqa: F811 + manager.active_nodes['fixture'] = ('1.0.0', str(path / 'fixture')) + + +def registry_git_pack(manager, path): # noqa: F811 + manager.active_nodes['fixture'] = ('nightly', str(path / 'fixture')) + manager.repo_cnr_map[REPO_URL] = {'id': 'fixture'} + + +def disabled_registry_git_pack(manager, path): # noqa: F811 + manager.nightly_inactive_nodes['fixture'] = str(path / '.disabled/fixture') + manager.repo_cnr_map[REPO_URL] = {'id': 'fixture'} + + +def unknown_git_pack(manager, path): # noqa: F811 + manager.unknown_active_nodes['fixture'] = git_pack(path / 'fixture', REPO_URL) + + +def disabled_unknown_git_pack(manager, path): # noqa: F811 + manager.unknown_inactive_nodes['fixture'] = git_pack(path / '.disabled/fixture', REPO_URL) + + +@pytest.mark.parametrize('installed, step, snapshot', [ + (registry_pack, 'unified_disable', {}), + (registry_git_pack, 'unified_disable', {}), + (unknown_git_pack, 'unified_disable', {}), + (disabled_registry_git_pack, 'unified_enable', pinned(REPO_URL)), + (disabled_unknown_git_pack, 'unified_enable', pinned(REPO_URL)), +]) +def test_pack_whose_directory_cannot_be_moved_is_reported_as_failed_and_the_restore_goes_on(manager, summary, tmp_path, installed, step, snapshot): # noqa: F811 + """Disabling and enabling move the directory of the pack.""" + installed(manager, tmp_path) + setattr(manager, step, Mock(side_effect=PermissionError(13, 'Permission denied'))) + + assert summary(**snapshot) == ['[ FAILED ] fixture'] + assert getattr(manager, step).call_count == 1 + + +def test_registry_git_pack_that_cannot_be_enabled_is_listed_once(manager, summary, tmp_path): # noqa: F811 + """The install that follows for a pack of the snapshot enables it as well, and raises for the same reason.""" + disabled_registry_git_pack(manager, tmp_path) + manager.unified_enable = Mock(side_effect=PermissionError(13, 'Permission denied')) + manager.install_by_id = AsyncMock(side_effect=PermissionError(13, 'Permission denied')) + + assert summary(**pinned(REPO_URL)) == ['[ FAILED ] fixture'] + + +def test_git_is_not_run_for_the_submodules_of_a_pack_that_has_none(core, tmp_path, monkeypatch): # noqa: F811 + """A restore goes through every git pack; on Windows each git command takes about half a second.""" + commits = repository(tmp_path / 'fixture') + ran = [] + execute = git.Git.execute + + def recorded(self, command, *args, **kwargs): + ran.append(list(command)) + return execute(self, command, *args, **kwargs) + + monkeypatch.setattr(git.Git, 'execute', recorded) + + assert core.repo_switch_commit(str(tmp_path / 'fixture'), commits[1]) is False + assert [command for command in ran if 'submodule' in command] == [] + + +def update_submodules_with_pygit2(tmp_path, clone, init=True, first=''): + """In a process of its own, with no git executable to fall back on.""" + (tmp_path / 'no-git').mkdir(exist_ok=True) + common = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'comfyui_manager', 'common') + code = (f'import sys; sys.path.insert(0, {common!r}); import git_compat\n{first}\n' + f'assert git_compat.USE_PYGIT2\ngit_compat.open_repo({str(clone)!r}).submodule_update(init={init})\n') + env = {key: value for key, value in os.environ.items() if key not in ('GIT_EXE_PATH', 'GIT_PYTHON_GIT_EXECUTABLE')} + return subprocess.run([sys.executable, '-c', code], env={**env, 'PATH': str(tmp_path / 'no-git'), 'CM_USE_PYGIT2': '1'}, + capture_output=True, text=True) + + +def test_pygit2_updates_the_fetched_submodules_beside_one_never_fetched(tmp_path): + """Without a git executable there is no fallback: pygit2 refuses a submodule that was never fetched.""" + pytest.importorskip('pygit2') + library = repository(tmp_path / 'library') + origin = tmp_path / 'origin' + origin.mkdir() + git_with_local_submodules(origin, 'init', '-q') + for name in ('a-never-fetched', 'b-fetched'): + git_with_local_submodules(origin, 'submodule', 'add', '-q', str(tmp_path / 'library'), name) + git_with_local_submodules(origin, 'commit', '-q', '-a', '-m', 'two submodules') + clone = tmp_path / 'clone' + git_with_local_submodules(tmp_path, 'clone', '-q', str(origin), str(clone)) + git_with_local_submodules(clone, 'submodule', 'update', '-q', '--init', 'b-fetched') + git_with_local_submodules(clone / 'b-fetched', 'checkout', '-q', library[0]) + + child = update_submodules_with_pygit2(tmp_path, clone, init=False) + + assert child.returncode == 0, child.stderr + assert head(clone / 'b-fetched') == library[1] + assert os.listdir(clone / 'a-never-fetched') == [] + + +def test_pygit2_updates_a_submodule_of_a_submodule(tmp_path): + """As `git submodule update --init --recursive` does, which the other backend runs.""" + pytest.importorskip('pygit2') + library, _ = repository_with_a_submodule(tmp_path) + outer = tmp_path / 'outer' + outer.mkdir() + git_with_local_submodules(outer, 'init', '-q') + git_with_local_submodules(outer, 'submodule', 'add', '-q', str(tmp_path / 'origin'), 'origin') + git_with_local_submodules(outer, 'commit', '-q', '-a', '-m', 'a submodule with a submodule') + clone = tmp_path / 'clone' + git_with_local_submodules(tmp_path, 'clone', '-q', str(outer), str(clone)) + + child = update_submodules_with_pygit2(tmp_path, clone) + + assert child.returncode == 0, child.stderr + assert (clone / 'origin/library/__init__.py').exists() + assert head(clone / 'origin/library') == library[1] + + git_with_local_submodules(clone / 'origin/library', 'checkout', '-q', library[0]) + child = update_submodules_with_pygit2(tmp_path, clone) + + assert child.returncode == 0, child.stderr + assert head(clone / 'origin/library') == library[1] + + +def test_pygit2_passes_over_a_submodule_that_gitmodules_alone_names(tmp_path): + """As git does: an entry left in `.gitmodules` for a submodule that was removed is not a failure.""" + pytest.importorskip('pygit2') + library, _ = repository_with_a_submodule(tmp_path) + with open(tmp_path / 'origin/.gitmodules', 'a') as gitmodules: + gitmodules.write(f'[submodule "a-removed"]\n\tpath = a-removed\n\turl = {tmp_path / "nowhere"}\n') + git_with_local_submodules(tmp_path / 'origin', 'commit', '-q', '-a', '-m', 'an entry without a submodule') + clone = tmp_path / 'clone' + git_with_local_submodules(tmp_path, 'clone', '-q', str(tmp_path / 'origin'), str(clone)) + + child = update_submodules_with_pygit2(tmp_path, clone) + + assert child.returncode == 0, child.stderr + assert head(clone / 'library') == library[1] + + +def add_repository_without_gitmodules(tmp_path, origin): + """A repository committed inside `origin` by `git add`: a commit in the tree that `.gitmodules` does not name.""" + repository(origin / 'stray') + git_with_local_submodules(origin, 'add', 'stray') + git_with_local_submodules(origin, 'commit', '-q', '-m', 'a repository added without `git submodule`') + clone = tmp_path / 'clone' + git_with_local_submodules(tmp_path, 'clone', '-q', str(origin), str(clone)) + return clone + + +def test_pygit2_passes_over_a_commit_in_the_tree_without_gitmodules(tmp_path): + """As `git clone --recursive` does: there is no url to fetch it from, and the nodepack is installed.""" + pytest.importorskip('pygit2') + repository(tmp_path / 'origin') + clone = add_repository_without_gitmodules(tmp_path, tmp_path / 'origin') + + child = update_submodules_with_pygit2(tmp_path, clone) + + assert child.returncode == 0, child.stderr + assert os.listdir(clone / 'stray') == [] + + +def test_pygit2_updates_the_submodules_beside_a_commit_that_gitmodules_does_not_name(tmp_path): + pytest.importorskip('pygit2') + library, _ = repository_with_a_submodule(tmp_path) + clone = add_repository_without_gitmodules(tmp_path, tmp_path / 'origin') + + child = update_submodules_with_pygit2(tmp_path, clone) + + assert child.returncode == 0, child.stderr + assert head(clone / 'library') == library[1] + + +def test_pygit2_does_not_pass_over_a_submodule_it_could_not_fetch(tmp_path): + """Without a git executable nothing else does the update: the caller must not take it as done.""" + pytest.importorskip('pygit2') + repository_with_a_submodule(tmp_path) + clone = tmp_path / 'clone' + git_with_local_submodules(tmp_path, 'clone', '-q', str(tmp_path / 'origin'), str(clone)) + shutil.rmtree(tmp_path / 'library') + + child = update_submodules_with_pygit2(tmp_path, clone) + + assert child.returncode != 0, child.stderr + assert 'GitCommandError' in child.stderr + + +def test_pygit2_does_not_pass_over_a_git_that_did_not_come_back(tmp_path): + pytest.importorskip('pygit2') + repository_with_a_submodule(tmp_path) + clone = tmp_path / 'clone' + git_with_local_submodules(tmp_path, 'clone', '-q', str(tmp_path / 'origin'), str(clone)) + shutil.rmtree(tmp_path / 'library') + late = ('import subprocess\n' + 'def late(command, timeout, **_):\n' + ' raise subprocess.TimeoutExpired(command, timeout)\n' + 'subprocess.run = late') + + child = update_submodules_with_pygit2(tmp_path, clone, first=late) + + assert child.returncode != 0, child.stderr + assert 'TimeoutExpired' in child.stderr + + +def packs_that_name(tmp_path, **files): + for name in ('first', 'second'): + (tmp_path / name).mkdir() + for file, text in files.items(): + (tmp_path / name / file).write_text(text) + return str(tmp_path / 'first'), str(tmp_path / 'second') + + +@pytest.fixture +def installs(core, monkeypatch): # noqa: F811 + """Replace what runs an install command; the answers are given in turn, and the directories it ran in are returned.""" + def installs(*answers): + answers, ran = list(answers), [] + + def try_install_script(url, repo_path, install_cmd, instant_execution=False): + ran.append(os.path.basename(repo_path)) + return answers.pop(0) + + monkeypatch.setattr(core, 'try_install_script', try_install_script) + monkeypatch.setattr(core.manager_util, 'PIPFixer', Mock()) + monkeypatch.setattr(core.manager_util, 'get_installed_packages', Mock(return_value={})) + monkeypatch.setattr(core.cm_global, 'pip_overrides', {}, raising=False) + return ran + + return installs + + +def test_requirement_that_was_not_installed_is_tried_for_the_next_pack(manager, installs, tmp_path): # noqa: F811 + first, second = packs_that_name(tmp_path, **{'requirements.txt': 'shareddep==1\n'}) + ran = installs(False, True) + + assert not manager.execute_install_script(None, first) + assert manager.execute_install_script(None, second) + assert ran == ['first', 'second'] + + +def test_requirement_after_one_that_failed_is_tried_for_the_next_pack(manager, installs, tmp_path): # noqa: F811 + first, second = packs_that_name(tmp_path, **{'requirements.txt': 'fails==1\nshareddep==1\n'}) + ran = installs(False, True, True) + + assert not manager.execute_install_script(None, first) + assert manager.execute_install_script(None, second) + assert ran == ['first', 'second', 'second'] + + +def test_requirement_that_was_installed_is_not_installed_twice(manager, installs, tmp_path): # noqa: F811 + first, second = packs_that_name(tmp_path, **{'requirements.txt': 'shareddep==1\n'}) + ran = installs(True) + + assert manager.execute_install_script(None, first) + assert manager.execute_install_script(None, second) + assert ran == ['first'] + + +def test_install_script_that_failed_is_run_when_asked_again(manager, installs, tmp_path): # noqa: F811 + first, _ = packs_that_name(tmp_path, **{'install.py': ''}) + ran = installs(False, True) + + assert not manager.execute_install_script(None, first) + assert manager.execute_install_script(None, first) + assert ran == ['first', 'first'] + + +def test_install_script_not_run_after_a_requirement_failed_is_run_when_asked_again(manager, installs, tmp_path): # noqa: F811 + first, _ = packs_that_name(tmp_path, **{'requirements.txt': 'fails==1\n', 'install.py': ''}) + ran = installs(False, True, True) + + assert not manager.execute_install_script(None, first) + assert manager.execute_install_script(None, first) + assert ran == ['first', 'first', 'first'] diff --git a/tests/test_scheduled_restore_command.py b/tests/test_scheduled_restore_command.py new file mode 100644 index 0000000000..bd6d713c9d --- /dev/null +++ b/tests/test_scheduled_restore_command.py @@ -0,0 +1,71 @@ +"""The scheduled snapshot restore tells cm_cli where the server's directories are. + +The cases run the real restore block of `comfyui_manager/prestartup_script.py`; `process_wrap` +is replaced and records the command it is given instead of running it. +""" +import ast +import os +import sys +import types +from pathlib import Path + +import pytest + +PRESTARTUP = Path(__file__).resolve().parent.parent / "comfyui_manager" / "prestartup_script.py" +COMFY = os.path.join(os.sep, "opt", "ComfyUI") +BASE = os.path.join(os.sep, "data", "comfy") + + +def restore(tmp_path, monkeypatch, environ): + """The command, working directory and environment the block starts cm-cli with.""" + scheduled = tmp_path / "restore-snapshot.json" + scheduled.write_text("{}") + block = [node for node in ast.parse(PRESTARTUP.read_text(encoding="utf-8")).body + if isinstance(node, ast.If) and ast.unparse(node.test) == "os.path.exists(restore_snapshot_path)"] + assert len(block) == 1 + + cli_args = types.ModuleType("comfy.cli_args") + cli_args.args = types.SimpleNamespace(listen="127.0.0.1") + monkeypatch.setitem(sys.modules, "comfy", types.ModuleType("comfy")) + monkeypatch.setitem(sys.modules, "comfy.cli_args", cli_args) + for name in ("COMFYUI_PATH", "COMFYUI_FOLDERS_BASE_PATH"): + monkeypatch.delenv(name, raising=False) + for name, value in environ.items(): + monkeypatch.setenv(name, value) + + started = [] + + def process_wrap(cmd, cwd=None, handler=None, env=None): + started.append((cmd, cwd, env)) + return 0 + + exec(compile(ast.Module(body=block, type_ignores=[]), str(PRESTARTUP), "exec"), { + "os": os, "sys": sys, "restore_snapshot_path": str(scheduled), "comfy_path": COMFY, + "custom_nodes_base_path": os.path.join(BASE, "custom_nodes"), + "manager_files_path": os.path.join(BASE, "user", "__manager"), + "folder_paths": types.SimpleNamespace(__file__=os.path.join(COMFY, "folder_paths.py"), + get_user_directory=lambda: os.path.join(BASE, "user")), + "default_conf": {}, "manager_security": types.SimpleNamespace(is_loopback_listener=lambda listen: True), + "process_wrap": process_wrap, + }) + assert len(started) == 1 and not scheduled.exists() + return started[0] + + +def test_cm_cli_is_given_the_comfyui_path(tmp_path, monkeypatch): + _, _, env = restore(tmp_path, monkeypatch, {}) + assert env.get("COMFYUI_PATH") == COMFY + + +def test_cm_cli_is_given_the_directories_of_the_server(tmp_path, monkeypatch): + cmd, _, _ = restore(tmp_path, monkeypatch, {}) + assert cmd[1:5] == ["-m", "cm_cli", "restore-snapshot", str(tmp_path / "restore-snapshot.json")] + # cm_cli takes --user-directory for the directory of the Manager's files + assert cmd[5:] == ["--user-directory", os.path.join(BASE, "user", "__manager"), + "--restore-to", os.path.join(BASE, "custom_nodes")] + + +@pytest.mark.parametrize("name", ["COMFYUI_PATH", "COMFYUI_FOLDERS_BASE_PATH"]) +def test_a_path_set_in_the_environment_is_kept(tmp_path, monkeypatch, name): + _, _, env = restore(tmp_path, monkeypatch, {name: os.path.join(os.sep, "elsewhere")}) + assert env[name] == os.path.join(os.sep, "elsewhere")