diff --git a/src/runpod_flash/cli/commands/build.py b/src/runpod_flash/cli/commands/build.py index 23e99506..f2fa745b 100644 --- a/src/runpod_flash/cli/commands/build.py +++ b/src/runpod_flash/cli/commands/build.py @@ -1,9 +1,11 @@ """Flash build command - Package Flash applications for deployment.""" import ast +import csv import hashlib import importlib.util import json +import posixpath import logging import re import shutil @@ -11,6 +13,7 @@ import subprocess import sys import tarfile +from email.parser import HeaderParser from pathlib import Path from typing import Optional @@ -160,18 +163,19 @@ def compute_source_fingerprint(project_dir: Path, files: list[Path]) -> str: PIP_MODULE = "pip" -# These are CUDA/GPU-oriented packages whose large CUDA builds are already -# provided by the GPU base images (runpod/pytorch:*) and therefore should -# not be bundled into the tarball. +# The torch family is never bundled. The GPU worker images ship torch, +# torchaudio, and triton from the base image, and install a torchvision pinned +# to that torch build. Each torchvision release requires one exact torch +# version, and the tarball is extracted ahead of site-packages on sys.path, so +# a bundled PyPI copy would shadow the paired one and fail to load against the +# image's torch. torch and triton are also too large to bundle. # Do NOT add packages here just because the GPU image ships them (e.g. numpy). -# The blacklist is defined strictly by size constraints, not by whether a -# package happens to be present in a particular base image. SIZE_PROHIBITIVE_PACKAGES: frozenset[str] = frozenset( { - "torch", # ~500 MB - "torchvision", # ~50 MB, requires torch - "torchaudio", # ~30 MB, requires torch - "triton", # ~150 MB, CUDA compiler + "torch", # ~550 MB + "torchvision", # small, but must match the image's torch build + "torchaudio", # must match the image's torch build + "triton", # ~250 MB, CUDA compiler } ) @@ -440,14 +444,27 @@ def run_build( # filter out excluded packages (auto + user-specified) if excluded_packages: auto_matched = set() + auto_pinned = [] user_matched = set() filtered_requirements = [] + torch_family = {_canonical_name(p) for p in SIZE_PROHIBITIVE_PACKAGES} + for req in requirements: - if should_exclude_package(req, excluded_packages): + # _requirement_name also reads "name @ url" forms, and wheel + # paths or URLs are named by their filename. + family_name = _requirement_name(req) or _wheel_distribution_name(req) + if family_name in torch_family or should_exclude_package( + req, excluded_packages + ): pkg_name = extract_package_name(req) - if pkg_name in SIZE_PROHIBITIVE_PACKAGES: - auto_matched.add(pkg_name) + if family_name in torch_family: + auto_matched.add(family_name) + if not _requirement_name(req) or re.search( + r"[<>=!~@]", req.split(";", 1)[0] + ): + # A wheel path or URL pins a version too. + auto_pinned.append(req.strip()) if pkg_name in user_excluded: user_matched.add(pkg_name) else: @@ -456,9 +473,15 @@ def run_build( requirements = filtered_requirements if auto_matched: - logger.debug( - "auto-excluded size-prohibitive packages: %s", - ", ".join(sorted(auto_matched)), + console.print( + f"[dim]not bundling {', '.join(sorted(auto_matched))}: " + "provided by the Runpod GPU worker image[/dim]" + ) + if auto_pinned: + print_warning( + console, + f"Version pins ignored: {', '.join(auto_pinned)}. Workers use the " + "torch-family versions in the Runpod GPU worker image.", ) # Only warn about unmatched user-specified packages (not auto-excludes) @@ -516,7 +539,11 @@ def run_build( with console.status("[dim]creating archive...[/dim]"): create_tarball( - build_dir, archive_path, app_name, excluded_packages=excluded_packages + build_dir, + archive_path, + app_name, + excluded_packages=excluded_packages, + root_requirements=requirements, ) size_mb = archive_path.stat().st_size / (1024 * 1024) @@ -779,9 +806,10 @@ def extract_package_name(requirement: str) -> str: >>> extract_package_name("my-package==1.0.0") "my-package" """ - # Split on version specifiers, extras, and environment markers - # This regex matches: < > = ! [ ; (common pip requirement delimiters) - package_name = re.split(r"[<>=!\[;]", requirement)[0].strip().lower() + # Split on version specifiers, extras, environment markers, direct + # references, and the whitespace/parenthesis before a specifier: + # < > = ! ~ [ ; @ ( and whitespace + package_name = re.split(r"[<>=!~\[;@(\s]", requirement.strip())[0].lower() return package_name @@ -1071,11 +1099,114 @@ def install_dependencies( return False +def _canonical_name(name: str) -> str: + """Canonical distribution name (PEP 503 runs of -_. collapse, lowercase).""" + return re.sub(r"[-_.]+", "_", name).lower() + + +def _dist_info_name(dist_info_dir: str) -> str: + """Distribution name from a "name-version.dist-info" directory name.""" + stem = dist_info_dir.removesuffix(".dist-info") + match = re.search(r"-\d", stem) + return stem[: match.start()] if match else stem + + +def _requirement_name(requirement: str) -> str: + """Canonical distribution name of a requirement or Requires-Dist string. + + Returns "" for lines that don't start with a name, such as bare URLs, + local paths or archive filenames, or pip options like "-r other.txt". + """ + match = re.match( + r"\s*([A-Za-z0-9][A-Za-z0-9._-]*)\s*(?:\[[^\]]*\])?\s*(?:$|[<>=!~;@(])", + requirement, + ) + if not match: + return "" + if requirement.strip() == match.group(1) and re.search( + r"\.(whl|zip|tar\.gz|tgz)$", match.group(1), re.I + ): + # A bare archive filename is a path, not a distribution name. + return "" + return _canonical_name(match.group(1)) + + +def _wheel_distribution_name(requirement: str) -> str: + """Canonical distribution name of a wheel path or URL, else "". + + Wheel filenames start with the distribution name (PEP 427), e.g. + https://download.pytorch.org/whl/cu128/torch-2.9.1%2Bcu128-cp312-...whl. + """ + location = re.split(r"[#?]", requirement.split(";", 1)[0].strip(), maxsplit=1)[0] + if not location.lower().endswith(".whl"): + return "" + filename = location.replace("\\", "/").rsplit("/", 1)[-1] + return _canonical_name(filename.split("-", 1)[0]) + + +def _excluded_only_dists( + build_dir: Path, excluded: set[str], roots: set[str] +) -> set[str]: + """Installed distributions that only excluded packages depend on. + + pip installs the whole dependency tree of an excluded package (torch pulls + in ~2 GB of nvidia/cuda wheels on Linux x86_64). The GPU worker image + installs those alongside its own torch, so they are dead weight in the + tarball. A distribution is kept if a root requirement, or any wheel that + the excluded packages did not pull in, depends on it. Markers and extras + are ignored, which can only keep more, never drop a needed wheel. + + Args: + build_dir: Directory pip installed into + excluded: Canonical names of excluded packages + roots: Canonical names of the user's requirements + + Returns: + Canonical names to drop, not including the excluded packages. + """ + requires: dict[str, set[str]] = {} + for dist_info in build_dir.glob("*.dist-info"): + name = _canonical_name(_dist_info_name(dist_info.name)) + metadata = dist_info / "METADATA" + if not metadata.is_file(): + # Unknown dependencies: we cannot prove anything is unused. + return set() + headers = HeaderParser().parsestr( + metadata.read_text(encoding="utf-8", errors="replace") + ) + deps = set() + for req in headers.get_all("Requires-Dist") or []: + dep = _requirement_name(req) + if not dep: + # An edge we can't read could protect any wheel. + return set() + deps.add(dep) + requires[name] = deps + + def closure(start: set[str], stop: set[str]) -> set[str]: + seen: set[str] = set() + stack = [n for n in start if n in requires] + while stack: + node = stack.pop() + if node in seen: + continue + seen.add(node) + stack.extend(d for d in requires[node] if d in requires and d not in stop) + return seen + + installed = set(requires) + pulled_by_excluded = closure(installed & excluded, stop=set()) + seed = ((installed - pulled_by_excluded) | (installed & roots)) - excluded + kept = closure(seed, stop=excluded) + return installed - kept - excluded + + def create_tarball( build_dir: Path, output_path: Path, app_name: str, excluded_packages: list[str] | None = None, + root_requirements: list[str] | None = None, ) -> None: """ Create gzipped tarball of build directory, excluding size-prohibitive packages. @@ -1089,12 +1220,87 @@ def create_tarball( output_path: Output archive path app_name: Application name (unused, for compatibility) excluded_packages: Package names to exclude from the archive + root_requirements: The user's requirements. Wheels that only excluded + packages depend on are dropped unless listed here. """ # Build set of normalized names for fast lookup excluded_normalized: set[str] = set() if excluded_packages: excluded_normalized = {_normalize_package_name(p) for p in excluded_packages} + # RECORD lists every file a wheel installed, relative to build_dir. + records: dict[str, set[str]] = {} + records_complete = True + for dist_info in build_dir.glob("*.dist-info"): + record = dist_info / "RECORD" + if not record.is_file(): + records_complete = False + break + paths = set() + with record.open(newline="", encoding="utf-8", errors="replace") as f: + for row in csv.reader(f): + if not row: + continue + record_path = posixpath.normpath(row[0].replace("\\", "/")) + if not record_path.startswith("../") and record_path not in (".", ".."): + paths.add(record_path) + records[_canonical_name(_dist_info_name(dist_info.name))] = paths + + # Drop the excluded packages plus wheels only they depend on. Without a + # RECORD for every wheel we cannot tell which files belong to whom, so + # fall back to dropping the excluded packages by name only. + dropped: set[str] = set() + if excluded_normalized: + excluded_canonical = {_canonical_name(p) for p in excluded_packages or []} + dropped = set(excluded_canonical) + requirement_lines = [ + r.strip() + for r in root_requirements or [] + if r.strip() and not r.strip().startswith("#") + ] + roots = {_requirement_name(r) for r in requirement_lines} + # --exclude removes only the named package, so its dependencies are + # protected like the user's own requirements. + roots |= excluded_canonical - { + _canonical_name(p) for p in SIZE_PROHIBITIVE_PACKAGES + } + # A requirement we can't name (bare URL, local path, -r file) may + # install any wheel, so we can't prove a wheel is unrequested. + if records_complete and "" not in roots: + # Only the torch family is provided by the image, so only its + # dependency tree is safe to prune. A user --exclude drops just + # the named package. + image_provided = excluded_canonical & { + _canonical_name(p) for p in SIZE_PROHIBITIVE_PACKAGES + } + dependency_only = ( + _excluded_only_dists(build_dir, image_provided, roots) + - excluded_canonical + ) + if dependency_only: + console.print( + f"[dim]not bundling {len(dependency_only)} packages that only " + "torch-family packages depend on[/dim]" + ) + logger.debug("dropped: %s", ", ".join(sorted(dependency_only))) + dropped |= dependency_only + + # Wheels can install top-level entries that don't share the package name + # (torch ships functorch/ and torchgen/, torchvision ships + # torchvision.libs/, the nvidia wheels share nvidia/). Drop every file + # that only dropped wheels installed. Top-level entries holding such files + # are walked file by file, so files a kept wheel installed and files no + # RECORD lists stay. Entries named after an excluded package (torch/) are + # still dropped whole, as before. + dropped_files: set[str] = set() + filtered_tops: set[str] = set() + if dropped and records_complete: + kept_files: set[str] = set() + for name, paths in records.items(): + (dropped_files if name in dropped else kept_files).update(paths) + dropped_files -= kept_files + filtered_tops = {p.split("/", 1)[0] for p in dropped_files} + def _is_excluded_top_dir(top_dir: str) -> bool: """Check if a top-level directory should be excluded from the tarball.""" # Check package directories (e.g. "numpy", "torch") @@ -1103,16 +1309,9 @@ def _is_excluded_top_dir(top_dir: str) -> bool: # Check dist-info directories (e.g. "numpy-1.24.0.dist-info") if top_dir.endswith(".dist-info"): - # dist-info format: "package_name-version.dist-info" - # Strip suffix, then split package name from version at first digit segment - stem = top_dir.removesuffix(".dist-info") - # Find the last hyphen followed by a digit (version separator) - match = re.search(r"-\d", stem) - dist_name = stem[: match.start()] if match else stem - if _normalize_package_name(dist_name) in excluded_normalized: - return True + return _canonical_name(_dist_info_name(top_dir)) in dropped - return False + return top_dir in dropped_files # Remove existing archive if output_path.exists(): @@ -1129,6 +1328,16 @@ def _is_excluded_top_dir(top_dir: str) -> bool: top_dir = rel.parts[0] if excluded_normalized and _is_excluded_top_dir(top_dir): continue + if top_dir in filtered_tops and item.is_dir() and not item.is_symlink(): + # Add only surviving files; tar creates their parent dirs on + # extract, so a dir emptied by filtering leaves no entry. + for path in sorted(item.rglob("*")): + if path.is_dir() and not path.is_symlink(): + continue + rel_path = path.relative_to(build_dir).as_posix() + if rel_path not in dropped_files: + tar.add(str(path), arcname=f"./{rel_path}", recursive=False) + continue arcname = f"./{rel}" tar.add(str(item), arcname=arcname) diff --git a/tests/unit/cli/commands/test_build.py b/tests/unit/cli/commands/test_build.py index fc38926b..e292a319 100644 --- a/tests/unit/cli/commands/test_build.py +++ b/tests/unit/cli/commands/test_build.py @@ -8,6 +8,7 @@ from runpod_flash.cli.commands.build import ( SIZE_PROHIBITIVE_PACKAGES, + _requirement_name, _find_runpod_flash, _resolve_pip_python_version, collect_requirements, @@ -43,6 +44,12 @@ def test_package_with_environment_marker(self): """Test extraction from package with environment marker.""" assert extract_package_name('torch>=2.0; python_version>"3.8"') == "torch" + def test_package_with_compatible_release_and_direct_reference(self): + """'~=' pins, 'name @ url' references, and old-style parens split off.""" + assert extract_package_name("sympy~=1.14") == "sympy" + assert extract_package_name("sympy @ https://example.com/s.whl") == "sympy" + assert extract_package_name("foo (>=1.0)") == "foo" + def test_package_with_multiple_specifiers(self): """Test extraction from package with multiple specifiers.""" assert extract_package_name("torch>=2.0.0,<3.0.0") == "torch" @@ -698,7 +705,95 @@ def test_auto_exclude_silent_when_not_in_requirements(self, tmp_path, capsys): run_build(project_dir, "test_app", no_deps=True) captured = capsys.readouterr() - assert "Auto-excluded size-prohibitive packages" not in captured.out + assert "not bundling" not in captured.out + + def _build_with_deps(self, tmp_path, deps, exclude=None): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "worker.py").write_text( + "from runpod_flash import remote, LiveServerless\n" + "gpu = LiveServerless(name='gpu_worker')\n" + f"@remote(gpu, dependencies={deps!r})\n" + "def my_func(prompt: str) -> str:\n" + " return prompt\n" + ) + + installed = [] + + def fake_install(_build_dir, reqs, _no_deps, target_python_version=None): + installed.extend(reqs) + return True + + with ( + patch( + "runpod_flash.cli.commands.build.install_dependencies", + side_effect=fake_install, + ), + self._bundle_patches(), + ): + run_build(project_dir, "test_app", no_deps=True, exclude=exclude) + + return [extract_package_name(r) for r in installed] + + @pytest.mark.parametrize( + "pinned", + [ + "torchvision~=0.24.1", + "torchvision @ https://example.com/torchvision.whl", + "https://download.pytorch.org/whl/cu128/" + "torchvision-0.24.1%2Bcu128-cp312-cp312-manylinux_2_28_x86_64.whl", + "https://example.com/torchvision-0.24.1-cp312-cp312-linux_x86_64.whl" + "#sha256=abc123", + ], + ) + def test_auto_exclude_catches_compatible_and_url_pins( + self, tmp_path, capsys, pinned + ): + """'~=' and 'name @ url' torch-family requirements are excluded and warned.""" + pkg_names = self._build_with_deps(tmp_path, [pinned, "requests"]) + + out = capsys.readouterr().out + assert not any("torchvision" in name for name in pkg_names) + assert "requests" in pkg_names + assert "Version pins ignored" in out + + @pytest.mark.parametrize( + "requirement", ["sympy~=1.14", "sympy @ https://example.com/sympy.whl"] + ) + def test_user_exclude_matches_compatible_and_url_pins( + self, tmp_path, capsys, requirement + ): + """--exclude sympy also catches 'sympy~=1.14' and 'sympy @ url'.""" + pkg_names = self._build_with_deps( + tmp_path, [requirement, "requests"], exclude="sympy" + ) + + out = capsys.readouterr().out + assert "sympy" not in pkg_names + assert "requests" in pkg_names + assert "No packages matched exclusions" not in out + + def test_auto_exclude_tells_user_image_provides_package(self, tmp_path, capsys): + """An unpinned torchvision is dropped with a visible notice, not a warning.""" + pkg_names = self._build_with_deps(tmp_path, ["torchvision", "requests"]) + + out = capsys.readouterr().out + assert "torchvision" not in pkg_names + assert "not bundling torchvision" in out + assert "worker image" in out + assert "Version pins ignored" not in out + + def test_auto_exclude_warns_when_version_pin_is_ignored(self, tmp_path, capsys): + """A pinned torch-family requirement warns that the pin has no effect.""" + pkg_names = self._build_with_deps( + tmp_path, ["torchvision==0.29.1", "torch", "requests"] + ) + + out = capsys.readouterr().out + assert "torchvision" not in pkg_names + assert "torch" not in pkg_names + assert "Version pins ignored" in out + assert "torchvision==0.29.1" in out def test_user_unmatched_warning_excludes_base_image_packages( self, tmp_path, capsys @@ -759,6 +854,289 @@ def test_tarball_excludes_base_image_packages(self, tmp_path): # requests should still be present assert any("requests" in n for n in names) + def _fake_wheel(self, build_dir, dist_info, files, requires=(), metadata=True): + (build_dir / dist_info).mkdir() + for f in files: + path = build_dir / f + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("") + if metadata: + lines = ["Metadata-Version: 2.1", f"Name: {dist_info.split('-')[0]}"] + lines += [f"Requires-Dist: {r}" for r in requires] + (build_dir / dist_info / "METADATA").write_text("\n".join(lines) + "\n") + record = [f"{f},," for f in files] + [f"{dist_info}/RECORD,,"] + (build_dir / dist_info / "RECORD").write_text("\n".join(record)) + + def _tarball_names(self, tmp_path, build_dir, roots): + import tarfile + + output = tmp_path / "test.tar.gz" + create_tarball( + build_dir, + output, + "test_app", + excluded_packages=sorted(SIZE_PROHIBITIVE_PACKAGES), + root_requirements=roots, + ) + with tarfile.open(output, "r:gz") as tar: + return {n.removeprefix("./") for n in tar.getnames() if n != "."} + + def _tarball_top_level(self, tmp_path, build_dir, roots): + names = self._tarball_names(tmp_path, build_dir, roots) + return {n.split("/")[0] for n in names} + + def _torch_tree(self, build_dir, nvidia_metadata=True): + wheel = self._fake_wheel + wheel( + build_dir, + "torch-2.14.1.dist-info", + ["torch/__init__.py"], + ["filelock", "nvidia-cublas==13.1.1.3; platform_system == 'Linux'"], + ) + wheel( + build_dir, + "torchvision-0.29.1.dist-info", + ["torchvision/__init__.py"], + ["torch==2.14.1", "numpy", "pillow>=5.3.0"], + ) + wheel( + build_dir, + "nvidia_cublas-13.1.1.3.dist-info", + ["nvidia/cublas/lib.so"], + metadata=nvidia_metadata, + ) + wheel(build_dir, "numpy-2.4.6.dist-info", ["numpy/__init__.py"]) + wheel(build_dir, "pillow-12.3.0.dist-info", ["PIL/__init__.py"]) + wheel(build_dir, "filelock-4.2.0.dist-info", ["filelock/__init__.py"]) + wheel( + build_dir, + "huggingface_hub-2.2.0.dist-info", + ["huggingface_hub/__init__.py"], + ["filelock"], + ) + wheel( + build_dir, + "timm-1.0.30.dist-info", + ["timm/__init__.py"], + ["torch", "torchvision", "huggingface_hub"], + ) + + def test_tarball_excludes_extra_top_level_dirs_of_excluded_wheels(self, tmp_path): + """Top-level dirs listed in an excluded wheel's RECORD are dropped too, + but a dir shared with a kept wheel (bin/) stays.""" + build_dir = tmp_path / "build" + build_dir.mkdir() + wheel = self._fake_wheel + wheel( + build_dir, + "torch-2.14.1.dist-info", + ["torch/__init__.py", "functorch/__init__.py", "bin/torchrun"], + ) + wheel( + build_dir, + "torchvision-0.29.1.dist-info", + ["torchvision/__init__.py", "torchvision.libs/libpng.so"], + ["torch"], + ) + wheel( + build_dir, + "timm-1.0.30.dist-info", + ["timm/__init__.py", "bin/timm-cli"], + ["torch", "torchvision"], + ) + + top_level = self._tarball_top_level(tmp_path, build_dir, ["timm"]) + + assert top_level == {"timm", "timm-1.0.30.dist-info", "bin"} + + def test_tarball_drops_wheels_only_torch_family_needs(self, tmp_path): + """nvidia/numpy pulled in only by torch/torchvision are dropped; deps of + kept wheels (filelock) and root requirements (pillow) stay.""" + build_dir = tmp_path / "build" + build_dir.mkdir() + self._torch_tree(build_dir) + + top_level = self._tarball_top_level(tmp_path, build_dir, ["timm", "pillow>=10"]) + + packages = {t for t in top_level if not t.endswith(".dist-info")} + assert packages == {"timm", "huggingface_hub", "filelock", "PIL"} + + def test_tarball_keeps_dependencies_when_metadata_missing(self, tmp_path): + """Without full metadata the dependency graph is unknown, so only the + excluded packages themselves are dropped.""" + build_dir = tmp_path / "build" + build_dir.mkdir() + self._torch_tree(build_dir, nvidia_metadata=False) + + top_level = self._tarball_top_level(tmp_path, build_dir, ["timm"]) + + assert "nvidia" in top_level + assert "numpy" in top_level + assert "torch" not in top_level + assert "torchvision" not in top_level + + def test_tarball_keeps_root_requirement_given_as_direct_url(self, tmp_path): + """A root written as 'name @ url' still protects that wheel.""" + build_dir = tmp_path / "build" + build_dir.mkdir() + self._torch_tree(build_dir) + + top_level = self._tarball_top_level( + tmp_path, build_dir, ["timm", "numpy @ https://example.com/numpy.whl"] + ) + + assert "numpy" in top_level + + def test_tarball_filters_files_in_dir_shared_with_kept_wheel(self, tmp_path): + """A kept nvidia wheel keeps its files under nvidia/, but files from an + nvidia wheel only torch needs are still dropped.""" + build_dir = tmp_path / "build" + build_dir.mkdir() + self._torch_tree(build_dir) + self._fake_wheel( + build_dir, "nvidia_cudnn-9.24.0.dist-info", ["nvidia/cudnn/lib.so"] + ) + + names = self._tarball_names(tmp_path, build_dir, ["timm", "nvidia-cudnn"]) + + assert "nvidia/cudnn/lib.so" in names + assert "nvidia/cublas/lib.so" not in names + assert not any(n.startswith("nvidia_cublas") for n in names) + + def test_tarball_drops_only_excluded_when_a_record_is_missing(self, tmp_path): + """Without every RECORD, file ownership is unknown, so only the + excluded packages themselves are dropped.""" + build_dir = tmp_path / "build" + build_dir.mkdir() + self._torch_tree(build_dir) + (build_dir / "huggingface_hub-2.2.0.dist-info" / "RECORD").unlink() + + top_level = self._tarball_top_level(tmp_path, build_dir, ["timm"]) + + assert {"nvidia", "numpy", "huggingface_hub"} <= top_level + assert "torch" not in top_level + assert "torchvision" not in top_level + + def test_tarball_keeps_dependencies_of_user_excluded_package(self, tmp_path): + """--exclude drops only the named package. The worker image doesn't + provide its dependencies, so they stay.""" + import tarfile + + build_dir = tmp_path / "build" + build_dir.mkdir() + self._fake_wheel( + build_dir, "scipy-1.17.0.dist-info", ["scipy/__init__.py"], ["numpy"] + ) + self._fake_wheel(build_dir, "numpy-2.4.6.dist-info", ["numpy/__init__.py"]) + + output = tmp_path / "test.tar.gz" + create_tarball( + build_dir, + output, + "test_app", + excluded_packages=sorted(SIZE_PROHIBITIVE_PACKAGES | {"scipy"}), + root_requirements=["scipy"], + ) + with tarfile.open(output, "r:gz") as tar: + top_level = {n.split("/")[1] for n in tar.getnames() if n != "."} + + assert "scipy" not in top_level + assert "numpy" in top_level + + def test_tarball_skips_pruning_when_a_requirement_has_no_name(self, tmp_path): + """A bare URL could install any wheel, so nothing beyond the excluded + packages is dropped.""" + build_dir = tmp_path / "build" + build_dir.mkdir() + self._torch_tree(build_dir) + + top_level = self._tarball_top_level( + tmp_path, build_dir, ["timm", "https://example.com/numpy.whl"] + ) + + assert {"numpy", "nvidia"} <= top_level + assert "torch" not in top_level + + def test_tarball_skips_pruning_on_unreadable_requires_dist(self, tmp_path): + """A Requires-Dist line without a name stops dependency pruning.""" + build_dir = tmp_path / "build" + build_dir.mkdir() + self._torch_tree(build_dir) + metadata = build_dir / "timm-1.0.30.dist-info" / "METADATA" + metadata.write_text(metadata.read_text() + "Requires-Dist: ./vendored.whl\n") + + top_level = self._tarball_top_level(tmp_path, build_dir, ["timm"]) + + assert {"numpy", "nvidia"} <= top_level + assert "torch" not in top_level + + def test_tarball_skips_pruning_for_bare_wheel_filename(self, tmp_path): + """A wheel filename root names no distribution, so pruning is skipped.""" + build_dir = tmp_path / "build" + build_dir.mkdir() + self._torch_tree(build_dir) + + top_level = self._tarball_top_level( + tmp_path, + build_dir, + ["timm", "numpy-2.4.6-cp312-cp312-manylinux_2_28_x86_64.whl"], + ) + + assert "numpy" in top_level + + def test_tarball_keeps_dependencies_of_user_excluded_torch_dependency( + self, tmp_path + ): + """--exclude sympy drops sympy only, even though torch also needs it.""" + import tarfile + + build_dir = tmp_path / "build" + build_dir.mkdir() + wheel = self._fake_wheel + wheel(build_dir, "torch-2.14.1.dist-info", ["torch/__init__.py"], ["sympy"]) + wheel(build_dir, "sympy-1.14.0.dist-info", ["sympy/__init__.py"], ["mpmath"]) + wheel(build_dir, "mpmath-1.3.0.dist-info", ["mpmath/__init__.py"]) + wheel(build_dir, "timm-1.0.30.dist-info", ["timm/__init__.py"], ["torch"]) + + output = tmp_path / "test.tar.gz" + create_tarball( + build_dir, + output, + "test_app", + excluded_packages=sorted(SIZE_PROHIBITIVE_PACKAGES | {"sympy"}), + root_requirements=["timm"], + ) + with tarfile.open(output, "r:gz") as tar: + top_level = {n.split("/")[1] for n in tar.getnames() if n != "."} + + assert "sympy" not in top_level + assert "mpmath" in top_level + + def test_tarball_keeps_unrecorded_files_next_to_dropped_ones(self, tmp_path): + """Files no RECORD lists stay, even beside files of a dropped wheel.""" + build_dir = tmp_path / "build" + build_dir.mkdir() + self._torch_tree(build_dir) + (build_dir / "nvidia" / "notes.txt").write_text("") + + names = self._tarball_names(tmp_path, build_dir, ["timm"]) + + assert "nvidia/notes.txt" in names + assert "nvidia/cublas/lib.so" not in names + + def test_tarball_normalizes_record_paths(self, tmp_path): + """RECORD paths written with ./ or backslashes still match on disk.""" + build_dir = tmp_path / "build" + build_dir.mkdir() + self._torch_tree(build_dir) + (build_dir / "nvidia_cublas-13.1.1.3.dist-info" / "RECORD").write_text( + "./nvidia\\cublas\\lib.so,,\n" + ) + + names = self._tarball_names(tmp_path, build_dir, ["timm"]) + + assert "nvidia/cublas/lib.so" not in names + def test_tarball_keeps_non_excluded_packages(self, tmp_path): """create_tarball keeps packages not in the exclusion list.""" build_dir = tmp_path / "build" @@ -938,3 +1316,24 @@ def test_handles_binary_files(self, tmp_path): f1.write_bytes(b"\x00\x01\x02\xff") result = compute_source_fingerprint(tmp_path, [f1]) assert len(result) == 64 + + +@pytest.mark.parametrize( + ("requirement", "expected"), + [ + ("torch", "torch"), + ("Pillow[extra] >= 5", "pillow"), + ("zope.interface", "zope_interface"), + ("filelock @ https://example.com/filelock.whl", "filelock"), + ("nvidia-cublas==13.1; platform_system == 'Linux'", "nvidia_cublas"), + ("foo (>=1.0)", "foo"), + ("https://example.com/numpy.whl", ""), + ("./wheels/numpy.whl", ""), + ("-r other.txt", ""), + ("numpy-2.4.6-cp312-cp312-manylinux_2_28_x86_64.whl", ""), + ("mypkg-1.0.tar.gz", ""), + ("my.whl==1.0", "my_whl"), + ], +) +def test_requirement_name(requirement, expected): + assert _requirement_name(requirement) == expected