diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 3b74153..3edf80f 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -45,6 +45,20 @@ jobs: - name: Run tests run: python -m unittest discover -s tests -v + # Same suite again with env vars exported that would flip outcomes if any + # test leaked the developer's shell environment into production code. + - name: Run tests with a hostile environment + run: python -m unittest discover -s tests + env: + SKIP_RATIO_CHECK: "true" + CLEANUP_MIN_RATIO: "5" + CLEANUP_SKIP_RATIO_CHECK: "true" + TRANSMISSION_RPC_USERNAME: x + TRANSMISSION_RPC_PASSWORD: y + FETCH_TORRENTS_DISTROS: arch + FETCH_TORRENTS_LOG_MAX_BYTES: "7" + FETCH_TORRENTS_ALWAYS_LOG: "false" + build: needs: test runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2023dc1..dbd1c0c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,6 +22,57 @@ When making changes to the fetch logic or features: - Verify regex patterns match all intended torrent names (e.g., Ubuntu variants) - Ratio checking is per ISO type (e.g., `installer-amd64`) rather than per distro +## Mutation testing + +Mutation testing checks that the tests actually pin behavior, not just +execute lines. It uses mutmut 2.x (`mutmut<3`): + +``` +mutmut run --paths-to-mutate fetch_torrents.py,torrent_scrape.py,configure_transmission.py \ + --tests-dir tests/ --runner "python -m unittest discover -s tests -f -q" +mutmut results # list surviving mutants +mutmut show # show one mutant's diff +``` + +mutmut edits the source files in place while it runs, so run it on a clean +working tree (or a copy of the repo). A full run takes a while (~40 minutes on +Windows). + +### Handling surviving mutants + +A survivor is either a missing test or a mutant that cannot change behavior +(an *equivalent* mutant). Deal with them in this order: + +1. **Write the missing test.** Most survivors are real gaps: an untested + boundary (`>=` vs `>`), a `continue` that could be a `break`, a branch whose + result is never asserted. Confirm the new test fails with the mutant applied + (`mutmut apply `, run the test, then `git checkout` the file). +2. **If no test can kill it, remove the cause.** An equivalent mutant usually + points at redundant code: a variable initialised but never read, or a + condition that an earlier step already guarantees. Simplify the code so the + mutant can no longer be generated. Pin the current behavior with a test + before refactoring. +3. **If the code is already as simple as it should be, exclude the line** with + `# pragma: no mutate` and say why on the same line, so the exclusion can be + reviewed: + + ```python + if os.getenv("SKIP_RATIO_CHECK", "false").lower() == "true": # pragma: no mutate (any default but "true" is equivalent) + ``` + + Don't use the pragma to hide a gap you could test. + +**Log and error-message strings.** mutmut changes string literals +(`"Could not…"` → `"XXCould not…XX"`), and most of these survive because tests +don't assert exact log text. They're expected and are not worth a pragma on +every log line. Assert the message only where its content matters, for example +an error that names the bad input. + +**In CI**, don't gate on zero survivors. Use a score threshold for the critical +modules, a committed baseline of known survivors (fail only on new ones), or +mutate only the lines changed in the PR. Run it on a schedule rather than on +every push, given how long it takes. + ## Cutting a release Releases and container publishing are automated from Git tags: diff --git a/configure_transmission.py b/configure_transmission.py index 264f2e5..9eba1f6 100644 --- a/configure_transmission.py +++ b/configure_transmission.py @@ -9,6 +9,7 @@ build_port_forwarding_overrides()'s first-run default - see its docstring - which only ever applies before there's anything on disk to disturb. """ + import json import os import sys @@ -17,25 +18,25 @@ def build_rpc_overrides(env): overrides = {} - whitelist = env.get('TRANSMISSION_RPC_WHITELIST', '').strip() + whitelist = env.get("TRANSMISSION_RPC_WHITELIST", "").strip() if whitelist: - overrides['rpc-whitelist-enabled'] = True - overrides['rpc-whitelist'] = whitelist + overrides["rpc-whitelist-enabled"] = True + overrides["rpc-whitelist"] = whitelist - username = env.get('TRANSMISSION_RPC_USERNAME', '').strip() - password = env.get('TRANSMISSION_RPC_PASSWORD', '').strip() + username = env.get("TRANSMISSION_RPC_USERNAME", "").strip() + password = env.get("TRANSMISSION_RPC_PASSWORD", "").strip() if username and password: - overrides['rpc-authentication-required'] = True - overrides['rpc-username'] = username + overrides["rpc-authentication-required"] = True + overrides["rpc-username"] = username # Plaintext is intentional: transmission-daemon hashes rpc-password # in place on its next start, same as if set by hand. - overrides['rpc-password'] = password + overrides["rpc-password"] = password return overrides def _parse_positive_int(value): - value = (value or '').strip() + value = (value or "").strip() if not value: return None try: @@ -54,21 +55,23 @@ def build_peer_limit_overrides(env): honor (or that blocks all peers, in the case of 0).""" overrides = {} - global_limit = _parse_positive_int(env.get('TRANSMISSION_PEER_LIMIT_GLOBAL', '')) + global_limit = _parse_positive_int(env.get("TRANSMISSION_PEER_LIMIT_GLOBAL", "")) if global_limit is not None: - overrides['peer-limit-global'] = global_limit + overrides["peer-limit-global"] = global_limit - per_torrent_limit = _parse_positive_int(env.get('TRANSMISSION_PEER_LIMIT_PER_TORRENT', '')) + per_torrent_limit = _parse_positive_int( + env.get("TRANSMISSION_PEER_LIMIT_PER_TORRENT", "") + ) if per_torrent_limit is not None: - overrides['peer-limit-per-torrent'] = per_torrent_limit + overrides["peer-limit-per-torrent"] = per_torrent_limit return overrides def _parse_bool(value): - value = (value or '').strip().lower() - if value in ('true', 'false'): - return value == 'true' + value = (value or "").strip().lower() + if value in ("true", "false"): + return value == "true" return None @@ -82,11 +85,11 @@ def build_port_forwarding_overrides(env, settings_file_exists): Transmission itself would otherwise pick its own default. Once the file exists, this is untouched unless TRANSMISSION_PORT_FORWARDING says otherwise, same as every other setting here.""" - explicit = _parse_bool(env.get('TRANSMISSION_PORT_FORWARDING', '')) + explicit = _parse_bool(env.get("TRANSMISSION_PORT_FORWARDING", "")) if explicit is not None: - return {'port-forwarding-enabled': explicit} + return {"port-forwarding-enabled": explicit} if not settings_file_exists: - return {'port-forwarding-enabled': False} + return {"port-forwarding-enabled": False} return {} @@ -95,7 +98,9 @@ def warn_if_open_without_auth(overrides): anyone who can reach the RPC port has full control (add/remove/delete data) - Transmission has no read-only RPC mode. Returns a warning message to log, or None if there's nothing to warn about.""" - if overrides.get('rpc-whitelist-enabled') and not overrides.get('rpc-authentication-required'): + if overrides.get("rpc-whitelist-enabled") and not overrides.get( + "rpc-authentication-required" + ): return ( "TRANSMISSION_RPC_WHITELIST is set but TRANSMISSION_RPC_USERNAME/" "TRANSMISSION_RPC_PASSWORD are not - the web UI/RPC endpoint is " @@ -114,22 +119,25 @@ def load_settings(path): if not os.path.exists(path): return {} try: - with open(path, 'r', encoding='utf-8') as f: + with open(path, "r", encoding="utf-8") as f: return json.load(f) except (json.JSONDecodeError, OSError) as exc: - print(f"configure_transmission: could not read {path} ({exc}) - starting fresh.", file=sys.stderr) + print( + f"configure_transmission: could not read {path} ({exc}) - starting fresh.", + file=sys.stderr, + ) return {} def save_settings(path, settings): tmp_path = f"{path}.tmp" - with open(tmp_path, 'w', encoding='utf-8') as f: + with open(tmp_path, "w", encoding="utf-8") as f: json.dump(settings, f, indent=4, sort_keys=True) os.replace(tmp_path, path) def main(): - settings_path = os.getenv('TRANSMISSION_SETTINGS_FILE', '/config/settings.json') + settings_path = os.getenv("TRANSMISSION_SETTINGS_FILE", "/config/settings.json") settings_file_exists = os.path.exists(settings_path) overrides = { **build_rpc_overrides(os.environ), diff --git a/fetch_torrents.py b/fetch_torrents.py index 085528a..6493bd0 100644 --- a/fetch_torrents.py +++ b/fetch_torrents.py @@ -1,30 +1,33 @@ #!/usr/bin/env python3 -import os -import re import json -import requests import logging import logging.handlers -import tempfile -import time +import os +import re import shutil import socket +import tempfile +import time from datetime import date -from bs4 import BeautifulSoup -from transmission_rpc import Client +import requests import torrent_scrape +from bs4 import BeautifulSoup +from transmission_rpc import Client # Configure logging -log_dir = os.getenv('FETCH_TORRENTS_LOG_DIR', '/logs') +log_dir = os.getenv("FETCH_TORRENTS_LOG_DIR", "/logs") log_file = os.path.join(log_dir, "fetch_torrents.log") ratio_log_file = os.path.join(log_dir, "fetch_torrents_ratios.log") ratio_history_file = os.path.join(log_dir, "fetch_torrents_ratio_history.json") -old_release_check_state_file = os.path.join(log_dir, "fetch_torrents_old_release_check_state.json") +old_release_check_state_file = os.path.join( + log_dir, "fetch_torrents_old_release_check_state.json" +) removed_history_file = os.path.join(log_dir, "fetch_torrents_removed_history.json") + def parse_log_level(env_var: str, default: int = logging.INFO) -> int: - value = os.getenv(env_var, '').strip() + value = os.getenv(env_var, "").strip() if not value: return default if value.isdigit(): @@ -37,14 +40,14 @@ def parse_log_level(env_var: str, default: int = logging.INFO) -> int: def parse_bool(env_var: str, default: bool = False) -> bool: - value = os.getenv(env_var, '').strip().lower() + value = os.getenv(env_var, "").strip().lower() if not value: return default - return value in ('1', 'true', 'yes', 'on') + return value in ("1", "true", "yes", "on") def parse_int(env_var: str, default: int) -> int: - value = os.getenv(env_var, '').strip() + value = os.getenv(env_var, "").strip() if not value: return default try: @@ -52,7 +55,10 @@ def parse_int(env_var: str, default: int) -> int: except ValueError: return default -log_level = parse_log_level('FETCH_TORRENTS_LOG_LEVEL', parse_log_level('LOG_LEVEL', logging.INFO)) + +log_level = parse_log_level( + "FETCH_TORRENTS_LOG_LEVEL", parse_log_level("LOG_LEVEL", logging.INFO) +) formatter = logging.Formatter("%(asctime)s %(levelname)s: %(message)s") stream_handler = logging.StreamHandler() @@ -62,21 +68,26 @@ def parse_int(env_var: str, default: int) -> int: # grows without bound for the lifetime of a "deploy and forget" container - # see FETCH_TORRENTS_LOG_MAX_BYTES/FETCH_TORRENTS_LOG_BACKUP_COUNT in the # README. 5MB x 3 backups keeps a generous amount of history bounded to ~20MB. -log_max_bytes = parse_int('FETCH_TORRENTS_LOG_MAX_BYTES', 5 * 1024 * 1024) -log_backup_count = parse_int('FETCH_TORRENTS_LOG_BACKUP_COUNT', 3) +log_max_bytes = parse_int("FETCH_TORRENTS_LOG_MAX_BYTES", 5 * 1024 * 1024) +log_backup_count = parse_int("FETCH_TORRENTS_LOG_BACKUP_COUNT", 3) file_handler = logging.handlers.RotatingFileHandler( - log_file, maxBytes=log_max_bytes, backupCount=log_backup_count, + log_file, + maxBytes=log_max_bytes, + backupCount=log_backup_count, ) file_handler.setFormatter(formatter) + class RatioOnlyFilter(logging.Filter): def filter(self, record: logging.LogRecord) -> bool: return record.getMessage().startswith("[ratio]") + class NonRatioFilter(logging.Filter): def filter(self, record: logging.LogRecord) -> bool: return not record.getMessage().startswith("[ratio]") + class ImportantMessageFilter(logging.Filter): def __init__(self, level: int, important_prefixes=None): super().__init__() @@ -104,7 +115,8 @@ def get_always_log_prefixes(): "Old-release check complete", ] -always_log_enabled = parse_bool('FETCH_TORRENTS_ALWAYS_LOG', True) + +always_log_enabled = parse_bool("FETCH_TORRENTS_ALWAYS_LOG", True) important_prefixes = get_always_log_prefixes() if always_log_enabled else [] stream_handler.setLevel(logging.DEBUG) @@ -121,12 +133,14 @@ def get_always_log_prefixes(): else: file_handler.setLevel(log_level) -ratio_handler = logging.FileHandler(ratio_log_file, mode='w', delay=True, encoding='utf-8') +ratio_handler = logging.FileHandler( + ratio_log_file, mode="w", delay=True, encoding="utf-8" +) ratio_handler.setLevel(logging.INFO) ratio_handler.addFilter(RatioOnlyFilter()) ratio_handler.setFormatter(formatter) -logger = logging.getLogger('fetch_torrents') +logger = logging.getLogger("fetch_torrents") logger.setLevel(logging.DEBUG) logger.addHandler(stream_handler) logger.addHandler(file_handler) @@ -144,12 +158,12 @@ def get_rpc_credentials(): TRANSMISSION_RPC_WHITELIST is widened beyond localhost) makes every one of this script's own localhost RPC calls fail with 401 - silently, since callers only log the failure - permanently disabling cleanup.""" - username = os.getenv('TRANSMISSION_RPC_USERNAME', '').strip() - password = os.getenv('TRANSMISSION_RPC_PASSWORD', '').strip() + username = os.getenv("TRANSMISSION_RPC_USERNAME", "").strip() + password = os.getenv("TRANSMISSION_RPC_PASSWORD", "").strip() return username or None, password or None -def wait_for_transmission_rpc(host='localhost', port=9091, timeout=30): +def wait_for_transmission_rpc(host="localhost", port=9091, timeout=30): start = time.time() while time.time() - start < timeout: try: @@ -164,43 +178,35 @@ def get_previous_ratios(log_file): if not os.path.exists(log_file): return {} ratios = {} - with open(log_file, 'r', encoding='utf-8') as f: + with open(log_file, "r", encoding="utf-8") as f: lines = f.readlines() # Only parse the latest ratio segment between the last start and end markers. - start_marker = '[ratio] RATIOS START' - end_marker = '[ratio] RATIOS END' - start_index = None - end_index = None - - for idx, line in enumerate(lines): - if start_marker in line: - start_index = idx - end_index = None - elif start_index is not None and end_marker in line: - end_index = idx - - if start_index is not None: - if end_index is not None and end_index > start_index: - lines = lines[start_index + 1:end_index] - else: - lines = lines[start_index + 1:] + start_marker = "[ratio] RATIOS START" + end_marker = "[ratio] RATIOS END" + starts = [idx for idx, line in enumerate(lines) if start_marker in line] + if starts: + lines = lines[starts[-1] + 1 :] + ends = [idx for idx, line in enumerate(lines) if end_marker in line] + if ends: + lines = lines[: ends[-1]] for line in lines: - match = re.search(r'\[ratio\]\s+(.+?)\s+→\s+(\d+\.\d+)', line) + match = re.search(r"\[ratio\]\s+(.+?)\s+→\s+(\d+\.\d+)", line) if match: name = match.group(1).strip() ratio = float(match.group(2)) ratios[name] = ratio return ratios + distro_patterns = { - 'ubuntu': re.compile(r'^ubuntu-|^lbuntu-|^xbuntu-'), - 'debian': re.compile(r'^debian-'), - 'kali': re.compile(r'^kali-linux-'), - 'arch': re.compile(r'^archlinux-'), - 'mint': re.compile(r'^linuxmint-'), - 'fedora': re.compile(r'^Fedora-Workstation-Live-'), + "ubuntu": re.compile(r"^ubuntu-|^lbuntu-|^xbuntu-"), + "debian": re.compile(r"^debian-"), + "kali": re.compile(r"^kali-linux-"), + "arch": re.compile(r"^archlinux-"), + "mint": re.compile(r"^linuxmint-"), + "fedora": re.compile(r"^Fedora-Workstation-Live-"), # devuan has no entry here deliberately: it publishes a single # multi-file torrent per release (see fetch_devuan_latest()), so its # name (e.g. "devuan_excalibur") carries no dotted version string for @@ -215,12 +221,12 @@ def get_previous_ratios(log_file): # instead of distro_patterns above wherever Devuan needs identifying without # a dotted version to parse: should_fetch_torrent()'s ratio gate and # _old_version_candidates()'s cleanup grouping. -DEVUAN_NAME_PATTERN = re.compile(r'^devuan_') +DEVUAN_NAME_PATTERN = re.compile(r"^devuan_") # Every distro fetch_torrents.py knows how to fetch, for FETCH_TORRENTS_DISTROS # validation - deliberately not the same set as distro_patterns above (see # the devuan comment there). -ALL_DISTROS = tuple(distro_patterns.keys()) + ('devuan',) +ALL_DISTROS = tuple(distro_patterns.keys()) + ("devuan",) # Distros fetched when FETCH_TORRENTS_DISTROS is unset. Arch and Devuan are # opt-in only - see tasks/01-demand-checks-for-default-distros.md: @@ -230,15 +236,16 @@ def get_previous_ratios(log_file): # - Devuan bundles every edition/arch into one multi-file torrent under a # single info-hash, so any demand check only ever reads the swarm as a # whole, never which of the bundled files anyone actually wants. -OPT_IN_DISTROS = ('arch', 'devuan') +OPT_IN_DISTROS = ("arch", "devuan") DEFAULT_DISTROS = tuple(d for d in ALL_DISTROS if d not in OPT_IN_DISTROS) -def parse_supported_distros(env_var='FETCH_TORRENTS_DISTROS'): - value = os.getenv(env_var, '').strip() + +def parse_supported_distros(env_var="FETCH_TORRENTS_DISTROS"): + value = os.getenv(env_var, "").strip() if not value: return list(DEFAULT_DISTROS) - requested = [entry.strip().lower() for entry in value.split(',') if entry.strip()] + requested = [entry.strip().lower() for entry in value.split(",") if entry.strip()] valid = [d for d in ALL_DISTROS if d in requested] invalid = [entry for entry in requested if entry not in ALL_DISTROS] @@ -282,12 +289,14 @@ def parse_supported_distros(env_var='FETCH_TORRENTS_DISTROS'): # # FETCH_TORRENTS_INCLUDE_LOW_DEMAND opts back into all of the above. LOW_DEMAND_PATTERN = re.compile( - r'(cloud-genericcloud|^kali-linux-\d+\.\d+-installer-netinst-)', re.IGNORECASE + r"(cloud-genericcloud|^kali-linux-\d+\.\d+-installer-netinst-)", re.IGNORECASE ) + def is_low_demand_variant(name): return bool(LOW_DEMAND_PATTERN.search(name)) + def filter_low_demand(torrents, include_low_demand): if include_low_demand: return dict(torrents) @@ -296,58 +305,63 @@ def filter_low_demand(torrents, include_low_demand): if is_low_demand_variant(name): logger.info( "Skipping %s – low-demand image family (set " - "FETCH_TORRENTS_INCLUDE_LOW_DEMAND=true to include).", name, + "FETCH_TORRENTS_INCLUDE_LOW_DEMAND=true to include).", + name, ) else: kept[name] = url return kept + def get_distro(name): for distro, pattern in distro_patterns.items(): if pattern.match(name): return distro return None + def version_to_tuple(v): try: - return tuple(map(int, v.split('.'))) + return tuple(map(int, v.split("."))) except ValueError as exc: raise ValueError(f"Invalid version string '{v}'") from exc + def parse_version_type(name, distro): - if distro == 'ubuntu': - parts = name.split('-') + if distro == "ubuntu": + parts = name.split("-") prefix = parts[0] version = parts[1] - suffix = '-'.join(parts[2:]) + suffix = "-".join(parts[2:]) type_ = f"{prefix}-{suffix}" - elif distro == 'debian': - parts = name.split('-') + elif distro == "debian": + parts = name.split("-") version = parts[1] arch = parts[2] - type_suffix = '-'.join(parts[3:]) + type_suffix = "-".join(parts[3:]) type_ = f"{arch}-{type_suffix}" - elif distro == 'kali': - parts = name.split('-') + elif distro == "kali": + parts = name.split("-") version = parts[2] - type_ = '-'.join(parts[3:]) - elif distro == 'arch': - parts = name.split('-') + type_ = "-".join(parts[3:]) + elif distro == "arch": + parts = name.split("-") version = parts[1] - type_ = '' - elif distro == 'mint': - parts = name.split('-') + type_ = "" + elif distro == "mint": + parts = name.split("-") version = parts[1] - type_ = '-'.join(parts[2:]) - elif distro == 'fedora': - parts = name.split('-') + type_ = "-".join(parts[2:]) + elif distro == "fedora": + parts = name.split("-") version = parts[-1] type_ = parts[-2] else: - version = '' - type_ = '' + version = "" + type_ = "" return version, type_ + def _should_fetch_devuan(name, ratios, removed_history): """Devuan bundles every edition/arch into one multi-file torrent per release (see fetch_devuan_latest()), so there's no per-name version to @@ -361,11 +375,14 @@ def _should_fetch_devuan(name, ratios, removed_history): if name in ratios: return True # already holding this exact release - prior_ratios = [r for n, r in ratios.items() if n != name and DEVUAN_NAME_PATTERN.match(n)] + prior_ratios = [ + r for n, r in ratios.items() if n != name and DEVUAN_NAME_PATTERN.match(n) + ] if not prior_ratios: prior_ratios = [ - record['ratio'] for n, record in removed_history.items() - if n != name and DEVUAN_NAME_PATTERN.match(n) and 'ratio' in record + record["ratio"] + for n, record in removed_history.items() + if n != name and DEVUAN_NAME_PATTERN.match(n) and "ratio" in record ] if not prior_ratios: @@ -383,7 +400,7 @@ def should_fetch_torrent(name, ratios, removed_history=None): below - bypassing the ratio gate for exactly the versions cleanup already gave up on. Live ratios still take priority when a name somehow appears in both.""" - if os.getenv('SKIP_RATIO_CHECK', 'false').lower() == 'true': + if os.getenv("SKIP_RATIO_CHECK", "false").lower() == "true": return True distro = get_distro(name) if not distro: @@ -410,19 +427,25 @@ def should_fetch_torrent(name, ratios, removed_history=None): if t == type_: type_ratios[version_to_tuple(v_str)] = r except Exception as exc: - logger.warning("Skipping stored ratio entry %s due to parse error: %s", n, exc) + logger.warning( + "Skipping stored ratio entry %s due to parse error: %s", n, exc + ) for n, record in removed_history.items(): - if n in ratios or 'ratio' not in record: + if n in ratios or "ratio" not in record: continue d = get_distro(n) if d == distro: try: v_str, t = parse_version_type(n, d) if t == type_: - type_ratios.setdefault(version_to_tuple(v_str), record['ratio']) + type_ratios.setdefault(version_to_tuple(v_str), record["ratio"]) except Exception as exc: - logger.warning("Skipping removed-history ratio entry %s due to parse error: %s", n, exc) + logger.warning( + "Skipping removed-history ratio entry %s due to parse error: %s", + n, + exc, + ) if not type_ratios: return True # no previous, fetch @@ -433,8 +456,16 @@ def should_fetch_torrent(name, ratios, removed_history=None): prev_max = max(prev_versions) return type_ratios[prev_max] >= 1.0 -def format_run_summary(elapsed, success_count, existing_count, not_found_count, failure_count, - old_added=None, old_skipped=None): + +def format_run_summary( + elapsed, + success_count, + existing_count, + not_found_count, + failure_count, + old_added=None, + old_skipped=None, +): """The single end-of-run log line. old_added/old_skipped are only non-None when FETCH_TORRENTS_CHECK_OLD_RELEASES ran, so the count of torrents that flag introduced is visible at a glance rather than only @@ -456,7 +487,7 @@ def _safe_watch_path(name): the check here means a future looser fetch_*() regex (parsing scraped HTML/JSON from upstream) can't turn a crafted name into a write/delete outside watch_dir.""" - if not name or name in ('.', '..') or '/' in name or '\\' in name: + if not name or name in (".", "..") or "/" in name or "\\" in name: return None return os.path.join(watch_dir, f"{name}.torrent") @@ -489,17 +520,20 @@ def download_torrent(name, url): logger.error(f"Failed to download {url}: {e}") return "failed" + def fetch_ubuntu_lts(): url = "https://releases.ubuntu.com/" try: - text = requests.get("https://changelogs.ubuntu.com/meta-release-lts", timeout=30).text - blocks = [b for b in text.strip().split("\n\n") if "Supported: 1" in b] + text = requests.get( + "https://changelogs.ubuntu.com/meta-release-lts", timeout=30 + ).text + blocks = [b for b in text.strip().split("\n\n") if "Supported: 1" in b] results = {} # newest first (optional – remove reversed() if order is irrelevant) for block in reversed(blocks): - version = re.search(r"Version:\s*([\d.]+)", block).group(1) - codename = re.search(r"Dist:\s*(\w+)", block).group(1) + version = re.search(r"Version:\s*([\d.]+)", block).group(1) + codename = re.search(r"Dist:\s*(\w+)", block).group(1) torrent_urls = [ f"https://releases.ubuntu.com/{codename}/ubuntu-{version}-desktop-amd64.iso.torrent", @@ -520,13 +554,14 @@ def fetch_ubuntu_lts(): logger.error(f"Ubuntu fetch error: {e}") return False + def fetch_debian_stable(): urls = [ - "https://cdimage.debian.org/debian-cd/current/amd64/bt-dvd/", - "https://cdimage.debian.org/debian-cd/current/arm64/bt-dvd/", - "https://cdimage.debian.org/debian-cd/current/amd64/bt-cd/", - "https://cdimage.debian.org/debian-cd/current/arm64/bt-cd/" - ] + "https://cdimage.debian.org/debian-cd/current/amd64/bt-dvd/", + "https://cdimage.debian.org/debian-cd/current/arm64/bt-dvd/", + "https://cdimage.debian.org/debian-cd/current/amd64/bt-cd/", + "https://cdimage.debian.org/debian-cd/current/arm64/bt-cd/", + ] results = {} for url in urls: @@ -534,9 +569,9 @@ def fetch_debian_stable(): r = requests.get(url, timeout=30) r.raise_for_status() soup = BeautifulSoup(r.text, "html.parser") - - for link in soup.find_all('a', href=True): - href = link['href'] + + for link in soup.find_all("a", href=True): + href = link["href"] if ".iso.torrent" in href: torrent_url = url + href # Keep the .iso extension so this matches what Transmission @@ -545,15 +580,16 @@ def fetch_debian_stable(): results[name] = torrent_url break else: - logger.warning("No Debian DVD-1 torrent found.") - + logger.warning("No Debian DVD-1 torrent found.") + except Exception as e: logger.error(f"Debian fetch error: {e}") return results + def _kali_torrent_urls(ver): - base_cd = f"https://cdimage.kali.org/kali-{ver}/kali-linux-{ver}-installer" + base_cd = f"https://cdimage.kali.org/kali-{ver}/kali-linux-{ver}-installer" base_arm = f"https://kali.download/arm-images/kali-{ver}/kali-linux-{ver}" base_cloud = f"https://kali.download/cloud-images/kali-{ver}/kali-linux-{ver}-cloud-genericcloud" return [ @@ -563,11 +599,9 @@ def _kali_torrent_urls(ver): f"{base_cd}-arm64.iso.torrent", f"{base_cd}-netinst-arm64.iso.torrent", f"{base_cd}-purple-amd64.iso.torrent", - f"{base_arm}-raspberry-pi-armhf.img.xz.torrent", f"{base_arm}-raspberry-pi-zero-2-w-armhf.img.xz.torrent", f"{base_arm}-raspberry-pi-zero-w-armel.img.xz.torrent", - f"{base_cloud}-amd64.tar.xz.torrent", f"{base_cloud}-arm64.tar.xz.torrent", ] @@ -614,8 +648,8 @@ def fetch_kali_old_versions(): soup = BeautifulSoup(html, "html.parser") versions = [] - for link in soup.find_all('a', href=True): - match = re.match(r'^kali-(\d+\.\d+)/$', link['href']) + for link in soup.find_all("a", href=True): + match = re.match(r"^kali-(\d+\.\d+)/$", link["href"]) if match: versions.append(match.group(1)) @@ -636,6 +670,7 @@ def fetch_kali_old_versions(): logger.error("Kali old-release fetch error: %s", exc) return {} + def fetch_arch_latest(): """Opt-in (FETCH_TORRENTS_DISTROS=arch,...): Arch's official torrent publishes no tracker at all - only DHT and ~400 HTTP webseeds - so real @@ -662,7 +697,7 @@ def fetch_arch_latest(): link = row.find("a", href=re.compile(torrent_url_pattern)) if not link: continue - href = link['href'] + href = link["href"] version = re.sub(torrent_url_pattern, "\\1", href) candidates.append((version, href)) @@ -704,6 +739,7 @@ def fetch_devuan_latest(): logger.error(f"Devuan fetch error: {e}") return False + def fetch_linuxmint_cinnamon(): url = "https://www.linuxmint.com/download.php" try: @@ -724,6 +760,7 @@ def fetch_linuxmint_cinnamon(): logger.error(f"Linux Mint fetch error: {e}") return False + def fetch_fedora_workstation(): url = "https://torrent.fedoraproject.org/torrents/" try: @@ -731,15 +768,17 @@ def fetch_fedora_workstation(): r.raise_for_status() soup = BeautifulSoup(r.text, "html.parser") - pattern = re.compile(r"^Fedora-Workstation-Live-([A-Za-z0-9_]+)-(\d+)\.torrent$") + pattern = re.compile( + r"^Fedora-Workstation-Live-([A-Za-z0-9_]+)-(\d+)\.torrent$" + ) latest = {} - for link in soup.find_all('a', href=True): - match = pattern.match(link['href']) + for link in soup.find_all("a", href=True): + match = pattern.match(link["href"]) if not match: continue arch, version = match.group(1), int(match.group(2)) if arch not in latest or version > latest[arch][0]: - latest[arch] = (version, link['href']) + latest[arch] = (version, link["href"]) if not latest: logger.warning("No Fedora Workstation torrents found on %s", url) @@ -770,19 +809,23 @@ def fetch_fedora_workstation_old_versions(): r.raise_for_status() soup = BeautifulSoup(r.text, "html.parser") - pattern = re.compile(r"^Fedora-Workstation-Live-([A-Za-z0-9_]+)-(\d+)\.torrent$") + pattern = re.compile( + r"^Fedora-Workstation-Live-([A-Za-z0-9_]+)-(\d+)\.torrent$" + ) by_arch = {} - for link in soup.find_all('a', href=True): - match = pattern.match(link['href']) + for link in soup.find_all("a", href=True): + match = pattern.match(link["href"]) if not match: continue arch, version = match.group(1), int(match.group(2)) - by_arch.setdefault(arch, []).append((version, link['href'])) + by_arch.setdefault(arch, []).append((version, link["href"])) results = {} for versions in by_arch.values(): versions.sort(key=lambda entry: entry[0], reverse=True) - for _version, href in versions[1:]: # newest is fetch_fedora_workstation()'s job + for _version, href in versions[ + 1: + ]: # newest is fetch_fedora_workstation()'s job name = href.replace(".torrent", "") results[name] = url + href return results @@ -792,7 +835,9 @@ def fetch_fedora_workstation_old_versions(): return {} -def has_unmet_demand(scrape_result, min_leechers, min_leecher_ratio, allow_zero_seeders): +def has_unmet_demand( + scrape_result, min_leechers, min_leecher_ratio, allow_zero_seeders +): """True if a live tracker scrape shows real, currently-unserved demand: at least min_leechers leechers in absolute terms (so a single leecher never justifies a download on its own), AND at least min_leecher_ratio @@ -816,16 +861,18 @@ def has_unmet_demand(scrape_result, min_leechers, min_leecher_ratio, allow_zero_ scraping failed.""" if not scrape_result: return False - leechers = scrape_result.get('leechers', 0) + leechers = scrape_result.get("leechers", 0) if leechers < min_leechers: return False - seeders = scrape_result.get('seeders', 0) + seeders = scrape_result.get("seeders", 0) if seeders == 0: return allow_zero_seeders return leechers >= min_leecher_ratio * seeders -def evaluate_old_release_demand(name, url, min_leechers, min_leecher_ratio, allow_zero_seeders, timeout=15): +def evaluate_old_release_demand( + name, url, min_leechers, min_leecher_ratio, allow_zero_seeders, timeout=15 +): """Downloads name's .torrent metadata and scrapes its trackers for live demand, without adding it to Transmission. Returns (keep, torrent_bytes, scrape_result): keep is True once has_unmet_demand() passes; @@ -853,7 +900,9 @@ def evaluate_old_release_demand(name, url, min_leechers, min_leecher_ratio, allo finally: os.remove(tmp.name) - keep = has_unmet_demand(scrape_result, min_leechers, min_leecher_ratio, allow_zero_seeders) + keep = has_unmet_demand( + scrape_result, min_leechers, min_leecher_ratio, allow_zero_seeders + ) return keep, torrent_bytes, scrape_result @@ -861,16 +910,20 @@ def load_old_release_check_state(path): if not os.path.exists(path): return {} try: - with open(path, 'r', encoding='utf-8') as f: + with open(path, "r", encoding="utf-8") as f: return json.load(f) except (json.JSONDecodeError, OSError) as exc: - logger.warning("Could not read old-release check state at %s (%s) - starting fresh.", path, exc) + logger.warning( + "Could not read old-release check state at %s (%s) - starting fresh.", + path, + exc, + ) return {} def save_old_release_check_state(path, state): tmp_path = f"{path}.tmp" - with open(tmp_path, 'w', encoding='utf-8') as f: + with open(tmp_path, "w", encoding="utf-8") as f: json.dump(state, f, indent=2, sort_keys=True) os.replace(tmp_path, path) @@ -883,7 +936,7 @@ def should_recheck_old_release(name, check_state, today, recheck_interval_days): record = check_state.get(name) if record is None: return True - last_checked = date.fromisoformat(record['last_checked']) + last_checked = date.fromisoformat(record["last_checked"]) return (today - last_checked).days >= recheck_interval_days @@ -891,16 +944,20 @@ def load_removed_history(path): if not os.path.exists(path): return {} try: - with open(path, 'r', encoding='utf-8') as f: + with open(path, "r", encoding="utf-8") as f: return json.load(f) except (json.JSONDecodeError, OSError) as exc: - logger.warning("Could not read removed-torrent history at %s (%s) - starting fresh.", path, exc) + logger.warning( + "Could not read removed-torrent history at %s (%s) - starting fresh.", + path, + exc, + ) return {} def save_removed_history(path, history): tmp_path = f"{path}.tmp" - with open(tmp_path, 'w', encoding='utf-8') as f: + with open(tmp_path, "w", encoding="utf-8") as f: json.dump(history, f, indent=2, sort_keys=True) os.replace(tmp_path, path) @@ -919,7 +976,11 @@ def record_removal(history, name, today, reason, ratio): Transmission stops reporting this (now-deleted) torrent - see its removed_history fallback.""" updated = dict(history) - updated[name] = {'removed_date': today.isoformat(), 'reason': reason, 'ratio': round(float(ratio), 3)} + updated[name] = { + "removed_date": today.isoformat(), + "reason": reason, + "ratio": round(float(ratio), 3), + } return updated @@ -931,14 +992,22 @@ def record_removal(history, name, today, reason, ratio): # check - no separate discovery needed. Debian and Mint don't expose a # similarly easy back-catalog with the current fetch approach. OLD_RELEASE_DISCOVERY = { - 'kali': fetch_kali_old_versions, - 'fedora': fetch_fedora_workstation_old_versions, + "kali": fetch_kali_old_versions, + "fedora": fetch_fedora_workstation_old_versions, } -def check_old_releases_for_demand(selected_distros, include_low_demand, min_leechers, min_leecher_ratio, - allow_zero_seeders, check_state=None, removed_history=None, today=None, - recheck_interval_days=7): +def check_old_releases_for_demand( + selected_distros, + include_low_demand, + min_leechers, + min_leecher_ratio, + allow_zero_seeders, + check_state=None, + removed_history=None, + today=None, + recheck_interval_days=7, +): """FETCH_TORRENTS_CHECK_OLD_RELEASES: for distros with old-release discovery, scrape each not-yet-seeded candidate's trackers and add it to the watch dir only if there's live unmet demand - no need to leech it @@ -984,22 +1053,32 @@ def check_old_releases_for_demand(selected_distros, include_low_demand, min_leec if name in removed_history: logger.info( "Skipping old release %s - previously removed on %s, not re-adding automatically.", - name, removed_history[name]['removed_date'], + name, + removed_history[name]["removed_date"], ) skipped += 1 continue - if not should_recheck_old_release(name, check_state, today, recheck_interval_days): - logger.debug("Skipping old release %s - checked recently, not due for recheck.", name) + if not should_recheck_old_release( + name, check_state, today, recheck_interval_days + ): + logger.debug( + "Skipping old release %s - checked recently, not due for recheck.", + name, + ) skipped += 1 continue keep, torrent_bytes, scrape_result = evaluate_old_release_demand( - name, url, min_leechers, min_leecher_ratio, allow_zero_seeders, + name, + url, + min_leechers, + min_leecher_ratio, + allow_zero_seeders, ) check_state[name] = { - 'last_checked': today.isoformat(), - 'leechers': scrape_result['leechers'] if scrape_result else None, + "last_checked": today.isoformat(), + "leechers": scrape_result["leechers"] if scrape_result else None, } if keep: @@ -1007,33 +1086,48 @@ def check_old_releases_for_demand(selected_distros, include_low_demand, min_leec f.write(torrent_bytes) logger.info( "Found unmet demand on old release %s (%d leechers via %s) - added.", - name, scrape_result['leechers'], scrape_result['tracker'], + name, + scrape_result["leechers"], + scrape_result["tracker"], ) added += 1 else: - if scrape_result is not None and scrape_result.get('seeders', 0) == 0 and not allow_zero_seeders: + if ( + scrape_result is not None + and scrape_result.get("seeders", 0) == 0 + and not allow_zero_seeders + ): logger.info( "Skipping old release %s - %d leechers but 0 seeders via %s (no verified-complete copy " "in the swarm; set FETCH_TORRENTS_OLD_RELEASE_ALLOW_ZERO_SEEDERS=true to gamble on it).", - name, scrape_result['leechers'], scrape_result['tracker'], + name, + scrape_result["leechers"], + scrape_result["tracker"], ) elif scrape_result is not None: logger.info( "Skipping old release %s - %d leechers / %d seeders via %s " "(need >= %d leechers and >= %.1fx as many leechers as seeders).", - name, scrape_result['leechers'], scrape_result.get('seeders', 0), - scrape_result['tracker'], min_leechers, min_leecher_ratio, + name, + scrape_result["leechers"], + scrape_result.get("seeders", 0), + scrape_result["tracker"], + min_leechers, + min_leecher_ratio, ) elif torrent_bytes is not None: logger.info( - "Skipping old release %s - could not determine live demand from any tracker.", name, + "Skipping old release %s - could not determine live demand from any tracker.", + name, ) skipped += 1 return added, skipped, check_state -def log_seed_ratios_via_http(rpc_url="http://localhost:9091/transmission/rpc", auth: tuple | None = None): +def log_seed_ratios_via_http( + rpc_url="http://localhost:9091/transmission/rpc", auth: tuple | None = None +): if auth is None: username, password = get_rpc_credentials() auth = (username, password) if username and password else None @@ -1043,7 +1137,7 @@ def log_seed_ratios_via_http(rpc_url="http://localhost:9091/transmission/rpc", a headers = {"X-Transmission-Session-Id": r.headers["X-Transmission-Session-Id"]} payload = { "method": "torrent-get", - "arguments": {"fields": ["name", "uploadRatio"]} + "arguments": {"fields": ["name", "uploadRatio"]}, } r = requests.post(rpc_url, json=payload, headers=headers, auth=auth, timeout=15) r.raise_for_status() @@ -1063,16 +1157,17 @@ def log_seed_ratios_via_http(rpc_url="http://localhost:9091/transmission/rpc", a logger.info("[ratio] RATIOS END") logger.info("") + def _devuan_sort_key(torrent): """Devuan releases carry no dotted version in their name to compare (see DEVUAN_NAME_PATTERN), so _old_version_candidates() orders them by when Transmission added them instead - the most-recently-added one is treated as the current release, the same role the highest parsed version plays for every other distro.""" - added = getattr(torrent, 'added_date', None) + added = getattr(torrent, "added_date", None) if added is None: return 0.0 - if hasattr(added, 'timestamp'): + if hasattr(added, "timestamp"): return added.timestamp() return float(added) @@ -1093,7 +1188,9 @@ def _old_version_candidates(torrents): groups = {} for torrent in torrents: if DEVUAN_NAME_PATTERN.match(torrent.name): - groups.setdefault(('devuan', ''), []).append((_devuan_sort_key(torrent), torrent)) + groups.setdefault(("devuan", ""), []).append( + (_devuan_sort_key(torrent), torrent) + ) continue distro = get_distro(torrent.name) if not distro: @@ -1102,7 +1199,9 @@ def _old_version_candidates(torrents): version_str, type_ = parse_version_type(torrent.name, distro) version = version_to_tuple(version_str) except Exception as exc: - logger.warning("Skipping %s during cleanup due to parse error: %s", torrent.name, exc) + logger.warning( + "Skipping %s during cleanup due to parse error: %s", torrent.name, exc + ) continue groups.setdefault((distro, type_), []).append((version, torrent)) @@ -1115,6 +1214,7 @@ def _old_version_candidates(torrents): candidates.extend(torrent for _version, torrent in entries[1:]) return candidates + def _remove_watch_file(name): """Best-effort: once cleanup deletes a torrent's downloaded data, also remove its leftover .torrent file from /watch - otherwise these @@ -1136,7 +1236,7 @@ def plan_cleanup(torrents, skip_ratio_check=False, min_ratio=1.0): to_remove = [] to_keep_low_ratio = [] for torrent in _old_version_candidates(torrents): - ratio = float(getattr(torrent, 'ratio', 0.0) or 0.0) + ratio = float(getattr(torrent, "ratio", 0.0) or 0.0) if not skip_ratio_check and ratio < min_ratio: to_keep_low_ratio.append((torrent, ratio)) else: @@ -1144,23 +1244,26 @@ def plan_cleanup(torrents, skip_ratio_check=False, min_ratio=1.0): return to_remove, to_keep_low_ratio + def cleanup_old_versions(): """Space-constrained option (CLEANUP_KEEP_ONLY_LATEST_VERSION=true): remove superseded versions as soon as they clear the ratio floor, regardless of whether anyone's still downloading them.""" username, password = get_rpc_credentials() - tc = Client(host='localhost', port=9091, username=username, password=password) + tc = Client(host="localhost", port=9091, username=username, password=password) torrents = tc.get_torrents() - skip_ratio_check = parse_bool('CLEANUP_SKIP_RATIO_CHECK', False) - min_ratio = float(os.getenv('CLEANUP_MIN_RATIO', '1.0')) + skip_ratio_check = parse_bool("CLEANUP_SKIP_RATIO_CHECK", False) + min_ratio = float(os.getenv("CLEANUP_MIN_RATIO", "1.0")) to_remove, to_keep_low_ratio = plan_cleanup(torrents, skip_ratio_check, min_ratio) for torrent, ratio in to_keep_low_ratio: logger.info( "Keeping old version %s – ratio %.3f is below cleanup threshold %.3f.", - torrent.name, ratio, min_ratio, + torrent.name, + ratio, + min_ratio, ) removed_history = load_removed_history(removed_history_file) @@ -1170,8 +1273,11 @@ def cleanup_old_versions(): tc.remove_torrent(torrent.id, delete_data=True) _remove_watch_file(torrent.name) removed_history = record_removal( - removed_history, torrent.name, today, 'keep_only_latest', - float(getattr(torrent, 'ratio', 0.0) or 0.0), + removed_history, + torrent.name, + today, + "keep_only_latest", + float(getattr(torrent, "ratio", 0.0) or 0.0), ) save_removed_history(removed_history_file, removed_history) @@ -1189,7 +1295,7 @@ def plan_disk_pressure_cleanup(torrents, used_percent, threshold_percent): if threshold_percent <= 0 or used_percent < threshold_percent: return [] candidates = _old_version_candidates(torrents) - candidates.sort(key=lambda t: float(getattr(t, 'ratio', 0.0) or 0.0)) + candidates.sort(key=lambda t: float(getattr(t, "ratio", 0.0) or 0.0)) return candidates @@ -1206,14 +1312,16 @@ def enforce_disk_usage_limit(threshold_percent, download_path="/downloads"): return username, password = get_rpc_credentials() - tc = Client(host='localhost', port=9091, username=username, password=password) + tc = Client(host="localhost", port=9091, username=username, password=password) torrents = tc.get_torrents() candidates = plan_disk_pressure_cleanup(torrents, used_percent, threshold_percent) if not candidates: logger.warning( "Downloads folder usage is %.1f%% (>= %.1f%% threshold) but no superseded " - "versions are available to free space.", used_percent, threshold_percent, + "versions are available to free space.", + used_percent, + threshold_percent, ) return @@ -1224,14 +1332,19 @@ def enforce_disk_usage_limit(threshold_percent, download_path="/downloads"): used_percent = (used / total * 100) if total else 0.0 if used_percent < threshold_percent: break - ratio = float(getattr(torrent, 'ratio', 0.0) or 0.0) + ratio = float(getattr(torrent, "ratio", 0.0) or 0.0) logger.warning( "Downloads folder usage %.1f%% >= %.1f%% threshold - removing %s (ratio %.3f) to free space.", - used_percent, threshold_percent, torrent.name, ratio, + used_percent, + threshold_percent, + torrent.name, + ratio, ) tc.remove_torrent(torrent.id, delete_data=True) _remove_watch_file(torrent.name) - removed_history = record_removal(removed_history, torrent.name, today, 'disk_pressure', ratio) + removed_history = record_removal( + removed_history, torrent.name, today, "disk_pressure", ratio + ) save_removed_history(removed_history_file, removed_history) @@ -1242,16 +1355,18 @@ def load_ratio_history(path): if not os.path.exists(path): return {} try: - with open(path, 'r', encoding='utf-8') as f: + with open(path, "r", encoding="utf-8") as f: return json.load(f) except (json.JSONDecodeError, OSError) as exc: - logger.warning("Could not read ratio history at %s (%s) – starting fresh.", path, exc) + logger.warning( + "Could not read ratio history at %s (%s) – starting fresh.", path, exc + ) return {} def save_ratio_history(path, history): tmp_path = f"{path}.tmp" - with open(tmp_path, 'w', encoding='utf-8') as f: + with open(tmp_path, "w", encoding="utf-8") as f: json.dump(history, f, indent=2, sort_keys=True) os.replace(tmp_path, path) @@ -1262,16 +1377,27 @@ def save_ratio_history(path, history): # taken every RATIO_HISTORY_SAMPLE_INTERVAL_DAYS, pruned once older than # window_days + the interval. That's a fixed handful of small numbers per # torrent forever, not one entry per run. -def update_ratio_history(history, current_ratios, today, sample_interval_days=RATIO_HISTORY_SAMPLE_INTERVAL_DAYS, window_days=30): +def update_ratio_history( + history, + current_ratios, + today, + sample_interval_days=RATIO_HISTORY_SAMPLE_INTERVAL_DAYS, + window_days=30, +): max_age_days = window_days + sample_interval_days updated = {} for name, ratio in current_ratios.items(): samples = list(history.get(name, [])) - if not samples or (today - date.fromisoformat(samples[-1]['date'])).days >= sample_interval_days: - samples.append({'date': today.isoformat(), 'ratio': ratio}) + if ( + not samples + or (today - date.fromisoformat(samples[-1]["date"])).days + >= sample_interval_days + ): + samples.append({"date": today.isoformat(), "ratio": ratio}) updated[name] = [ - s for s in samples - if (today - date.fromisoformat(s['date'])).days <= max_age_days + s + for s in samples + if (today - date.fromisoformat(s["date"])).days <= max_age_days ] return updated @@ -1280,13 +1406,19 @@ def _stagnation_anchor(samples, today, window_days): """Ratio of the oldest recorded sample that's at least window_days old, or None if we don't have one yet - i.e. this torrent is too new to judge either way.""" - old_enough = [s for s in samples if (today - date.fromisoformat(s['date'])).days >= window_days] + old_enough = [ + s + for s in samples + if (today - date.fromisoformat(s["date"])).days >= window_days + ] if not old_enough: return None - return min(old_enough, key=lambda s: s['date'])['ratio'] + return min(old_enough, key=lambda s: s["date"])["ratio"] -def plan_stagnation_cleanup(torrents, history, today, window_days=30, min_ratio_delta=0.02): +def plan_stagnation_cleanup( + torrents, history, today, window_days=30, min_ratio_delta=0.02 +): """Default cleanup path: among superseded (non-latest) versions per (distro, type) group, remove ones whose ratio has grown less than min_ratio_delta over the last window_days - nobody's downloading them @@ -1296,7 +1428,7 @@ def plan_stagnation_cleanup(torrents, history, today, window_days=30, min_ratio_ to_remove = [] to_keep = [] for torrent in _old_version_candidates(torrents): - ratio = float(getattr(torrent, 'ratio', 0.0) or 0.0) + ratio = float(getattr(torrent, "ratio", 0.0) or 0.0) anchor = _stagnation_anchor(history.get(torrent.name, []), today, window_days) if anchor is None: to_keep.append((torrent, None)) @@ -1311,43 +1443,58 @@ def plan_stagnation_cleanup(torrents, history, today, window_days=30, min_ratio_ def cleanup_stagnant_torrents(): username, password = get_rpc_credentials() - tc = Client(host='localhost', port=9091, username=username, password=password) + tc = Client(host="localhost", port=9091, username=username, password=password) torrents = tc.get_torrents() - window_days = int(os.getenv('CLEANUP_STAGNATION_WINDOW_DAYS', '30')) - min_ratio_delta = float(os.getenv('CLEANUP_STAGNATION_MIN_RATIO_DELTA', '0.02')) + window_days = int(os.getenv("CLEANUP_STAGNATION_WINDOW_DAYS", "30")) + min_ratio_delta = float(os.getenv("CLEANUP_STAGNATION_MIN_RATIO_DELTA", "0.02")) today = date.today() history = load_ratio_history(ratio_history_file) - current_ratios = {t.name: float(getattr(t, 'ratio', 0.0) or 0.0) for t in torrents} - history = update_ratio_history(history, current_ratios, today, window_days=window_days) + current_ratios = {t.name: float(getattr(t, "ratio", 0.0) or 0.0) for t in torrents} + history = update_ratio_history( + history, current_ratios, today, window_days=window_days + ) - to_remove, to_keep = plan_stagnation_cleanup(torrents, history, today, window_days, min_ratio_delta) + to_remove, to_keep = plan_stagnation_cleanup( + torrents, history, today, window_days, min_ratio_delta + ) for torrent, delta in to_keep: if delta is None: logger.info( "Keeping old version %s – not enough ratio history yet (need %d days).", - torrent.name, window_days, + torrent.name, + window_days, ) else: logger.info( "Keeping old version %s – ratio grew by %.3f over the last %d days.", - torrent.name, delta, window_days, + torrent.name, + delta, + window_days, ) removed_history = load_removed_history(removed_history_file) for torrent in to_remove: logger.info( "Removing stagnant torrent: %s (no meaningful ratio growth in %d days)", - torrent.name, window_days, + torrent.name, + window_days, ) tc.remove_torrent(torrent.id, delete_data=True) _remove_watch_file(torrent.name) - removed_history = record_removal(removed_history, torrent.name, today, 'stagnant', current_ratios[torrent.name]) + removed_history = record_removal( + removed_history, + torrent.name, + today, + "stagnant", + current_ratios[torrent.name], + ) save_removed_history(removed_history_file, removed_history) save_ratio_history(ratio_history_file, history) + if __name__ == "__main__": start_time = time.time() logger.info("Starting torrent fetch run.") @@ -1361,24 +1508,29 @@ def cleanup_stagnant_torrents(): not_found_count = 0 distro_funcs = [ - ('ubuntu', fetch_ubuntu_lts), - ('debian', fetch_debian_stable), - ('kali', fetch_kali_latest), - ('arch', fetch_arch_latest), - ('mint', fetch_linuxmint_cinnamon), - ('fedora', fetch_fedora_workstation), - ('devuan', fetch_devuan_latest), + ("ubuntu", fetch_ubuntu_lts), + ("debian", fetch_debian_stable), + ("kali", fetch_kali_latest), + ("arch", fetch_arch_latest), + ("mint", fetch_linuxmint_cinnamon), + ("fedora", fetch_fedora_workstation), + ("devuan", fetch_devuan_latest), ] selected_distros = parse_supported_distros() logger.info("Selected distros for this run: %s", ", ".join(selected_distros)) - include_low_demand = parse_bool('FETCH_TORRENTS_INCLUDE_LOW_DEMAND', False) - logger.info("Including low-demand image families (arm64/cloud/etc.): %s", include_low_demand) + include_low_demand = parse_bool("FETCH_TORRENTS_INCLUDE_LOW_DEMAND", False) + logger.info( + "Including low-demand image families (arm64/cloud/etc.): %s", include_low_demand + ) for distro, func in distro_funcs: if distro not in selected_distros: - logger.info("Skipping distro %s because it is not enabled by FETCH_TORRENTS_DISTROS.", distro) + logger.info( + "Skipping distro %s because it is not enabled by FETCH_TORRENTS_DISTROS.", + distro, + ) continue logger.info(f"Fetching latest {distro} torrents...") @@ -1397,34 +1549,61 @@ def cleanup_stagnant_torrents(): else: failure_count += 1 else: - logger.info(f"Skipping {name} due to low ratio on previous version.") + logger.info( + f"Skipping {name} due to low ratio on previous version." + ) else: failure_count += 1 old_added = old_skipped = None - check_old_releases = parse_bool('FETCH_TORRENTS_CHECK_OLD_RELEASES', False) + check_old_releases = parse_bool("FETCH_TORRENTS_CHECK_OLD_RELEASES", False) if check_old_releases: - old_release_min_leechers = int(os.getenv('FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHERS', '10')) - old_release_min_leecher_ratio = float(os.getenv('FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHER_RATIO', '0.1')) - old_release_allow_zero_seeders = parse_bool('FETCH_TORRENTS_OLD_RELEASE_ALLOW_ZERO_SEEDERS', False) - old_release_recheck_days = int(os.getenv('FETCH_TORRENTS_OLD_RELEASE_RECHECK_DAYS', '7')) - skip_previously_removed = parse_bool('FETCH_TORRENTS_OLD_RELEASE_SKIP_REMOVED', True) + old_release_min_leechers = int( + os.getenv("FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHERS", "10") + ) + old_release_min_leecher_ratio = float( + os.getenv("FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHER_RATIO", "0.1") + ) + old_release_allow_zero_seeders = parse_bool( + "FETCH_TORRENTS_OLD_RELEASE_ALLOW_ZERO_SEEDERS", False + ) + old_release_recheck_days = int( + os.getenv("FETCH_TORRENTS_OLD_RELEASE_RECHECK_DAYS", "7") + ) + skip_previously_removed = parse_bool( + "FETCH_TORRENTS_OLD_RELEASE_SKIP_REMOVED", True + ) check_state = load_old_release_check_state(old_release_check_state_file) - removed_history = load_removed_history(removed_history_file) if skip_previously_removed else {} + removed_history = ( + load_removed_history(removed_history_file) + if skip_previously_removed + else {} + ) logger.info( "Checking old releases for unmet demand (min leechers: %d, min leecher/seeder ratio: %.1fx, " "allow zero-seeder swarms: %s, recheck interval: %d days)...", - old_release_min_leechers, old_release_min_leecher_ratio, old_release_allow_zero_seeders, + old_release_min_leechers, + old_release_min_leecher_ratio, + old_release_allow_zero_seeders, old_release_recheck_days, ) old_added, old_skipped, check_state = check_old_releases_for_demand( - selected_distros, include_low_demand, old_release_min_leechers, old_release_min_leecher_ratio, - old_release_allow_zero_seeders, check_state, removed_history, date.today(), old_release_recheck_days, + selected_distros, + include_low_demand, + old_release_min_leechers, + old_release_min_leecher_ratio, + old_release_allow_zero_seeders, + check_state, + removed_history, + date.today(), + old_release_recheck_days, ) save_old_release_check_state(old_release_check_state_file, check_state) - logger.info("Old-release check complete: %d added, %d skipped.", old_added, old_skipped) + logger.info( + "Old-release check complete: %d added, %d skipped.", old_added, old_skipped + ) if wait_for_transmission_rpc(): try: @@ -1432,10 +1611,12 @@ def cleanup_stagnant_torrents(): except Exception as exc: logger.error("Could not query Transmission: %s", exc) else: - logger.error("Transmission RPC not available on localhost:9091; skipping ratio logs.") - + logger.error( + "Transmission RPC not available on localhost:9091; skipping ratio logs." + ) + try: - if parse_bool('CLEANUP_KEEP_ONLY_LATEST_VERSION', False): + if parse_bool("CLEANUP_KEEP_ONLY_LATEST_VERSION", False): cleanup_old_versions() else: cleanup_stagnant_torrents() @@ -1443,15 +1624,27 @@ def cleanup_stagnant_torrents(): logger.error("Could not clean up old versions: %s", exc) try: - disk_usage_threshold = float(os.getenv('CLEANUP_DISK_USAGE_THRESHOLD_PERCENT', '95')) + disk_usage_threshold = float( + os.getenv("CLEANUP_DISK_USAGE_THRESHOLD_PERCENT", "95") + ) enforce_disk_usage_limit(disk_usage_threshold) except Exception as exc: logger.error("Could not enforce disk usage limit: %s", exc) total, used, free = shutil.disk_usage("/downloads") - logger.info(f"Downloads folder usage: {used // (2**30)} GB used / {total // (2**30)} GB total") + logger.info( + f"Downloads folder usage: {used // (2**30)} GB used / {total // (2**30)} GB total" + ) elapsed = time.time() - start_time - logger.info(format_run_summary( - elapsed, success_count, existing_count, not_found_count, failure_count, old_added, old_skipped, - )) + logger.info( + format_run_summary( + elapsed, + success_count, + existing_count, + not_found_count, + failure_count, + old_added, + old_skipped, + ) + ) diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 0000000..9991311 --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,61 @@ +"""Shared test helpers. + +Not a test module itself (doesn't match the test*.py discovery pattern); +`unittest discover -s tests` puts tests/ on sys.path, so test modules import +this as `import helpers`. +""" + +import os +import shutil +import tempfile +import unittest.mock + +# Env vars production code reads at call time. A developer who happens to +# have any of these exported (e.g. SKIP_RATIO_CHECK=true) must not be able +# to flip test outcomes, so isolated_env() removes them unless a test sets +# them explicitly. +_ISOLATED_PREFIXES = ( + "SKIP_RATIO_CHECK", + "CLEANUP_", + "TRANSMISSION_", + "FETCH_TORRENTS_", +) +# Read at import time to choose where fetch_torrents writes its logs; the +# test modules set it to a temp dir before importing, so it must survive. +_PRESERVED = frozenset({"FETCH_TORRENTS_LOG_DIR"}) + + +def scrubbed_environ(**overrides): + """A copy of os.environ without the isolated keys, plus overrides.""" + env = { + key: value + for key, value in os.environ.items() + if key in _PRESERVED or not key.startswith(_ISOLATED_PREFIXES) + } + env.update(overrides) + return env + + +def isolated_env(**overrides): + """patch.dict replacing os.environ with scrubbed_environ(**overrides). + + Usable as a context manager, or started/stopped manually (see + use_isolated_env()). The scrubbed copy is taken when this is called, not + when it is entered, so call it at the point of use.""" + return unittest.mock.patch.dict( + os.environ, scrubbed_environ(**overrides), clear=True + ) + + +def use_isolated_env(testcase, **overrides): + """Apply isolated_env() for the rest of a test (call from setUp).""" + patcher = isolated_env(**overrides) + patcher.start() + testcase.addCleanup(patcher.stop) + + +def make_temp_dir(testcase, prefix): + """mkdtemp() that is removed again when the test finishes.""" + path = tempfile.mkdtemp(prefix=prefix) + testcase.addCleanup(shutil.rmtree, path, ignore_errors=True) + return path diff --git a/tests/test_configure_transmission.py b/tests/test_configure_transmission.py index 9292a45..75a0e6f 100644 --- a/tests/test_configure_transmission.py +++ b/tests/test_configure_transmission.py @@ -2,9 +2,9 @@ Run with: python -m unittest discover -s tests """ + import os import sys -import tempfile import unittest REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -12,6 +12,7 @@ sys.path.insert(0, REPO_ROOT) import configure_transmission as ct +import helpers class BuildRpcOverridesTests(unittest.TestCase): @@ -19,56 +20,62 @@ def test_no_relevant_env_vars_yields_no_overrides(self): self.assertEqual(ct.build_rpc_overrides({}), {}) def test_whitelist_env_var_broadens_and_keeps_whitelist_enabled(self): - overrides = ct.build_rpc_overrides({'TRANSMISSION_RPC_WHITELIST': '10.0.0.*'}) + overrides = ct.build_rpc_overrides({"TRANSMISSION_RPC_WHITELIST": "10.0.0.*"}) self.assertEqual( overrides, - {'rpc-whitelist-enabled': True, 'rpc-whitelist': '10.0.0.*'}, + {"rpc-whitelist-enabled": True, "rpc-whitelist": "10.0.0.*"}, ) def test_blank_whitelist_env_var_is_ignored(self): - self.assertEqual(ct.build_rpc_overrides({'TRANSMISSION_RPC_WHITELIST': ' '}), {}) + self.assertEqual( + ct.build_rpc_overrides({"TRANSMISSION_RPC_WHITELIST": " "}), {} + ) def test_username_and_password_enable_authentication(self): - overrides = ct.build_rpc_overrides({ - 'TRANSMISSION_RPC_USERNAME': 'alice', - 'TRANSMISSION_RPC_PASSWORD': 'hunter2', - }) + overrides = ct.build_rpc_overrides( + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + } + ) self.assertEqual( overrides, { - 'rpc-authentication-required': True, - 'rpc-username': 'alice', - 'rpc-password': 'hunter2', + "rpc-authentication-required": True, + "rpc-username": "alice", + "rpc-password": "hunter2", }, ) def test_username_without_password_does_not_enable_authentication(self): - overrides = ct.build_rpc_overrides({'TRANSMISSION_RPC_USERNAME': 'alice'}) + overrides = ct.build_rpc_overrides({"TRANSMISSION_RPC_USERNAME": "alice"}) self.assertEqual(overrides, {}) def test_password_without_username_does_not_enable_authentication(self): - overrides = ct.build_rpc_overrides({'TRANSMISSION_RPC_PASSWORD': 'hunter2'}) + overrides = ct.build_rpc_overrides({"TRANSMISSION_RPC_PASSWORD": "hunter2"}) self.assertEqual(overrides, {}) def test_whitelist_and_auth_can_combine(self): - overrides = ct.build_rpc_overrides({ - 'TRANSMISSION_RPC_WHITELIST': '*', - 'TRANSMISSION_RPC_USERNAME': 'alice', - 'TRANSMISSION_RPC_PASSWORD': 'hunter2', - }) + overrides = ct.build_rpc_overrides( + { + "TRANSMISSION_RPC_WHITELIST": "*", + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + } + ) self.assertEqual( overrides, { - 'rpc-whitelist-enabled': True, - 'rpc-whitelist': '*', - 'rpc-authentication-required': True, - 'rpc-username': 'alice', - 'rpc-password': 'hunter2', + "rpc-whitelist-enabled": True, + "rpc-whitelist": "*", + "rpc-authentication-required": True, + "rpc-username": "alice", + "rpc-password": "hunter2", }, ) @@ -78,44 +85,58 @@ def test_no_relevant_env_vars_yields_no_overrides(self): self.assertEqual(ct.build_peer_limit_overrides({}), {}) def test_global_limit_env_var_sets_override(self): - overrides = ct.build_peer_limit_overrides({'TRANSMISSION_PEER_LIMIT_GLOBAL': '1000'}) + overrides = ct.build_peer_limit_overrides( + {"TRANSMISSION_PEER_LIMIT_GLOBAL": "1000"} + ) - self.assertEqual(overrides, {'peer-limit-global': 1000}) + self.assertEqual(overrides, {"peer-limit-global": 1000}) def test_per_torrent_limit_env_var_sets_override(self): - overrides = ct.build_peer_limit_overrides({'TRANSMISSION_PEER_LIMIT_PER_TORRENT': '300'}) + overrides = ct.build_peer_limit_overrides( + {"TRANSMISSION_PEER_LIMIT_PER_TORRENT": "300"} + ) - self.assertEqual(overrides, {'peer-limit-per-torrent': 300}) + self.assertEqual(overrides, {"peer-limit-per-torrent": 300}) def test_both_limits_can_combine(self): - overrides = ct.build_peer_limit_overrides({ - 'TRANSMISSION_PEER_LIMIT_GLOBAL': '1000', - 'TRANSMISSION_PEER_LIMIT_PER_TORRENT': '300', - }) + overrides = ct.build_peer_limit_overrides( + { + "TRANSMISSION_PEER_LIMIT_GLOBAL": "1000", + "TRANSMISSION_PEER_LIMIT_PER_TORRENT": "300", + } + ) - self.assertEqual(overrides, {'peer-limit-global': 1000, 'peer-limit-per-torrent': 300}) + self.assertEqual( + overrides, {"peer-limit-global": 1000, "peer-limit-per-torrent": 300} + ) def test_blank_env_vars_are_ignored(self): - overrides = ct.build_peer_limit_overrides({ - 'TRANSMISSION_PEER_LIMIT_GLOBAL': ' ', - 'TRANSMISSION_PEER_LIMIT_PER_TORRENT': '', - }) + overrides = ct.build_peer_limit_overrides( + { + "TRANSMISSION_PEER_LIMIT_GLOBAL": " ", + "TRANSMISSION_PEER_LIMIT_PER_TORRENT": "", + } + ) self.assertEqual(overrides, {}) def test_non_numeric_env_vars_are_ignored(self): - overrides = ct.build_peer_limit_overrides({ - 'TRANSMISSION_PEER_LIMIT_GLOBAL': 'unlimited', - 'TRANSMISSION_PEER_LIMIT_PER_TORRENT': 'lots', - }) + overrides = ct.build_peer_limit_overrides( + { + "TRANSMISSION_PEER_LIMIT_GLOBAL": "unlimited", + "TRANSMISSION_PEER_LIMIT_PER_TORRENT": "lots", + } + ) self.assertEqual(overrides, {}) def test_zero_and_negative_values_are_ignored(self): - overrides = ct.build_peer_limit_overrides({ - 'TRANSMISSION_PEER_LIMIT_GLOBAL': '0', - 'TRANSMISSION_PEER_LIMIT_PER_TORRENT': '-5', - }) + overrides = ct.build_peer_limit_overrides( + { + "TRANSMISSION_PEER_LIMIT_GLOBAL": "0", + "TRANSMISSION_PEER_LIMIT_PER_TORRENT": "-5", + } + ) self.assertEqual(overrides, {}) @@ -124,7 +145,7 @@ class BuildPortForwardingOverridesTests(unittest.TestCase): def test_no_env_var_on_first_run_defaults_to_disabled(self): overrides = ct.build_port_forwarding_overrides({}, settings_file_exists=False) - self.assertEqual(overrides, {'port-forwarding-enabled': False}) + self.assertEqual(overrides, {"port-forwarding-enabled": False}) def test_no_env_var_on_later_run_yields_no_override(self): overrides = ct.build_port_forwarding_overrides({}, settings_file_exists=True) @@ -133,28 +154,32 @@ def test_no_env_var_on_later_run_yields_no_override(self): def test_explicit_true_enables_it_even_on_first_run(self): overrides = ct.build_port_forwarding_overrides( - {'TRANSMISSION_PORT_FORWARDING': 'true'}, settings_file_exists=False, + {"TRANSMISSION_PORT_FORWARDING": "true"}, + settings_file_exists=False, ) - self.assertEqual(overrides, {'port-forwarding-enabled': True}) + self.assertEqual(overrides, {"port-forwarding-enabled": True}) def test_explicit_false_disables_it_on_a_later_run(self): overrides = ct.build_port_forwarding_overrides( - {'TRANSMISSION_PORT_FORWARDING': 'false'}, settings_file_exists=True, + {"TRANSMISSION_PORT_FORWARDING": "false"}, + settings_file_exists=True, ) - self.assertEqual(overrides, {'port-forwarding-enabled': False}) + self.assertEqual(overrides, {"port-forwarding-enabled": False}) def test_blank_env_var_on_first_run_falls_back_to_default(self): overrides = ct.build_port_forwarding_overrides( - {'TRANSMISSION_PORT_FORWARDING': ' '}, settings_file_exists=False, + {"TRANSMISSION_PORT_FORWARDING": " "}, + settings_file_exists=False, ) - self.assertEqual(overrides, {'port-forwarding-enabled': False}) + self.assertEqual(overrides, {"port-forwarding-enabled": False}) def test_invalid_env_var_on_later_run_yields_no_override(self): overrides = ct.build_port_forwarding_overrides( - {'TRANSMISSION_PORT_FORWARDING': 'sometimes'}, settings_file_exists=True, + {"TRANSMISSION_PORT_FORWARDING": "sometimes"}, + settings_file_exists=True, ) self.assertEqual(overrides, {}) @@ -162,18 +187,22 @@ def test_invalid_env_var_on_later_run_yields_no_override(self): class WarnIfOpenWithoutAuthTests(unittest.TestCase): def test_warns_when_whitelist_broadened_without_auth(self): - warning = ct.warn_if_open_without_auth({'rpc-whitelist-enabled': True, 'rpc-whitelist': '*'}) + warning = ct.warn_if_open_without_auth( + {"rpc-whitelist-enabled": True, "rpc-whitelist": "*"} + ) self.assertIsNotNone(warning) def test_no_warning_when_auth_also_required(self): - warning = ct.warn_if_open_without_auth({ - 'rpc-whitelist-enabled': True, - 'rpc-whitelist': '*', - 'rpc-authentication-required': True, - 'rpc-username': 'alice', - 'rpc-password': 'hunter2', - }) + warning = ct.warn_if_open_without_auth( + { + "rpc-whitelist-enabled": True, + "rpc-whitelist": "*", + "rpc-authentication-required": True, + "rpc-username": "alice", + "rpc-password": "hunter2", + } + ) self.assertIsNone(warning) @@ -183,21 +212,25 @@ def test_no_warning_when_whitelist_untouched(self): class ApplyOverridesTests(unittest.TestCase): def test_overrides_take_precedence_but_other_keys_survive(self): - settings = {'peer-port': 51413, 'rpc-whitelist-enabled': True, 'rpc-whitelist': '127.0.0.1'} - overrides = {'rpc-whitelist': '10.0.0.*'} + settings = { + "peer-port": 51413, + "rpc-whitelist-enabled": True, + "rpc-whitelist": "127.0.0.1", + } + overrides = {"rpc-whitelist": "10.0.0.*"} merged = ct.apply_overrides(settings, overrides) - self.assertEqual(merged['peer-port'], 51413) - self.assertEqual(merged['rpc-whitelist'], '10.0.0.*') - self.assertTrue(merged['rpc-whitelist-enabled']) + self.assertEqual(merged["peer-port"], 51413) + self.assertEqual(merged["rpc-whitelist"], "10.0.0.*") + self.assertTrue(merged["rpc-whitelist-enabled"]) class SettingsFilePersistenceTests(unittest.TestCase): def test_round_trips_through_disk(self): - tmp_dir = tempfile.mkdtemp(prefix='configure_transmission_') - path = os.path.join(tmp_dir, 'settings.json') - settings = {'rpc-whitelist': '10.0.0.*'} + tmp_dir = helpers.make_temp_dir(self, "configure_transmission_") + path = os.path.join(tmp_dir, "settings.json") + settings = {"rpc-whitelist": "10.0.0.*"} ct.save_settings(path, settings) loaded = ct.load_settings(path) @@ -205,16 +238,16 @@ def test_round_trips_through_disk(self): self.assertEqual(loaded, settings) def test_missing_file_returns_empty_settings(self): - tmp_dir = tempfile.mkdtemp(prefix='configure_transmission_') - path = os.path.join(tmp_dir, 'does-not-exist.json') + tmp_dir = helpers.make_temp_dir(self, "configure_transmission_") + path = os.path.join(tmp_dir, "does-not-exist.json") self.assertEqual(ct.load_settings(path), {}) def test_corrupt_file_returns_empty_settings_instead_of_raising(self): - tmp_dir = tempfile.mkdtemp(prefix='configure_transmission_') - path = os.path.join(tmp_dir, 'settings.json') - with open(path, 'w', encoding='utf-8') as f: - f.write('{not valid json') + tmp_dir = helpers.make_temp_dir(self, "configure_transmission_") + path = os.path.join(tmp_dir, "settings.json") + with open(path, "w", encoding="utf-8") as f: + f.write("{not valid json") self.assertEqual(ct.load_settings(path), {}) diff --git a/tests/test_fetch_torrents.py b/tests/test_fetch_torrents.py index 5318978..41db1d7 100644 --- a/tests/test_fetch_torrents.py +++ b/tests/test_fetch_torrents.py @@ -8,8 +8,10 @@ Run with: python -m unittest discover -s tests """ + import json import os +import shutil import subprocess import sys import tempfile @@ -19,20 +21,42 @@ from datetime import date, timedelta from unittest.mock import MagicMock -if 'transmission_rpc' not in sys.modules: - stub = types.ModuleType('transmission_rpc') +if "transmission_rpc" not in sys.modules: + stub = types.ModuleType("transmission_rpc") stub.Client = MagicMock() - sys.modules['transmission_rpc'] = stub + sys.modules["transmission_rpc"] = stub -os.environ.setdefault('FETCH_TORRENTS_LOG_DIR', tempfile.mkdtemp(prefix='fetch_torrents_test_logs_')) +_TEST_LOG_DIR = tempfile.mkdtemp(prefix="fetch_torrents_test_logs_") +os.environ.setdefault("FETCH_TORRENTS_LOG_DIR", _TEST_LOG_DIR) REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if REPO_ROOT not in sys.path: sys.path.insert(0, REPO_ROOT) +import helpers + import fetch_torrents as ft +def setUpModule(): + # Baseline for every test here: the developer's exported + # SKIP_RATIO_CHECK/CLEANUP_*/TRANSMISSION_*/FETCH_TORRENTS_* can't leak in + # (including into the subprocess tests, which copy os.environ). + patcher = helpers.isolated_env() + patcher.start() + unittest.addModuleCleanup(patcher.stop) + + +def tearDownModule(): + # The import-time log dir can only be removed once ft's handlers have + # released their files (Windows refuses to delete open files). + for handler in (ft.file_handler, ft.ratio_handler): + ft.logger.removeHandler(handler) + handler.close() + if os.environ.get("FETCH_TORRENTS_LOG_DIR") == _TEST_LOG_DIR: + shutil.rmtree(_TEST_LOG_DIR, ignore_errors=True) + + class FakeTorrent: def __init__(self, name, id_, ratio=0.0, added_date=None): self.name = name @@ -44,6 +68,17 @@ def __repr__(self): return f"FakeTorrent({self.name!r}, ratio={self.ratio})" +def fixed_date(today): + """A drop-in for fetch_torrents' `date` whose today() is pinned.""" + + class FixedDate(date): + @classmethod + def today(cls): + return today + + return FixedDate + + def names_of(torrents): return sorted(t.name for t in torrents) @@ -243,45 +278,69 @@ class ShouldFetchTorrentRatioKeyTests(unittest.TestCase): actually reports (which include the arch suffix and .iso extension), for all supported distros.""" + def setUp(self): + # Pinned explicitly: SKIP_RATIO_CHECK=true would make every case True. + helpers.use_isolated_env(self, SKIP_RATIO_CHECK="false") + def test_finds_previous_version_and_honors_low_ratio_ubuntu(self): ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.5} - self.assertFalse(ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios)) + self.assertFalse( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) def test_finds_previous_version_and_honors_low_ratio_debian(self): ratios = {"debian-12.4.0-amd64-DVD-1.iso": 0.5} - self.assertFalse(ft.should_fetch_torrent("debian-12.5.0-amd64-DVD-1.iso", ratios)) + self.assertFalse( + ft.should_fetch_torrent("debian-12.5.0-amd64-DVD-1.iso", ratios) + ) def test_finds_previous_version_and_honors_low_ratio_kali(self): ratios = {"kali-linux-2023.4-installer-amd64.iso": 0.3} - self.assertFalse(ft.should_fetch_torrent("kali-linux-2024.1-installer-amd64.iso", ratios)) + self.assertFalse( + ft.should_fetch_torrent("kali-linux-2024.1-installer-amd64.iso", ratios) + ) def test_finds_previous_version_and_honors_low_ratio_arch(self): ratios = {"archlinux-2024.04.01-x86_64.iso": 0.2} - self.assertFalse(ft.should_fetch_torrent("archlinux-2024.05.01-x86_64.iso", ratios)) + self.assertFalse( + ft.should_fetch_torrent("archlinux-2024.05.01-x86_64.iso", ratios) + ) def test_finds_previous_version_and_allows_high_ratio_ubuntu(self): ratios = {"ubuntu-23.10-desktop-amd64.iso": 1.5} - self.assertTrue(ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios)) + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) def test_finds_previous_version_and_allows_high_ratio_debian(self): ratios = {"debian-12.4.0-amd64-DVD-1.iso": 1.5} - self.assertTrue(ft.should_fetch_torrent("debian-12.5.0-amd64-DVD-1.iso", ratios)) + self.assertTrue( + ft.should_fetch_torrent("debian-12.5.0-amd64-DVD-1.iso", ratios) + ) def test_finds_previous_version_and_honors_low_ratio_mint(self): ratios = {"linuxmint-22.2-cinnamon-64bit.iso": 0.4} - self.assertFalse(ft.should_fetch_torrent("linuxmint-22.3-cinnamon-64bit.iso", ratios)) + self.assertFalse( + ft.should_fetch_torrent("linuxmint-22.3-cinnamon-64bit.iso", ratios) + ) def test_finds_previous_version_and_allows_high_ratio_mint(self): ratios = {"linuxmint-22.2-cinnamon-64bit.iso": 1.5} - self.assertTrue(ft.should_fetch_torrent("linuxmint-22.3-cinnamon-64bit.iso", ratios)) + self.assertTrue( + ft.should_fetch_torrent("linuxmint-22.3-cinnamon-64bit.iso", ratios) + ) def test_finds_previous_version_and_honors_low_ratio_fedora(self): ratios = {"Fedora-Workstation-Live-x86_64-43": 0.4} - self.assertFalse(ft.should_fetch_torrent("Fedora-Workstation-Live-x86_64-44", ratios)) + self.assertFalse( + ft.should_fetch_torrent("Fedora-Workstation-Live-x86_64-44", ratios) + ) def test_finds_previous_version_and_allows_high_ratio_fedora(self): ratios = {"Fedora-Workstation-Live-x86_64-43": 1.5} - self.assertTrue(ft.should_fetch_torrent("Fedora-Workstation-Live-x86_64-44", ratios)) + self.assertTrue( + ft.should_fetch_torrent("Fedora-Workstation-Live-x86_64-44", ratios) + ) class ShouldFetchTorrentRemovedHistoryFallbackTests(unittest.TestCase): @@ -292,24 +351,40 @@ class ShouldFetchTorrentRemovedHistoryFallbackTests(unittest.TestCase): the ratio gate for exactly the versions cleanup already gave up on. removed_history (record_removal()'s output) is the fallback source.""" + def setUp(self): + # Pinned explicitly: SKIP_RATIO_CHECK=true would make every case True. + helpers.use_isolated_env(self, SKIP_RATIO_CHECK="false") + def test_falls_back_to_removed_history_ratio_when_not_live(self): removed_history = { - 'ubuntu-23.10-desktop-amd64.iso': { - 'removed_date': '2026-01-01', 'reason': 'stagnant', 'ratio': 0.4, + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.4, }, } self.assertFalse( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios={}, removed_history=removed_history) + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios={}, + removed_history=removed_history, + ) ) def test_removed_history_ratio_above_floor_allows_fetch(self): removed_history = { - 'ubuntu-23.10-desktop-amd64.iso': { - 'removed_date': '2026-01-01', 'reason': 'keep_only_latest', 'ratio': 1.5, + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-01-01", + "reason": "keep_only_latest", + "ratio": 1.5, }, } self.assertTrue( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios={}, removed_history=removed_history) + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios={}, + removed_history=removed_history, + ) ) def test_live_ratio_takes_priority_over_removed_history(self): @@ -317,31 +392,91 @@ def test_live_ratio_takes_priority_over_removed_history(self): # still reports hasn't been removed), but live data winning if it ever # happens is the safer default. removed_history = { - 'ubuntu-23.10-desktop-amd64.iso': { - 'removed_date': '2026-01-01', 'reason': 'stagnant', 'ratio': 0.1, + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.1, }, } - ratios = {'ubuntu-23.10-desktop-amd64.iso': 1.5} + ratios = {"ubuntu-23.10-desktop-amd64.iso": 1.5} self.assertTrue( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios, removed_history=removed_history) + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios, + removed_history=removed_history, + ) + ) + + def test_low_live_ratio_is_not_overridden_by_high_removed_history(self): + removed_history = { + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 1.5, + } + } + ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.4} + self.assertFalse( + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios, + removed_history=removed_history, + ) ) def test_removed_history_entry_without_ratio_is_ignored(self): # Legacy entries recorded before this field existed have no 'ratio' # key - must not crash, and must not count as a usable prior version. removed_history = { - 'ubuntu-23.10-desktop-amd64.iso': {'removed_date': '2026-01-01', 'reason': 'stagnant'}, + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + }, } - self.assertTrue( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios={}, removed_history=removed_history) + # Skipped up front, not by tripping over the missing key and logging + # a parse-error warning. + with self.assertNoLogs(ft.logger, level="WARNING"): + result = ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios={}, + removed_history=removed_history, + ) + self.assertTrue(result) + + def test_skipped_removed_history_entry_does_not_hide_later_ones(self): + removed_history = { + "ubuntu-23.04-desktop-amd64.iso": { + "removed_date": "2025-06-01", + "reason": "stagnant", + }, + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.4, + }, + } + self.assertFalse( + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios={}, + removed_history=removed_history, + ) ) def test_removed_history_from_a_different_distro_is_not_consulted(self): removed_history = { - 'debian-12.4.0-amd64-DVD-1.iso': {'removed_date': '2026-01-01', 'reason': 'stagnant', 'ratio': 0.1}, + "debian-12.4.0-amd64-DVD-1.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.1, + }, } self.assertTrue( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios={}, removed_history=removed_history) + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios={}, + removed_history=removed_history, + ) ) @@ -351,6 +486,10 @@ class ShouldFetchTorrentDevuanRatioGateTests(unittest.TestCase): fetching a new Devuan release on the ratio achieved by whichever Devuan release we currently hold, using the same 1.0 floor as other distros.""" + def setUp(self): + # Pinned explicitly: SKIP_RATIO_CHECK=true would make every case True. + helpers.use_isolated_env(self, SKIP_RATIO_CHECK="false") + def test_no_previous_devuan_release_fetches(self): self.assertTrue(ft.should_fetch_torrent("devuan_excalibur", ratios={})) @@ -370,20 +509,157 @@ def test_already_holding_this_exact_release_is_always_fetched(self): def test_falls_back_to_removed_history_when_no_live_ratio(self): removed_history = { - "devuan_daedalus": {"removed_date": "2026-01-01", "reason": "stagnant", "ratio": 0.4}, + "devuan_daedalus": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.4, + }, } self.assertFalse( - ft.should_fetch_torrent("devuan_excalibur", ratios={}, removed_history=removed_history) + ft.should_fetch_torrent( + "devuan_excalibur", ratios={}, removed_history=removed_history + ) ) def test_other_distros_removed_history_not_consulted_for_devuan(self): removed_history = { - "debian-12.4.0-amd64-DVD-1.iso": {"removed_date": "2026-01-01", "reason": "stagnant", "ratio": 0.1}, + "debian-12.4.0-amd64-DVD-1.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.1, + }, + } + self.assertTrue( + ft.should_fetch_torrent( + "devuan_excalibur", ratios={}, removed_history=removed_history + ) + ) + + +class ShouldFetchTorrentBoundaryTests(unittest.TestCase): + """The 1.0 floor is inclusive, only strictly older versions count as the + "previous" one, and the branches that bypass the gate entirely.""" + + def setUp(self): + helpers.use_isolated_env(self, SKIP_RATIO_CHECK="false") + + def test_previous_ratio_exactly_at_floor_allows_fetch(self): + ratios = {"ubuntu-23.10-desktop-amd64.iso": 1.0} + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_previous_ratio_just_below_floor_blocks_fetch(self): + ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.999} + self.assertFalse( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_devuan_previous_ratio_exactly_at_floor_allows_fetch(self): + self.assertTrue( + ft.should_fetch_torrent("devuan_excalibur", {"devuan_daedalus": 1.0}) + ) + + def test_devuan_previous_ratio_just_below_floor_blocks_fetch(self): + self.assertFalse( + ft.should_fetch_torrent("devuan_excalibur", {"devuan_daedalus": 0.999}) + ) + + def test_only_newer_versions_on_record_fetches(self): + # A newer release with a low ratio isn't a "previous" one, so there + # is nothing to gate on. + ratios = {"ubuntu-24.10-desktop-amd64.iso": 0.1} + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_same_version_on_record_is_not_its_own_predecessor(self): + ratios = {"ubuntu-24.04-desktop-amd64.iso": 0.1} + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_highest_older_version_decides_not_any_older_version(self): + ratios = { + "ubuntu-22.04-desktop-amd64.iso": 5.0, + "ubuntu-23.10-desktop-amd64.iso": 0.4, + } + self.assertFalse( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_other_architecture_ratio_is_not_consulted(self): + ratios = {"ubuntu-23.10-desktop-arm64.iso": 0.1} + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_devuan_ignores_non_devuan_ratios(self): + ratios = {"debian-12.4.0-amd64-DVD-1.iso": 5.0, "devuan_daedalus": 0.4} + self.assertFalse(ft.should_fetch_torrent("devuan_excalibur", ratios)) + + def test_devuan_removed_history_for_the_same_release_is_not_a_predecessor(self): + removed_history = { + "devuan_excalibur": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.1, + } } self.assertTrue( - ft.should_fetch_torrent("devuan_excalibur", ratios={}, removed_history=removed_history) + ft.should_fetch_torrent( + "devuan_excalibur", ratios={}, removed_history=removed_history + ) + ) + + def test_devuan_live_ratio_beats_removed_history(self): + removed_history = { + "devuan_chimaera": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.1, + } + } + self.assertTrue( + ft.should_fetch_torrent( + "devuan_excalibur", + {"devuan_daedalus": 1.5}, + removed_history=removed_history, + ) ) + def test_skip_ratio_check_fetches_despite_low_previous_ratio(self): + ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.1} + with helpers.isolated_env(SKIP_RATIO_CHECK="TRUE"): + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_unknown_distro_fetches(self): + ratios = {"slackware-15.0-install-dvd.iso": 0.1} + self.assertTrue( + ft.should_fetch_torrent("slackware-15.1-install-dvd.iso", ratios) + ) + + def test_unparseable_name_of_known_distro_fetches_and_logs_error(self): + ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.1} + with self.assertLogs(ft.logger, level="ERROR") as logs: + result = ft.should_fetch_torrent("ubuntu-noble-desktop-amd64.iso", ratios) + + self.assertTrue(result) + self.assertEqual(len(logs.records), 1) + self.assertIn("ubuntu-noble-desktop-amd64.iso", logs.output[0]) + + def test_unparseable_stored_ratio_entry_is_skipped(self): + ratios = { + "ubuntu-mantic-desktop-amd64.iso": 0.1, + "ubuntu-23.10-desktop-amd64.iso": 1.5, + } + with self.assertLogs(ft.logger, level="WARNING"): + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + class RatioLogSurvivesRestartTests(unittest.TestCase): """Each container run is a fresh `python fetch_torrents.py` process: the @@ -394,9 +670,9 @@ class RatioLogSurvivesRestartTests(unittest.TestCase): about what happens between import and the first get_previous_ratios() call.""" def test_previous_ratios_are_readable_after_a_fresh_start(self): - log_dir = tempfile.mkdtemp(prefix='fetch_torrents_ratio_restart_') - ratio_log_path = os.path.join(log_dir, 'fetch_torrents_ratios.log') - with open(ratio_log_path, 'w', encoding='utf-8') as f: + log_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_restart_") + ratio_log_path = os.path.join(log_dir, "fetch_torrents_ratios.log") + with open(ratio_log_path, "w", encoding="utf-8") as f: f.write( "2026-08-01 00:00:00,000 INFO: [ratio] RATIOS START\n" "2026-08-01 00:00:00,000 INFO: [ratio] archlinux-2026.04.01-x86_64.iso → 0.664\n" @@ -415,13 +691,17 @@ def test_previous_ratios_are_readable_after_a_fresh_start(self): ) env = dict(os.environ, FETCH_TORRENTS_LOG_DIR=log_dir) result = subprocess.run( - [sys.executable, '-c', script], - env=env, capture_output=True, text=True, timeout=30, + [sys.executable, "-c", script], + env=env, + capture_output=True, + text=True, + timeout=30, ) self.assertEqual(result.returncode, 0, result.stderr) self.assertIn( - "archlinux-2026.04.01-x86_64.iso", result.stdout, + "archlinux-2026.04.01-x86_64.iso", + result.stdout, "get_previous_ratios() found nothing from the prior run's ratio " "log - it was likely wiped by the ratio FileHandler's mode='w' " "open at import time, before ever being read. That silently " @@ -429,6 +709,117 @@ def test_previous_ratios_are_readable_after_a_fresh_start(self): ) +class GetPreviousRatiosTests(unittest.TestCase): + """get_previous_ratios() is what feeds should_fetch_torrent()'s ratios: + only the last START/END segment of the previous run's ratio log counts, + so a ratio from an older run can't satisfy (or block) the gate.""" + + def _write_log(self, *messages): + path = os.path.join( + helpers.make_temp_dir(self, "fetch_torrents_prev_ratios_"), + "fetch_torrents_ratios.log", + ) + with open(path, "w", encoding="utf-8") as f: + f.writelines( + f"2026-08-01 00:00:00,000 INFO: {message}\n" for message in messages + ) + return path + + def test_missing_file_returns_empty(self): + missing = os.path.join( + helpers.make_temp_dir(self, "fetch_torrents_prev_ratios_"), "nope.log" + ) + self.assertEqual(ft.get_previous_ratios(missing), {}) + + def test_without_markers_every_ratio_line_is_parsed(self): + path = self._write_log( + "[ratio] ubuntu-24.04-desktop-amd64.iso → 1.250", + "unrelated line", + "[ratio] debian-12.5.0-amd64-DVD-1.iso → 0.664", + ) + self.assertEqual( + ft.get_previous_ratios(path), + { + "ubuntu-24.04-desktop-amd64.iso": 1.25, + "debian-12.5.0-amd64-DVD-1.iso": 0.664, + }, + ) + + def test_only_the_last_complete_segment_is_returned(self): + path = self._write_log( + "[ratio] RATIOS START", + "[ratio] old-only.iso → 0.100", + "[ratio] shared.iso → 0.200", + "[ratio] RATIOS END", + "[ratio] RATIOS START", + "[ratio] shared.iso → 1.500", + "[ratio] RATIOS END", + "[ratio] after-end.iso → 0.300", + ) + self.assertEqual(ft.get_previous_ratios(path), {"shared.iso": 1.5}) + + def test_start_without_end_parses_to_end_of_file(self): + # A run that crashed mid-way never wrote its END marker. + path = self._write_log( + "[ratio] RATIOS START", + "[ratio] old.iso → 0.100", + "[ratio] RATIOS END", + "[ratio] RATIOS START", + "[ratio] crashed-run.iso → 0.700", + ) + self.assertEqual(ft.get_previous_ratios(path), {"crashed-run.iso": 0.7}) + + def test_end_before_the_last_start_is_ignored(self): + path = self._write_log( + "[ratio] RATIOS END", + "[ratio] before-start.iso → 0.100", + "[ratio] RATIOS START", + "[ratio] current.iso → 0.900", + ) + self.assertEqual(ft.get_previous_ratios(path), {"current.iso": 0.9}) + + def test_last_end_after_the_last_start_closes_the_segment(self): + # Not written by a normal run; pins current behavior. + path = self._write_log( + "[ratio] RATIOS START", + "[ratio] first.iso → 0.100", + "[ratio] RATIOS END", + "[ratio] second.iso → 0.200", + "[ratio] RATIOS END", + "[ratio] after.iso → 0.300", + ) + self.assertEqual( + ft.get_previous_ratios(path), {"first.iso": 0.1, "second.iso": 0.2} + ) + + def test_padded_names_are_stripped(self): + path = self._write_log( + "[ratio] archlinux-2026.04.01-x86_64.iso → 0.664", + ) + self.assertEqual( + ft.get_previous_ratios(path), {"archlinux-2026.04.01-x86_64.iso": 0.664} + ) + + def test_names_with_internal_spaces_keep_them(self): + path = self._write_log("[ratio] Some Torrent Name → 2.000") + self.assertEqual(ft.get_previous_ratios(path), {"Some Torrent Name": 2.0}) + + def test_ratio_without_decimal_point_is_not_matched(self): + # Pins current behavior: log_seed_ratios_via_http() always writes + # three decimals, so an integer ratio means a line it didn't write. + path = self._write_log("[ratio] whole.iso → 1") + self.assertEqual(ft.get_previous_ratios(path), {}) + + def test_duplicate_name_in_a_segment_last_one_wins(self): + path = self._write_log( + "[ratio] RATIOS START", + "[ratio] dup.iso → 0.100", + "[ratio] dup.iso → 1.100", + "[ratio] RATIOS END", + ) + self.assertEqual(ft.get_previous_ratios(path), {"dup.iso": 1.1}) + + class LogFileRotationTests(unittest.TestCase): """fetch_torrents.log must be bounded (RotatingFileHandler), not a plain FileHandler appending forever for the life of a 'deploy and forget' @@ -438,7 +829,7 @@ class LogFileRotationTests(unittest.TestCase): was in effect during test collection).""" def _run(self, extra_env=None): - log_dir = tempfile.mkdtemp(prefix='fetch_torrents_log_rotation_') + log_dir = helpers.make_temp_dir(self, "fetch_torrents_log_rotation_") script = ( "import os, sys, types\n" "from unittest.mock import MagicMock\n" @@ -453,8 +844,11 @@ def _run(self, extra_env=None): ) env = dict(os.environ, FETCH_TORRENTS_LOG_DIR=log_dir, **(extra_env or {})) result = subprocess.run( - [sys.executable, '-c', script], - env=env, capture_output=True, text=True, timeout=30, + [sys.executable, "-c", script], + env=env, + capture_output=True, + text=True, + timeout=30, ) self.assertEqual(result.returncode, 0, result.stderr) lines = result.stdout.splitlines() @@ -462,21 +856,27 @@ def _run(self, extra_env=None): def test_defaults_to_a_bounded_rotating_handler(self): handler_type, max_bytes, backup_count = self._run() - self.assertEqual(handler_type, 'RotatingFileHandler') + self.assertEqual(handler_type, "RotatingFileHandler") self.assertEqual(max_bytes, 5 * 1024 * 1024) self.assertEqual(backup_count, 3) def test_env_vars_override_rotation_settings(self): - _handler_type, max_bytes, backup_count = self._run({ - 'FETCH_TORRENTS_LOG_MAX_BYTES': '1000', 'FETCH_TORRENTS_LOG_BACKUP_COUNT': '1', - }) + _handler_type, max_bytes, backup_count = self._run( + { + "FETCH_TORRENTS_LOG_MAX_BYTES": "1000", + "FETCH_TORRENTS_LOG_BACKUP_COUNT": "1", + } + ) self.assertEqual(max_bytes, 1000) self.assertEqual(backup_count, 1) def test_invalid_env_vars_fall_back_to_defaults(self): - _handler_type, max_bytes, backup_count = self._run({ - 'FETCH_TORRENTS_LOG_MAX_BYTES': 'not-a-number', 'FETCH_TORRENTS_LOG_BACKUP_COUNT': 'also-bad', - }) + _handler_type, max_bytes, backup_count = self._run( + { + "FETCH_TORRENTS_LOG_MAX_BYTES": "not-a-number", + "FETCH_TORRENTS_LOG_BACKUP_COUNT": "also-bad", + } + ) self.assertEqual(max_bytes, 5 * 1024 * 1024) self.assertEqual(backup_count, 3) @@ -489,16 +889,14 @@ class FetchUbuntuLtsNamingTests(unittest.TestCase): tasks/02-ratio-key-mismatch-ubuntu-debian.md.""" def test_candidate_names_include_arch_and_iso_suffix(self): - meta_release = ( - "Dist: noble\n" - "Version: 24.04\n" - "Supported: 1\n" - ) + meta_release = "Dist: noble\nVersion: 24.04\nSupported: 1\n" class FakeResponse: text = meta_release - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_ubuntu_lts() self.assertTrue(results) @@ -519,7 +917,9 @@ class FakeResponse: def raise_for_status(self): pass - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_debian_stable() self.assertIn("debian-12.5.0-amd64-DVD-1.iso", results) @@ -537,7 +937,9 @@ def test_candidate_name_matches_torrent_internal_name(self): class FakeResponse: text = html - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_linuxmint_cinnamon() self.assertIn("linuxmint-22.3-cinnamon-64bit.iso", results) @@ -570,16 +972,16 @@ class FakeResponse: def raise_for_status(self): pass - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_fedora_workstation() self.assertEqual( results, { - "Fedora-Workstation-Live-x86_64-44": - "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-x86_64-44.torrent", - "Fedora-Workstation-Live-aarch64-44": - "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-aarch64-44.torrent", + "Fedora-Workstation-Live-x86_64-44": "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-x86_64-44.torrent", + "Fedora-Workstation-Live-aarch64-44": "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-aarch64-44.torrent", }, ) @@ -606,12 +1008,16 @@ class FakeResponse: def raise_for_status(self): pass - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_arch_latest() self.assertEqual( results, - {"archlinux-2026.09.01": "https://archlinux.org/releng/releases/2026.09.01/torrent/"}, + { + "archlinux-2026.09.01": "https://archlinux.org/releng/releases/2026.09.01/torrent/" + }, ) def test_no_available_releases_returns_false(self): @@ -623,7 +1029,9 @@ class FakeResponse: def raise_for_status(self): pass - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): result = ft.fetch_arch_latest() self.assertFalse(result) @@ -640,13 +1048,15 @@ def test_extracts_current_release_torrent(self): html = ( '' + "" ) class FakeResponse: text = html - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_devuan_latest() self.assertEqual( @@ -658,7 +1068,9 @@ def test_returns_false_when_no_torrent_link_found(self): class FakeResponse: text = "nothing here" - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): result = ft.fetch_devuan_latest() self.assertFalse(result) @@ -673,35 +1085,42 @@ class ParseSupportedDistrosTests(unittest.TestCase): fetched just because FETCH_TORRENTS_DISTROS is unset.""" def test_default_excludes_opt_in_distros(self): - with unittest.mock.patch.dict(os.environ, {}, clear=False): - os.environ.pop('FETCH_TORRENTS_DISTROS', None) + with helpers.isolated_env(): result = ft.parse_supported_distros() - self.assertNotIn('arch', result) - self.assertNotIn('devuan', result) - for expected in ('ubuntu', 'debian', 'kali', 'mint', 'fedora'): + self.assertNotIn("arch", result) + self.assertNotIn("devuan", result) + for expected in ("ubuntu", "debian", "kali", "mint", "fedora"): self.assertIn(expected, result) def test_explicit_opt_in_distros_are_honored(self): - with unittest.mock.patch.dict(os.environ, {'FETCH_TORRENTS_DISTROS': 'arch,devuan'}): + with unittest.mock.patch.dict( + os.environ, {"FETCH_TORRENTS_DISTROS": "arch,devuan"} + ): result = ft.parse_supported_distros() - self.assertEqual(sorted(result), ['arch', 'devuan']) + self.assertEqual(sorted(result), ["arch", "devuan"]) def test_can_mix_default_and_opt_in_distros(self): - with unittest.mock.patch.dict(os.environ, {'FETCH_TORRENTS_DISTROS': 'ubuntu,arch'}): + with unittest.mock.patch.dict( + os.environ, {"FETCH_TORRENTS_DISTROS": "ubuntu,arch"} + ): result = ft.parse_supported_distros() - self.assertEqual(sorted(result), ['arch', 'ubuntu']) + self.assertEqual(sorted(result), ["arch", "ubuntu"]) def test_unknown_entries_are_dropped_but_valid_ones_kept(self): - with unittest.mock.patch.dict(os.environ, {'FETCH_TORRENTS_DISTROS': 'ubuntu,nonsense'}): + with unittest.mock.patch.dict( + os.environ, {"FETCH_TORRENTS_DISTROS": "ubuntu,nonsense"} + ): result = ft.parse_supported_distros() - self.assertEqual(result, ['ubuntu']) + self.assertEqual(result, ["ubuntu"]) def test_all_invalid_falls_back_to_default_not_all_distros(self): - with unittest.mock.patch.dict(os.environ, {'FETCH_TORRENTS_DISTROS': 'nonsense'}): + with unittest.mock.patch.dict( + os.environ, {"FETCH_TORRENTS_DISTROS": "nonsense"} + ): result = ft.parse_supported_distros() self.assertEqual(sorted(result), sorted(ft.DEFAULT_DISTROS)) - self.assertNotIn('arch', result) - self.assertNotIn('devuan', result) + self.assertNotIn("arch", result) + self.assertNotIn("devuan", result) class LowDemandVariantTests(unittest.TestCase): @@ -791,8 +1210,8 @@ class DownloadTorrent404Tests(unittest.TestCase): same way as a genuine download failure.""" def setUp(self): - self.tmp_watch_dir = tempfile.mkdtemp(prefix='fetch_torrents_test_watch_') - patcher = unittest.mock.patch.object(ft, 'watch_dir', self.tmp_watch_dir) + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) patcher.start() self.addCleanup(patcher.stop) @@ -800,13 +1219,18 @@ def test_404_response_is_not_found_not_failed(self): class FakeResponse: status_code = 404 - with unittest.mock.patch.object(ft.requests, 'get', return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): status = ft.download_torrent( - 'ubuntu-14.04.6-live-server-amd64.iso', 'https://example.invalid/x.torrent' + "ubuntu-14.04.6-live-server-amd64.iso", + "https://example.invalid/x.torrent", ) - self.assertEqual(status, 'not_found') - dest = os.path.join(self.tmp_watch_dir, 'ubuntu-14.04.6-live-server-amd64.iso.torrent') + self.assertEqual(status, "not_found") + dest = os.path.join( + self.tmp_watch_dir, "ubuntu-14.04.6-live-server-amd64.iso.torrent" + ) self.assertFalse(os.path.exists(dest)) def test_other_http_error_is_still_failed(self): @@ -816,30 +1240,36 @@ class FakeResponse: def raise_for_status(self): raise Exception("500 Server Error") - with unittest.mock.patch.object(ft.requests, 'get', return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): status = ft.download_torrent( - 'ubuntu-24.04-desktop-amd64.iso', 'https://example.invalid/x.torrent' + "ubuntu-24.04-desktop-amd64.iso", "https://example.invalid/x.torrent" ) - self.assertEqual(status, 'failed') + self.assertEqual(status, "failed") def test_successful_download_is_still_added(self): class FakeResponse: status_code = 200 - content = b'fake torrent bytes' + content = b"fake torrent bytes" def raise_for_status(self): pass - with unittest.mock.patch.object(ft.requests, 'get', return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): status = ft.download_torrent( - 'ubuntu-24.04-desktop-amd64.iso', 'https://example.invalid/x.torrent' + "ubuntu-24.04-desktop-amd64.iso", "https://example.invalid/x.torrent" ) - self.assertEqual(status, 'added') - dest = os.path.join(self.tmp_watch_dir, 'ubuntu-24.04-desktop-amd64.iso.torrent') - with open(dest, 'rb') as f: - self.assertEqual(f.read(), b'fake torrent bytes') + self.assertEqual(status, "added") + dest = os.path.join( + self.tmp_watch_dir, "ubuntu-24.04-desktop-amd64.iso.torrent" + ) + with open(dest, "rb") as f: + self.assertEqual(f.read(), b"fake torrent bytes") class SafeWatchPathTests(unittest.TestCase): @@ -850,48 +1280,58 @@ class SafeWatchPathTests(unittest.TestCase): upstream content into a path escape.""" def setUp(self): - patcher = unittest.mock.patch.object(ft, 'watch_dir', '/watch') + patcher = unittest.mock.patch.object(ft, "watch_dir", "/watch") patcher.start() self.addCleanup(patcher.stop) def test_normal_name_resolves_under_watch_dir(self): self.assertEqual( - ft._safe_watch_path('ubuntu-24.04-desktop-amd64.iso'), - os.path.join('/watch', 'ubuntu-24.04-desktop-amd64.iso.torrent'), + ft._safe_watch_path("ubuntu-24.04-desktop-amd64.iso"), + os.path.join("/watch", "ubuntu-24.04-desktop-amd64.iso.torrent"), ) def test_rejects_path_separators(self): - self.assertIsNone(ft._safe_watch_path('archlinux-../../../etc/cron.d/evil')) - self.assertIsNone(ft._safe_watch_path('foo/bar')) - self.assertIsNone(ft._safe_watch_path('foo\\bar')) + self.assertIsNone(ft._safe_watch_path("archlinux-../../../etc/cron.d/evil")) + self.assertIsNone(ft._safe_watch_path("foo/bar")) + self.assertIsNone(ft._safe_watch_path("foo\\bar")) def test_rejects_empty_or_dot_names(self): - self.assertIsNone(ft._safe_watch_path('')) + self.assertIsNone(ft._safe_watch_path("")) self.assertIsNone(ft._safe_watch_path(None)) - self.assertIsNone(ft._safe_watch_path('.')) - self.assertIsNone(ft._safe_watch_path('..')) + self.assertIsNone(ft._safe_watch_path(".")) + self.assertIsNone(ft._safe_watch_path("..")) class DownloadTorrentUnsafeNameTests(unittest.TestCase): def setUp(self): - self.tmp_watch_dir = tempfile.mkdtemp(prefix='fetch_torrents_test_watch_') - patcher = unittest.mock.patch.object(ft, 'watch_dir', self.tmp_watch_dir) + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) patcher.start() self.addCleanup(patcher.stop) def test_download_torrent_refuses_unsafe_name_without_any_request(self): - with unittest.mock.patch.object(ft.requests, 'get') as mock_get: - status = ft.download_torrent('../evil', 'https://example.invalid/x.torrent') + with unittest.mock.patch.object(ft.requests, "get") as mock_get: + status = ft.download_torrent("../evil", "https://example.invalid/x.torrent") mock_get.assert_not_called() - self.assertEqual(status, 'failed') + self.assertEqual(status, "failed") self.assertEqual(os.listdir(self.tmp_watch_dir), []) def test_remove_watch_file_refuses_unsafe_name_without_touching_disk(self): - with unittest.mock.patch.object(ft.os, 'remove') as mock_remove: - ft._remove_watch_file('../evil') + # Asserted on disk rather than by patching os.remove: that patch is + # process-wide, so a log rotation triggered by the warning would + # trip it too. + parent = helpers.make_temp_dir(self, "fetch_torrents_test_escape_") + watch_dir = os.path.join(parent, "watch") + os.mkdir(watch_dir) + outside = os.path.join(parent, "evil.torrent") + with open(outside, "w", encoding="utf-8") as f: + f.write("must survive") + + with unittest.mock.patch.object(ft, "watch_dir", watch_dir): + ft._remove_watch_file("../evil") - mock_remove.assert_not_called() + self.assertTrue(os.path.exists(outside)) class UpdateRatioHistoryTests(unittest.TestCase): @@ -903,7 +1343,9 @@ class UpdateRatioHistoryTests(unittest.TestCase): def test_new_torrent_gets_a_first_sample(self): today = date(2026, 8, 29) - history = ft.update_ratio_history({}, {"archlinux-2026.08.01-x86_64.iso": 1.2}, today) + history = ft.update_ratio_history( + {}, {"archlinux-2026.08.01-x86_64.iso": 1.2}, today + ) self.assertEqual( history, @@ -916,7 +1358,9 @@ def test_recent_sample_is_not_duplicated(self): "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-25", "ratio": 1.0}], } - updated = ft.update_ratio_history(history, {"archlinux-2026.08.01-x86_64.iso": 1.1}, today) + updated = ft.update_ratio_history( + history, {"archlinux-2026.08.01-x86_64.iso": 1.1}, today + ) # Only 4 days since the last sample (< the 7-day interval) - no new # sample yet, and the ratio value on the existing sample is untouched. @@ -931,7 +1375,9 @@ def test_new_sample_added_once_interval_elapses(self): "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-20", "ratio": 1.0}], } - updated = ft.update_ratio_history(history, {"archlinux-2026.08.01-x86_64.iso": 1.3}, today) + updated = ft.update_ratio_history( + history, {"archlinux-2026.08.01-x86_64.iso": 1.3}, today + ) self.assertEqual( updated, @@ -987,7 +1433,9 @@ def test_latest_or_only_version_is_never_removed_even_if_flat(self): "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-06-30", "ratio": 1.0}], } - to_remove, to_keep = ft.plan_stagnation_cleanup(torrents, history, today, window_days=30, min_ratio_delta=0.02) + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, window_days=30, min_ratio_delta=0.02 + ) self.assertEqual(to_remove, []) self.assertEqual(to_keep, []) @@ -1002,7 +1450,9 @@ def test_old_version_with_no_ratio_growth_is_removed(self): "ubuntu-23.10-desktop-amd64.iso": [{"date": "2026-07-20", "ratio": 1.495}], } - to_remove, to_keep = ft.plan_stagnation_cleanup(torrents, history, today, window_days=30, min_ratio_delta=0.02) + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, window_days=30, min_ratio_delta=0.02 + ) self.assertEqual(names_of(to_remove), ["ubuntu-23.10-desktop-amd64.iso"]) self.assertEqual(to_keep, []) @@ -1017,7 +1467,9 @@ def test_old_version_still_growing_is_kept(self): "ubuntu-23.10-desktop-amd64.iso": [{"date": "2026-07-20", "ratio": 1.2}], } - to_remove, to_keep = ft.plan_stagnation_cleanup(torrents, history, today, window_days=30, min_ratio_delta=0.02) + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, window_days=30, min_ratio_delta=0.02 + ) self.assertEqual(to_remove, []) self.assertEqual(len(to_keep), 1) @@ -1036,7 +1488,9 @@ def test_old_version_without_enough_history_is_kept(self): "ubuntu-23.10-desktop-amd64.iso": [{"date": "2026-08-24", "ratio": 0.1}], } - to_remove, to_keep = ft.plan_stagnation_cleanup(torrents, history, today, window_days=30, min_ratio_delta=0.02) + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, window_days=30, min_ratio_delta=0.02 + ) self.assertEqual(to_remove, []) self.assertEqual(len(to_keep), 1) @@ -1051,7 +1505,9 @@ def test_old_version_with_no_history_at_all_is_kept(self): FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1), ] - to_remove, to_keep = ft.plan_stagnation_cleanup(torrents, {}, today, window_days=30, min_ratio_delta=0.02) + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, {}, today, window_days=30, min_ratio_delta=0.02 + ) self.assertEqual(to_remove, []) self.assertEqual(len(to_keep), 1) @@ -1060,7 +1516,9 @@ def test_unrelated_single_torrent_groups_are_ignored(self): today = date(2026, 8, 29) torrents = [FakeTorrent("some-random-linux-distro-1.0.iso", 1, ratio=5.0)] - to_remove, to_keep = ft.plan_stagnation_cleanup(torrents, {}, today, window_days=30, min_ratio_delta=0.02) + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, {}, today, window_days=30, min_ratio_delta=0.02 + ) self.assertEqual(to_remove, []) self.assertEqual(to_keep, []) @@ -1071,9 +1529,11 @@ class RatioHistoryPersistenceTests(unittest.TestCase): tolerate a missing or corrupt file rather than crashing a run.""" def test_round_trips_through_disk(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_ratio_history_') - path = os.path.join(tmp_dir, 'fetch_torrents_ratio_history.json') - history = {"archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-29", "ratio": 1.2}]} + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_history_") + path = os.path.join(tmp_dir, "fetch_torrents_ratio_history.json") + history = { + "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-29", "ratio": 1.2}] + } ft.save_ratio_history(path, history) loaded = ft.load_ratio_history(path) @@ -1081,16 +1541,16 @@ def test_round_trips_through_disk(self): self.assertEqual(loaded, history) def test_missing_file_returns_empty_history(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_ratio_history_') - path = os.path.join(tmp_dir, 'does-not-exist.json') + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_history_") + path = os.path.join(tmp_dir, "does-not-exist.json") self.assertEqual(ft.load_ratio_history(path), {}) def test_corrupt_file_returns_empty_history_instead_of_raising(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_ratio_history_') - path = os.path.join(tmp_dir, 'fetch_torrents_ratio_history.json') - with open(path, 'w', encoding='utf-8') as f: - f.write('{not valid json') + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_history_") + path = os.path.join(tmp_dir, "fetch_torrents_ratio_history.json") + with open(path, "w", encoding="utf-8") as f: + f.write("{not valid json") self.assertEqual(ft.load_ratio_history(path), {}) @@ -1113,7 +1573,9 @@ def test_returns_only_the_non_newest_versions(self): class FakeResponse: text = html - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_kali_old_versions() self.assertTrue(results) @@ -1126,7 +1588,9 @@ def test_single_version_folder_returns_nothing(self): class FakeResponse: text = html - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_kali_old_versions() self.assertEqual(results, {}) @@ -1152,16 +1616,16 @@ class FakeResponse: def raise_for_status(self): pass - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_fedora_workstation_old_versions() self.assertEqual( results, { - "Fedora-Workstation-Live-x86_64-43": - "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-x86_64-43.torrent", - "Fedora-Workstation-Live-aarch64-43": - "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-aarch64-43.torrent", + "Fedora-Workstation-Live-x86_64-43": "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-x86_64-43.torrent", + "Fedora-Workstation-Live-aarch64-43": "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-aarch64-43.torrent", }, ) @@ -1174,7 +1638,9 @@ class FakeResponse: def raise_for_status(self): pass - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_fedora_workstation_old_versions() self.assertEqual(results, {}) @@ -1191,20 +1657,28 @@ class HasUnmetDemandTests(unittest.TestCase): def test_none_scrape_result_is_no_demand(self): self.assertFalse( - ft.has_unmet_demand(None, min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False) + ft.has_unmet_demand( + None, min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False + ) ) def test_below_absolute_threshold_is_no_demand(self): self.assertFalse( ft.has_unmet_demand( - {'leechers': 0, 'seeders': 0}, min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False + {"leechers": 0, "seeders": 0}, + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) ) def test_single_leecher_on_huge_swarm_is_no_demand(self): self.assertFalse( ft.has_unmet_demand( - {'leechers': 1, 'seeders': 1000}, min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False + {"leechers": 1, "seeders": 1000}, + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) ) @@ -1212,21 +1686,30 @@ def test_meets_absolute_but_not_ratio_is_no_demand(self): # 11 leechers clears a min_leechers=10 floor, but 11 < 1.2x the 10 seeders. self.assertFalse( ft.has_unmet_demand( - {'leechers': 11, 'seeders': 10}, min_leechers=10, min_leecher_ratio=1.2, allow_zero_seeders=False + {"leechers": 11, "seeders": 10}, + min_leechers=10, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) ) def test_meets_both_absolute_and_ratio_thresholds_is_demand(self): self.assertTrue( ft.has_unmet_demand( - {'leechers': 25, 'seeders': 10}, min_leechers=10, min_leecher_ratio=1.2, allow_zero_seeders=False + {"leechers": 25, "seeders": 10}, + min_leechers=10, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) ) def test_exactly_at_ratio_threshold_is_demand(self): self.assertTrue( ft.has_unmet_demand( - {'leechers': 12, 'seeders': 10}, min_leechers=10, min_leecher_ratio=1.2, allow_zero_seeders=False + {"leechers": 12, "seeders": 10}, + min_leechers=10, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) ) @@ -1238,25 +1721,36 @@ def test_zero_seeders_is_excluded_by_default(self): # opts into the gamble via allow_zero_seeders. self.assertFalse( ft.has_unmet_demand( - {'leechers': 10, 'seeders': 0}, min_leechers=10, min_leecher_ratio=1.2, allow_zero_seeders=False + {"leechers": 10, "seeders": 0}, + min_leechers=10, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) ) def test_zero_seeders_counts_as_demand_when_explicitly_allowed(self): self.assertTrue( ft.has_unmet_demand( - {'leechers': 10, 'seeders': 0}, min_leechers=10, min_leecher_ratio=1.2, allow_zero_seeders=True + {"leechers": 10, "seeders": 0}, + min_leechers=10, + min_leecher_ratio=1.2, + allow_zero_seeders=True, ) ) def test_zero_seeders_still_needs_the_absolute_floor_when_allowed(self): self.assertFalse( ft.has_unmet_demand( - {'leechers': 5, 'seeders': 0}, min_leechers=10, min_leecher_ratio=1.2, allow_zero_seeders=True + {"leechers": 5, "seeders": 0}, + min_leechers=10, + min_leecher_ratio=1.2, + allow_zero_seeders=True, ) ) - def test_lopsided_but_substantial_swarms_still_count_as_demand_at_default_ratio(self): + def test_lopsided_but_substantial_swarms_still_count_as_demand_at_default_ratio( + self, + ): # FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHER_RATIO defaults to 0.1: we # have no way of knowing whether existing seeders have spare upload # capacity or are bandwidth-limited, so leechers don't need to @@ -1264,12 +1758,18 @@ def test_lopsided_but_substantial_swarms_still_count_as_demand_at_default_ratio( # and 100:1000 both clearly indicate room for another seeder. self.assertTrue( ft.has_unmet_demand( - {'leechers': 10, 'seeders': 100}, min_leechers=1, min_leecher_ratio=0.1, allow_zero_seeders=False + {"leechers": 10, "seeders": 100}, + min_leechers=1, + min_leecher_ratio=0.1, + allow_zero_seeders=False, ) ) self.assertTrue( ft.has_unmet_demand( - {'leechers': 100, 'seeders': 1000}, min_leechers=1, min_leecher_ratio=0.1, allow_zero_seeders=False + {"leechers": 100, "seeders": 1000}, + min_leechers=1, + min_leecher_ratio=0.1, + allow_zero_seeders=False, ) ) @@ -1279,7 +1779,10 @@ def test_extreme_oversupply_still_fails_at_default_ratio(self): # permissive 0.1 default - it needs at least 67.3 leechers. self.assertFalse( ft.has_unmet_demand( - {'leechers': 3, 'seeders': 673}, min_leechers=1, min_leecher_ratio=0.1, allow_zero_seeders=False + {"leechers": 3, "seeders": 673}, + min_leechers=1, + min_leecher_ratio=0.1, + allow_zero_seeders=False, ) ) @@ -1289,11 +1792,18 @@ def test_404_is_not_kept_and_not_scraped(self): class FakeResponse: status_code = 404 - with unittest.mock.patch.object(ft.requests, 'get', return_value=FakeResponse()), \ - unittest.mock.patch.object(ft.torrent_scrape, 'scrape_torrent') as mock_scrape: + with ( + unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()), + unittest.mock.patch.object( + ft.torrent_scrape, "scrape_torrent" + ) as mock_scrape, + ): keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( - 'ubuntu-14.04.6-desktop-amd64.iso', 'https://example.invalid/x.torrent', - min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False, + "ubuntu-14.04.6-desktop-amd64.iso", + "https://example.invalid/x.torrent", + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) mock_scrape.assert_not_called() @@ -1302,10 +1812,15 @@ class FakeResponse: self.assertIsNone(scrape_result) def test_download_failure_is_not_kept(self): - with unittest.mock.patch.object(ft.requests, 'get', side_effect=ConnectionError("nope")): + with unittest.mock.patch.object( + ft.requests, "get", side_effect=ConnectionError("nope") + ): keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( - 'ubuntu-14.04.6-desktop-amd64.iso', 'https://example.invalid/x.torrent', - min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False, + "ubuntu-14.04.6-desktop-amd64.iso", + "https://example.invalid/x.torrent", + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) self.assertFalse(keep) @@ -1315,75 +1830,108 @@ def test_download_failure_is_not_kept(self): def test_scrape_below_threshold_is_not_kept(self): class FakeResponse: status_code = 200 - content = b'fake torrent bytes' + content = b"fake torrent bytes" def raise_for_status(self): pass - with unittest.mock.patch.object(ft.requests, 'get', return_value=FakeResponse()), \ - unittest.mock.patch.object( - ft.torrent_scrape, 'scrape_torrent', - return_value={'tracker': 'http://t.example/scrape', 'seeders': 10, 'leechers': 0, 'completed': 0}, - ): + with ( + unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()), + unittest.mock.patch.object( + ft.torrent_scrape, + "scrape_torrent", + return_value={ + "tracker": "http://t.example/scrape", + "seeders": 10, + "leechers": 0, + "completed": 0, + }, + ), + ): keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( - 'ubuntu-14.04.6-desktop-amd64.iso', 'https://example.invalid/x.torrent', - min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False, + "ubuntu-14.04.6-desktop-amd64.iso", + "https://example.invalid/x.torrent", + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) self.assertFalse(keep) - self.assertEqual(torrent_bytes, b'fake torrent bytes') - self.assertEqual(scrape_result['leechers'], 0) + self.assertEqual(torrent_bytes, b"fake torrent bytes") + self.assertEqual(scrape_result["leechers"], 0) def test_scrape_meeting_threshold_is_kept(self): class FakeResponse: status_code = 200 - content = b'fake torrent bytes' + content = b"fake torrent bytes" def raise_for_status(self): pass - with unittest.mock.patch.object(ft.requests, 'get', return_value=FakeResponse()), \ - unittest.mock.patch.object( - ft.torrent_scrape, 'scrape_torrent', - return_value={'tracker': 'http://t.example/scrape', 'seeders': 10, 'leechers': 30, 'completed': 0}, - ): + with ( + unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()), + unittest.mock.patch.object( + ft.torrent_scrape, + "scrape_torrent", + return_value={ + "tracker": "http://t.example/scrape", + "seeders": 10, + "leechers": 30, + "completed": 0, + }, + ), + ): keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( - 'ubuntu-14.04.6-desktop-amd64.iso', 'https://example.invalid/x.torrent', - min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False, + "ubuntu-14.04.6-desktop-amd64.iso", + "https://example.invalid/x.torrent", + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) self.assertTrue(keep) - self.assertEqual(torrent_bytes, b'fake torrent bytes') - self.assertEqual(scrape_result['leechers'], 30) + self.assertEqual(torrent_bytes, b"fake torrent bytes") + self.assertEqual(scrape_result["leechers"], 30) def test_meets_absolute_floor_but_already_well_seeded_is_not_kept(self): class FakeResponse: status_code = 200 - content = b'fake torrent bytes' + content = b"fake torrent bytes" def raise_for_status(self): pass # 1 leecher clears a min_leechers=1 floor, but 1000 seeders already # cover it - not unmet demand. - with unittest.mock.patch.object(ft.requests, 'get', return_value=FakeResponse()), \ - unittest.mock.patch.object( - ft.torrent_scrape, 'scrape_torrent', - return_value={'tracker': 'http://t.example/scrape', 'seeders': 1000, 'leechers': 1, 'completed': 0}, - ): + with ( + unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()), + unittest.mock.patch.object( + ft.torrent_scrape, + "scrape_torrent", + return_value={ + "tracker": "http://t.example/scrape", + "seeders": 1000, + "leechers": 1, + "completed": 0, + }, + ), + ): keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( - 'ubuntu-14.04.6-desktop-amd64.iso', 'https://example.invalid/x.torrent', - min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False, + "ubuntu-14.04.6-desktop-amd64.iso", + "https://example.invalid/x.torrent", + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) self.assertFalse(keep) - self.assertEqual(scrape_result['leechers'], 1) + self.assertEqual(scrape_result["leechers"], 1) class CheckOldReleasesForDemandTests(unittest.TestCase): def setUp(self): - self.tmp_watch_dir = tempfile.mkdtemp(prefix='fetch_torrents_test_watch_') - patcher = unittest.mock.patch.object(ft, 'watch_dir', self.tmp_watch_dir) + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) patcher.start() self.addCleanup(patcher.stop) @@ -1391,71 +1939,136 @@ def test_adds_only_torrents_with_unmet_demand(self): # Neither name matches LOW_DEMAND_PATTERN - that filtering is # covered separately by test_low_demand_variant_is_filtered_before_evaluation. candidates = { - 'kali-linux-2026.1-installer-purple-amd64.iso': 'https://example.invalid/dead.torrent', - 'kali-linux-2026.1-installer-amd64.iso': 'https://example.invalid/alive.torrent', + "kali-linux-2026.1-installer-purple-amd64.iso": "https://example.invalid/dead.torrent", + "kali-linux-2026.1-installer-amd64.iso": "https://example.invalid/alive.torrent", } - def fake_evaluate(name, url, min_leechers, min_leecher_ratio, allow_zero_seeders, timeout=15): - if 'purple' in name: - return False, b'dead bytes', {'tracker': 't', 'seeders': 5, 'leechers': 0, 'completed': 0} - return True, b'alive bytes', {'tracker': 't', 'seeders': 5, 'leechers': 30, 'completed': 0} + def fake_evaluate( + name, url, min_leechers, min_leecher_ratio, allow_zero_seeders, timeout=15 + ): + if "purple" in name: + return ( + False, + b"dead bytes", + {"tracker": "t", "seeders": 5, "leechers": 0, "completed": 0}, + ) + return ( + True, + b"alive bytes", + {"tracker": "t", "seeders": 5, "leechers": 30, "completed": 0}, + ) - with unittest.mock.patch.dict(ft.OLD_RELEASE_DISCOVERY, {'kali': lambda: candidates}, clear=True), \ - unittest.mock.patch.object(ft, 'evaluate_old_release_demand', side_effect=fake_evaluate): - added, skipped, _ = ft.check_old_releases_for_demand(['kali'], include_low_demand=False, min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False) + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True + ), + unittest.mock.patch.object( + ft, "evaluate_old_release_demand", side_effect=fake_evaluate + ), + ): + added, skipped, _ = ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + ) self.assertEqual(added, 1) self.assertEqual(skipped, 1) - alive_path = os.path.join(self.tmp_watch_dir, 'kali-linux-2026.1-installer-amd64.iso.torrent') - dead_path = os.path.join(self.tmp_watch_dir, 'kali-linux-2026.1-installer-purple-amd64.iso.torrent') - with open(alive_path, 'rb') as f: - self.assertEqual(f.read(), b'alive bytes') + alive_path = os.path.join( + self.tmp_watch_dir, "kali-linux-2026.1-installer-amd64.iso.torrent" + ) + dead_path = os.path.join( + self.tmp_watch_dir, "kali-linux-2026.1-installer-purple-amd64.iso.torrent" + ) + with open(alive_path, "rb") as f: + self.assertEqual(f.read(), b"alive bytes") self.assertFalse(os.path.exists(dead_path)) def test_skips_distros_without_old_release_discovery(self): with unittest.mock.patch.dict(ft.OLD_RELEASE_DISCOVERY, {}, clear=True): - added, skipped, _ = ft.check_old_releases_for_demand(['ubuntu'], include_low_demand=False, min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False) + added, skipped, _ = ft.check_old_releases_for_demand( + ["ubuntu"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + ) self.assertEqual((added, skipped), (0, 0)) def test_already_present_torrent_is_not_re_evaluated(self): - name = 'kali-linux-2026.1-installer-amd64.iso' - existing_path = os.path.join(self.tmp_watch_dir, f'{name}.torrent') - with open(existing_path, 'wb') as f: - f.write(b'already here') - - with unittest.mock.patch.dict( - ft.OLD_RELEASE_DISCOVERY, - {'kali': lambda: {name: 'https://example.invalid/x.torrent'}}, clear=True, - ), \ - unittest.mock.patch.object(ft, 'evaluate_old_release_demand') as mock_evaluate: - added, skipped, _ = ft.check_old_releases_for_demand(['kali'], include_low_demand=False, min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False) + name = "kali-linux-2026.1-installer-amd64.iso" + existing_path = os.path.join(self.tmp_watch_dir, f"{name}.torrent") + with open(existing_path, "wb") as f: + f.write(b"already here") + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, + {"kali": lambda: {name: "https://example.invalid/x.torrent"}}, + clear=True, + ), + unittest.mock.patch.object( + ft, "evaluate_old_release_demand" + ) as mock_evaluate, + ): + added, skipped, _ = ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + ) mock_evaluate.assert_not_called() self.assertEqual((added, skipped), (0, 0)) def test_unsafe_candidate_name_is_skipped_without_evaluation(self): - name = 'kali-linux-2026.1-installer-../../etc/cron.d/evil' - - with unittest.mock.patch.dict( - ft.OLD_RELEASE_DISCOVERY, - {'kali': lambda: {name: 'https://example.invalid/x.torrent'}}, clear=True, - ), \ - unittest.mock.patch.object(ft, 'evaluate_old_release_demand') as mock_evaluate: - added, skipped, _ = ft.check_old_releases_for_demand(['kali'], include_low_demand=False, min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False) + name = "kali-linux-2026.1-installer-../../etc/cron.d/evil" + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, + {"kali": lambda: {name: "https://example.invalid/x.torrent"}}, + clear=True, + ), + unittest.mock.patch.object( + ft, "evaluate_old_release_demand" + ) as mock_evaluate, + ): + added, skipped, _ = ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + ) mock_evaluate.assert_not_called() self.assertEqual((added, skipped), (0, 1)) def test_low_demand_variant_is_filtered_before_evaluation(self): - name = 'kali-linux-2026.1-installer-netinst-amd64.iso' - - with unittest.mock.patch.dict( - ft.OLD_RELEASE_DISCOVERY, - {'kali': lambda: {name: 'https://example.invalid/x.torrent'}}, clear=True, - ), \ - unittest.mock.patch.object(ft, 'evaluate_old_release_demand') as mock_evaluate: - added, skipped, _ = ft.check_old_releases_for_demand(['kali'], include_low_demand=False, min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False) + name = "kali-linux-2026.1-installer-netinst-amd64.iso" + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, + {"kali": lambda: {name: "https://example.invalid/x.torrent"}}, + clear=True, + ), + unittest.mock.patch.object( + ft, "evaluate_old_release_demand" + ) as mock_evaluate, + ): + added, skipped, _ = ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + ) mock_evaluate.assert_not_called() self.assertEqual((added, skipped), (0, 0)) @@ -1468,7 +2081,11 @@ class FormatRunSummaryTests(unittest.TestCase): def test_without_old_release_check(self): summary = ft.format_run_summary( - elapsed=12.5, success_count=3, existing_count=1, not_found_count=0, failure_count=0, + elapsed=12.5, + success_count=3, + existing_count=1, + not_found_count=0, + failure_count=0, ) self.assertEqual( summary, @@ -1477,8 +2094,13 @@ def test_without_old_release_check(self): def test_with_old_release_check_appends_counts(self): summary = ft.format_run_summary( - elapsed=12.5, success_count=3, existing_count=1, not_found_count=0, failure_count=0, - old_added=2, old_skipped=6, + elapsed=12.5, + success_count=3, + existing_count=1, + not_found_count=0, + failure_count=0, + old_added=2, + old_skipped=6, ) self.assertEqual( summary, @@ -1488,8 +2110,13 @@ def test_with_old_release_check_appends_counts(self): def test_still_starts_with_run_complete_for_always_log_filter(self): summary = ft.format_run_summary( - elapsed=1.0, success_count=0, existing_count=0, not_found_count=0, failure_count=0, - old_added=0, old_skipped=0, + elapsed=1.0, + success_count=0, + existing_count=0, + not_found_count=0, + failure_count=0, + old_added=0, + old_skipped=0, ) self.assertTrue(summary.startswith("Run complete in")) @@ -1501,26 +2128,42 @@ class ShouldRecheckOldReleaseTests(unittest.TestCase): a while instead.""" def test_never_checked_is_due(self): - self.assertTrue(ft.should_recheck_old_release('x', {}, date(2026, 1, 8), recheck_interval_days=7)) + self.assertTrue( + ft.should_recheck_old_release( + "x", {}, date(2026, 1, 8), recheck_interval_days=7 + ) + ) def test_checked_recently_is_not_due(self): - state = {'x': {'last_checked': '2026-01-05', 'leechers': 0}} - self.assertFalse(ft.should_recheck_old_release('x', state, date(2026, 1, 8), recheck_interval_days=7)) + state = {"x": {"last_checked": "2026-01-05", "leechers": 0}} + self.assertFalse( + ft.should_recheck_old_release( + "x", state, date(2026, 1, 8), recheck_interval_days=7 + ) + ) def test_checked_exactly_at_interval_is_due(self): - state = {'x': {'last_checked': '2026-01-01', 'leechers': 0}} - self.assertTrue(ft.should_recheck_old_release('x', state, date(2026, 1, 8), recheck_interval_days=7)) + state = {"x": {"last_checked": "2026-01-01", "leechers": 0}} + self.assertTrue( + ft.should_recheck_old_release( + "x", state, date(2026, 1, 8), recheck_interval_days=7 + ) + ) def test_checked_past_interval_is_due(self): - state = {'x': {'last_checked': '2025-12-01', 'leechers': 0}} - self.assertTrue(ft.should_recheck_old_release('x', state, date(2026, 1, 8), recheck_interval_days=7)) + state = {"x": {"last_checked": "2025-12-01", "leechers": 0}} + self.assertTrue( + ft.should_recheck_old_release( + "x", state, date(2026, 1, 8), recheck_interval_days=7 + ) + ) class OldReleaseCheckStatePersistenceTests(unittest.TestCase): def test_round_trips_through_disk(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_old_release_state_') - path = os.path.join(tmp_dir, 'fetch_torrents_old_release_check_state.json') - state = {'x': {'last_checked': '2026-01-08', 'leechers': 0}} + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_old_release_state_") + path = os.path.join(tmp_dir, "fetch_torrents_old_release_check_state.json") + state = {"x": {"last_checked": "2026-01-08", "leechers": 0}} ft.save_old_release_check_state(path, state) loaded = ft.load_old_release_check_state(path) @@ -1528,16 +2171,16 @@ def test_round_trips_through_disk(self): self.assertEqual(loaded, state) def test_missing_file_returns_empty_state(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_old_release_state_') - path = os.path.join(tmp_dir, 'does-not-exist.json') + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_old_release_state_") + path = os.path.join(tmp_dir, "does-not-exist.json") self.assertEqual(ft.load_old_release_check_state(path), {}) def test_corrupt_file_returns_empty_state_instead_of_raising(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_old_release_state_') - path = os.path.join(tmp_dir, 'fetch_torrents_old_release_check_state.json') - with open(path, 'w', encoding='utf-8') as f: - f.write('{not valid json') + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_old_release_state_") + path = os.path.join(tmp_dir, "fetch_torrents_old_release_check_state.json") + with open(path, "w", encoding="utf-8") as f: + f.write("{not valid json") self.assertEqual(ft.load_old_release_check_state(path), {}) @@ -1546,21 +2189,25 @@ class RecordRemovalTests(unittest.TestCase): def test_adds_entry_without_mutating_input(self): history = {} updated = ft.record_removal( - history, 'kali-linux-2026.1-installer-amd64.iso', date(2026, 2, 1), 'stagnant', 0.42, + history, + "kali-linux-2026.1-installer-amd64.iso", + date(2026, 2, 1), + "stagnant", + 0.42, ) self.assertEqual(history, {}) # input untouched self.assertEqual( - updated['kali-linux-2026.1-installer-amd64.iso'], - {'removed_date': '2026-02-01', 'reason': 'stagnant', 'ratio': 0.42}, + updated["kali-linux-2026.1-installer-amd64.iso"], + {"removed_date": "2026-02-01", "reason": "stagnant", "ratio": 0.42}, ) class RemovedHistoryPersistenceTests(unittest.TestCase): def test_round_trips_through_disk(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_removed_history_') - path = os.path.join(tmp_dir, 'fetch_torrents_removed_history.json') - history = {'x': {'removed_date': '2026-02-01', 'reason': 'stagnant'}} + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_removed_history_") + path = os.path.join(tmp_dir, "fetch_torrents_removed_history.json") + history = {"x": {"removed_date": "2026-02-01", "reason": "stagnant"}} ft.save_removed_history(path, history) loaded = ft.load_removed_history(path) @@ -1568,16 +2215,16 @@ def test_round_trips_through_disk(self): self.assertEqual(loaded, history) def test_missing_file_returns_empty_history(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_removed_history_') - path = os.path.join(tmp_dir, 'does-not-exist.json') + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_removed_history_") + path = os.path.join(tmp_dir, "does-not-exist.json") self.assertEqual(ft.load_removed_history(path), {}) def test_corrupt_file_returns_empty_history_instead_of_raising(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_removed_history_') - path = os.path.join(tmp_dir, 'fetch_torrents_removed_history.json') - with open(path, 'w', encoding='utf-8') as f: - f.write('{not valid json') + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_removed_history_") + path = os.path.join(tmp_dir, "fetch_torrents_removed_history.json") + with open(path, "w", encoding="utf-8") as f: + f.write("{not valid json") self.assertEqual(ft.load_removed_history(path), {}) @@ -1589,21 +2236,34 @@ class CheckOldReleasesForDemandStateTests(unittest.TestCase): network, and must record a fresh check for anything it does evaluate.""" def setUp(self): - self.tmp_watch_dir = tempfile.mkdtemp(prefix='fetch_torrents_test_watch_') - patcher = unittest.mock.patch.object(ft, 'watch_dir', self.tmp_watch_dir) + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) patcher.start() self.addCleanup(patcher.stop) def test_recently_checked_candidate_is_skipped_without_network_call(self): - name = 'kali-linux-2026.1-installer-amd64.iso' - candidates = {name: 'https://example.invalid/x.torrent'} - check_state = {name: {'last_checked': '2026-01-05', 'leechers': 0}} - - with unittest.mock.patch.dict(ft.OLD_RELEASE_DISCOVERY, {'kali': lambda: candidates}, clear=True), \ - unittest.mock.patch.object(ft, 'evaluate_old_release_demand') as mock_evaluate: + name = "kali-linux-2026.1-installer-amd64.iso" + candidates = {name: "https://example.invalid/x.torrent"} + check_state = {name: {"last_checked": "2026-01-05", "leechers": 0}} + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True + ), + unittest.mock.patch.object( + ft, "evaluate_old_release_demand" + ) as mock_evaluate, + ): added, skipped, new_state = ft.check_old_releases_for_demand( - ['kali'], include_low_demand=False, min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False, - check_state=check_state, removed_history={}, today=date(2026, 1, 8), recheck_interval_days=7, + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + check_state=check_state, + removed_history={}, + today=date(2026, 1, 8), + recheck_interval_days=7, ) mock_evaluate.assert_not_called() @@ -1611,53 +2271,100 @@ def test_recently_checked_candidate_is_skipped_without_network_call(self): self.assertEqual(new_state, check_state) def test_previously_removed_candidate_is_skipped_without_network_call(self): - name = 'kali-linux-2026.1-installer-amd64.iso' - candidates = {name: 'https://example.invalid/x.torrent'} - removed_history = {name: {'removed_date': '2026-01-01', 'reason': 'stagnant'}} - - with unittest.mock.patch.dict(ft.OLD_RELEASE_DISCOVERY, {'kali': lambda: candidates}, clear=True), \ - unittest.mock.patch.object(ft, 'evaluate_old_release_demand') as mock_evaluate: + name = "kali-linux-2026.1-installer-amd64.iso" + candidates = {name: "https://example.invalid/x.torrent"} + removed_history = {name: {"removed_date": "2026-01-01", "reason": "stagnant"}} + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True + ), + unittest.mock.patch.object( + ft, "evaluate_old_release_demand" + ) as mock_evaluate, + ): added, skipped, new_state = ft.check_old_releases_for_demand( - ['kali'], include_low_demand=False, min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False, - check_state={}, removed_history=removed_history, today=date(2026, 1, 8), recheck_interval_days=7, + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + check_state={}, + removed_history=removed_history, + today=date(2026, 1, 8), + recheck_interval_days=7, ) mock_evaluate.assert_not_called() self.assertEqual((added, skipped), (0, 1)) def test_due_candidate_is_evaluated_and_recorded(self): - name = 'kali-linux-2026.1-installer-amd64.iso' - candidates = {name: 'https://example.invalid/x.torrent'} - - with unittest.mock.patch.dict(ft.OLD_RELEASE_DISCOVERY, {'kali': lambda: candidates}, clear=True), \ - unittest.mock.patch.object( - ft, 'evaluate_old_release_demand', - return_value=(True, b'bytes', {'tracker': 't', 'seeders': 1, 'leechers': 30, 'completed': 0}), - ): + name = "kali-linux-2026.1-installer-amd64.iso" + candidates = {name: "https://example.invalid/x.torrent"} + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True + ), + unittest.mock.patch.object( + ft, + "evaluate_old_release_demand", + return_value=( + True, + b"bytes", + {"tracker": "t", "seeders": 1, "leechers": 30, "completed": 0}, + ), + ), + ): added, skipped, new_state = ft.check_old_releases_for_demand( - ['kali'], include_low_demand=False, min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False, - check_state={}, removed_history={}, today=date(2026, 1, 8), recheck_interval_days=7, + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + check_state={}, + removed_history={}, + today=date(2026, 1, 8), + recheck_interval_days=7, ) self.assertEqual((added, skipped), (1, 0)) - self.assertEqual(new_state[name], {'last_checked': '2026-01-08', 'leechers': 30}) + self.assertEqual( + new_state[name], {"last_checked": "2026-01-08", "leechers": 30} + ) def test_no_demand_result_is_still_recorded_for_future_recheck_gating(self): - name = 'kali-linux-2026.1-installer-arm64.iso' - candidates = {name: 'https://example.invalid/x.torrent'} - - with unittest.mock.patch.dict(ft.OLD_RELEASE_DISCOVERY, {'kali': lambda: candidates}, clear=True), \ - unittest.mock.patch.object( - ft, 'evaluate_old_release_demand', - return_value=(False, b'bytes', {'tracker': 't', 'seeders': 1, 'leechers': 0, 'completed': 0}), - ): + name = "kali-linux-2026.1-installer-arm64.iso" + candidates = {name: "https://example.invalid/x.torrent"} + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True + ), + unittest.mock.patch.object( + ft, + "evaluate_old_release_demand", + return_value=( + False, + b"bytes", + {"tracker": "t", "seeders": 1, "leechers": 0, "completed": 0}, + ), + ), + ): added, skipped, new_state = ft.check_old_releases_for_demand( - ['kali'], include_low_demand=False, min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False, - check_state={}, removed_history={}, today=date(2026, 1, 8), recheck_interval_days=7, + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + check_state={}, + removed_history={}, + today=date(2026, 1, 8), + recheck_interval_days=7, ) self.assertEqual((added, skipped), (0, 1)) - self.assertEqual(new_state[name], {'last_checked': '2026-01-08', 'leechers': 0}) + self.assertEqual(new_state[name], {"last_checked": "2026-01-08", "leechers": 0}) class CleanupRpcAuthTests(unittest.TestCase): @@ -1670,12 +2377,14 @@ class CleanupRpcAuthTests(unittest.TestCase): failure is only logged, never surfaced.""" def setUp(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_cleanup_auth_') + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_cleanup_auth_") for attr, filename in ( - ('removed_history_file', 'removed_history.json'), - ('ratio_history_file', 'ratio_history.json'), + ("removed_history_file", "removed_history.json"), + ("ratio_history_file", "ratio_history.json"), ): - patcher = unittest.mock.patch.object(ft, attr, os.path.join(tmp_dir, filename)) + patcher = unittest.mock.patch.object( + ft, attr, os.path.join(tmp_dir, filename) + ) patcher.start() self.addCleanup(patcher.stop) @@ -1685,29 +2394,55 @@ def _mock_client(self): return client def test_cleanup_old_versions_passes_configured_credentials(self): - with unittest.mock.patch.dict(os.environ, { - 'TRANSMISSION_RPC_USERNAME': 'alice', 'TRANSMISSION_RPC_PASSWORD': 'hunter2', - }), unittest.mock.patch.object(ft, 'Client', return_value=self._mock_client()) as mock_client_cls: + with ( + unittest.mock.patch.dict( + os.environ, + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + }, + ), + unittest.mock.patch.object( + ft, "Client", return_value=self._mock_client() + ) as mock_client_cls, + ): ft.cleanup_old_versions() - mock_client_cls.assert_called_once_with(host='localhost', port=9091, username='alice', password='hunter2') + mock_client_cls.assert_called_once_with( + host="localhost", port=9091, username="alice", password="hunter2" + ) def test_cleanup_old_versions_passes_no_credentials_when_unset(self): - with unittest.mock.patch.dict(os.environ, {}, clear=False): - os.environ.pop('TRANSMISSION_RPC_USERNAME', None) - os.environ.pop('TRANSMISSION_RPC_PASSWORD', None) - with unittest.mock.patch.object(ft, 'Client', return_value=self._mock_client()) as mock_client_cls: - ft.cleanup_old_versions() + with ( + helpers.isolated_env(), + unittest.mock.patch.object( + ft, "Client", return_value=self._mock_client() + ) as mock_client_cls, + ): + ft.cleanup_old_versions() - mock_client_cls.assert_called_once_with(host='localhost', port=9091, username=None, password=None) + mock_client_cls.assert_called_once_with( + host="localhost", port=9091, username=None, password=None + ) def test_cleanup_stagnant_torrents_passes_configured_credentials(self): - with unittest.mock.patch.dict(os.environ, { - 'TRANSMISSION_RPC_USERNAME': 'alice', 'TRANSMISSION_RPC_PASSWORD': 'hunter2', - }), unittest.mock.patch.object(ft, 'Client', return_value=self._mock_client()) as mock_client_cls: + with ( + unittest.mock.patch.dict( + os.environ, + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + }, + ), + unittest.mock.patch.object( + ft, "Client", return_value=self._mock_client() + ) as mock_client_cls, + ): ft.cleanup_stagnant_torrents() - mock_client_cls.assert_called_once_with(host='localhost', port=9091, username='alice', password='hunter2') + mock_client_cls.assert_called_once_with( + host="localhost", port=9091, username="alice", password="hunter2" + ) class LogSeedRatiosViaHttpTests(unittest.TestCase): @@ -1728,50 +2463,77 @@ def json(self): def test_session_id_request_has_a_timeout(self): with unittest.mock.patch.object( - ft.requests, 'post', + ft.requests, + "post", side_effect=[self.FakeSessionIdResponse(), self.FakeTorrentGetResponse()], ) as mock_post: ft.log_seed_ratios_via_http() first_call_kwargs = mock_post.call_args_list[0].kwargs - self.assertIn('timeout', first_call_kwargs) + self.assertIn("timeout", first_call_kwargs) def test_uses_configured_credentials_when_auth_not_explicitly_passed(self): - with unittest.mock.patch.dict(os.environ, { - 'TRANSMISSION_RPC_USERNAME': 'alice', 'TRANSMISSION_RPC_PASSWORD': 'hunter2', - }), unittest.mock.patch.object( - ft.requests, 'post', - side_effect=[self.FakeSessionIdResponse(), self.FakeTorrentGetResponse()], - ) as mock_post: + with ( + unittest.mock.patch.dict( + os.environ, + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + }, + ), + unittest.mock.patch.object( + ft.requests, + "post", + side_effect=[ + self.FakeSessionIdResponse(), + self.FakeTorrentGetResponse(), + ], + ) as mock_post, + ): ft.log_seed_ratios_via_http() second_call_kwargs = mock_post.call_args_list[1].kwargs - self.assertEqual(second_call_kwargs.get('auth'), ('alice', 'hunter2')) + self.assertEqual(second_call_kwargs.get("auth"), ("alice", "hunter2")) def test_no_auth_when_credentials_unset(self): - with unittest.mock.patch.dict(os.environ, {}, clear=False): - os.environ.pop('TRANSMISSION_RPC_USERNAME', None) - os.environ.pop('TRANSMISSION_RPC_PASSWORD', None) - with unittest.mock.patch.object( - ft.requests, 'post', - side_effect=[self.FakeSessionIdResponse(), self.FakeTorrentGetResponse()], - ) as mock_post: - ft.log_seed_ratios_via_http() + with ( + helpers.isolated_env(), + unittest.mock.patch.object( + ft.requests, + "post", + side_effect=[ + self.FakeSessionIdResponse(), + self.FakeTorrentGetResponse(), + ], + ) as mock_post, + ): + ft.log_seed_ratios_via_http() second_call_kwargs = mock_post.call_args_list[1].kwargs - self.assertIsNone(second_call_kwargs.get('auth')) + self.assertIsNone(second_call_kwargs.get("auth")) def test_explicit_auth_argument_still_wins(self): - with unittest.mock.patch.dict(os.environ, { - 'TRANSMISSION_RPC_USERNAME': 'alice', 'TRANSMISSION_RPC_PASSWORD': 'hunter2', - }), unittest.mock.patch.object( - ft.requests, 'post', - side_effect=[self.FakeSessionIdResponse(), self.FakeTorrentGetResponse()], - ) as mock_post: - ft.log_seed_ratios_via_http(auth=('bob', 'other')) + with ( + unittest.mock.patch.dict( + os.environ, + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + }, + ), + unittest.mock.patch.object( + ft.requests, + "post", + side_effect=[ + self.FakeSessionIdResponse(), + self.FakeTorrentGetResponse(), + ], + ) as mock_post, + ): + ft.log_seed_ratios_via_http(auth=("bob", "other")) second_call_kwargs = mock_post.call_args_list[1].kwargs - self.assertEqual(second_call_kwargs.get('auth'), ('bob', 'other')) + self.assertEqual(second_call_kwargs.get("auth"), ("bob", "other")) class CleanupRemovesWatchFileTests(unittest.TestCase): @@ -1781,67 +2543,73 @@ class CleanupRemovesWatchFileTests(unittest.TestCase): by default) ever cleans them up.""" def setUp(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_cleanup_watchfile_') + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_cleanup_watchfile_") for attr, filename in ( - ('removed_history_file', 'removed_history.json'), - ('ratio_history_file', 'ratio_history.json'), + ("removed_history_file", "removed_history.json"), + ("ratio_history_file", "ratio_history.json"), ): - patcher = unittest.mock.patch.object(ft, attr, os.path.join(tmp_dir, filename)) + patcher = unittest.mock.patch.object( + ft, attr, os.path.join(tmp_dir, filename) + ) patcher.start() self.addCleanup(patcher.stop) - self.tmp_watch_dir = tempfile.mkdtemp(prefix='fetch_torrents_test_watch_') - patcher = unittest.mock.patch.object(ft, 'watch_dir', self.tmp_watch_dir) + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) patcher.start() self.addCleanup(patcher.stop) def _touch_watch_file(self, name): path = os.path.join(self.tmp_watch_dir, f"{name}.torrent") - with open(path, 'w', encoding='utf-8') as f: - f.write('fake torrent bytes') + with open(path, "w", encoding="utf-8") as f: + f.write("fake torrent bytes") return path def test_cleanup_old_versions_removes_the_watch_file(self): - old_path = self._touch_watch_file('ubuntu-23.10-desktop-amd64.iso') - self._touch_watch_file('ubuntu-24.04-desktop-amd64.iso') + old_path = self._touch_watch_file("ubuntu-23.10-desktop-amd64.iso") + self._touch_watch_file("ubuntu-24.04-desktop-amd64.iso") torrents = [ - FakeTorrent('ubuntu-24.04-desktop-amd64.iso', 1, ratio=2.0), - FakeTorrent('ubuntu-23.10-desktop-amd64.iso', 2, ratio=2.0), + FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0), ] client = MagicMock() client.get_torrents.return_value = torrents - with unittest.mock.patch.dict(os.environ, {'CLEANUP_SKIP_RATIO_CHECK': 'true'}), \ - unittest.mock.patch.object(ft, 'Client', return_value=client): + with ( + unittest.mock.patch.dict(os.environ, {"CLEANUP_SKIP_RATIO_CHECK": "true"}), + unittest.mock.patch.object(ft, "Client", return_value=client), + ): ft.cleanup_old_versions() client.remove_torrent.assert_called_once_with(2, delete_data=True) self.assertFalse(os.path.exists(old_path)) history = ft.load_removed_history(ft.removed_history_file) - self.assertEqual(history['ubuntu-23.10-desktop-amd64.iso']['ratio'], 2.0) + self.assertEqual(history["ubuntu-23.10-desktop-amd64.iso"]["ratio"], 2.0) def test_missing_watch_file_does_not_raise(self): # No .torrent file created for the superseded version - cleanup must # still succeed and remove the torrent from Transmission. torrents = [ - FakeTorrent('ubuntu-24.04-desktop-amd64.iso', 1, ratio=2.0), - FakeTorrent('ubuntu-23.10-desktop-amd64.iso', 2, ratio=2.0), + FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0), ] client = MagicMock() client.get_torrents.return_value = torrents - with unittest.mock.patch.dict(os.environ, {'CLEANUP_SKIP_RATIO_CHECK': 'true'}), \ - unittest.mock.patch.object(ft, 'Client', return_value=client): + with ( + unittest.mock.patch.dict(os.environ, {"CLEANUP_SKIP_RATIO_CHECK": "true"}), + unittest.mock.patch.object(ft, "Client", return_value=client), + ): ft.cleanup_old_versions() # must not raise client.remove_torrent.assert_called_once_with(2, delete_data=True) def test_cleanup_stagnant_torrents_removes_the_watch_file(self): - old_path = self._touch_watch_file('ubuntu-23.10-desktop-amd64.iso') - self._touch_watch_file('ubuntu-24.04-desktop-amd64.iso') + old_path = self._touch_watch_file("ubuntu-23.10-desktop-amd64.iso") + self._touch_watch_file("ubuntu-24.04-desktop-amd64.iso") torrents = [ - FakeTorrent('ubuntu-24.04-desktop-amd64.iso', 1, ratio=2.0), - FakeTorrent('ubuntu-23.10-desktop-amd64.iso', 2, ratio=2.0), + FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0), ] client = MagicMock() client.get_torrents.return_value = torrents @@ -1852,18 +2620,25 @@ def test_cleanup_stagnant_torrents_removes_the_watch_file(self): # history - before plan_stagnation_cleanup() ever sees it. Ratio # unchanged since then, so plan_stagnation_cleanup() must select this # torrent for removal. - anchor_date = (date.today() - timedelta(days=32)).isoformat() - history = {'ubuntu-23.10-desktop-amd64.iso': [{'date': anchor_date, 'ratio': 2.0}]} - with open(ft.ratio_history_file, 'w', encoding='utf-8') as f: + # The clock is pinned so a run crossing midnight can't shift the offset. + today = date(2026, 8, 1) + anchor_date = (today - timedelta(days=32)).isoformat() + history = { + "ubuntu-23.10-desktop-amd64.iso": [{"date": anchor_date, "ratio": 2.0}] + } + with open(ft.ratio_history_file, "w", encoding="utf-8") as f: json.dump(history, f) - with unittest.mock.patch.object(ft, 'Client', return_value=client): + with ( + unittest.mock.patch.object(ft, "Client", return_value=client), + unittest.mock.patch.object(ft, "date", fixed_date(today)), + ): ft.cleanup_stagnant_torrents() client.remove_torrent.assert_called_once_with(2, delete_data=True) self.assertFalse(os.path.exists(old_path)) history = ft.load_removed_history(ft.removed_history_file) - self.assertEqual(history['ubuntu-23.10-desktop-amd64.iso']['ratio'], 2.0) + self.assertEqual(history["ubuntu-23.10-desktop-amd64.iso"]["ratio"], 2.0) class PlanDiskPressureCleanupTests(unittest.TestCase): @@ -1877,24 +2652,45 @@ def test_below_threshold_returns_nothing(self): FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1), ] - self.assertEqual(ft.plan_disk_pressure_cleanup(torrents, used_percent=80.0, threshold_percent=95.0), []) + self.assertEqual( + ft.plan_disk_pressure_cleanup( + torrents, used_percent=80.0, threshold_percent=95.0 + ), + [], + ) def test_disabled_when_threshold_is_zero_or_negative(self): torrents = [ FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1), ] - self.assertEqual(ft.plan_disk_pressure_cleanup(torrents, used_percent=99.0, threshold_percent=0), []) - self.assertEqual(ft.plan_disk_pressure_cleanup(torrents, used_percent=99.0, threshold_percent=-1), []) + self.assertEqual( + ft.plan_disk_pressure_cleanup( + torrents, used_percent=99.0, threshold_percent=0 + ), + [], + ) + self.assertEqual( + ft.plan_disk_pressure_cleanup( + torrents, used_percent=99.0, threshold_percent=-1 + ), + [], + ) def test_at_or_above_threshold_returns_candidates_lowest_ratio_first(self): torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0), # latest - never a candidate + FakeTorrent( + "ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0 + ), # latest - never a candidate FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.5), FakeTorrent("ubuntu-22.04.1-live-server-amd64.iso", 3, ratio=0.2), - FakeTorrent("ubuntu-24.04.1-live-server-amd64.iso", 4, ratio=5.0), # latest of this group + FakeTorrent( + "ubuntu-24.04.1-live-server-amd64.iso", 4, ratio=5.0 + ), # latest of this group ] - result = ft.plan_disk_pressure_cleanup(torrents, used_percent=95.0, threshold_percent=95.0) + result = ft.plan_disk_pressure_cleanup( + torrents, used_percent=95.0, threshold_percent=95.0 + ) self.assertEqual( names_of(result), ["ubuntu-22.04.1-live-server-amd64.iso", "ubuntu-23.10-desktop-amd64.iso"], @@ -1904,7 +2700,12 @@ def test_at_or_above_threshold_returns_candidates_lowest_ratio_first(self): def test_no_superseded_candidates_returns_empty(self): torrents = [FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=0.0)] - self.assertEqual(ft.plan_disk_pressure_cleanup(torrents, used_percent=99.0, threshold_percent=95.0), []) + self.assertEqual( + ft.plan_disk_pressure_cleanup( + torrents, used_percent=99.0, threshold_percent=95.0 + ), + [], + ) class EnforceDiskUsageLimitTests(unittest.TestCase): @@ -1914,38 +2715,48 @@ class EnforceDiskUsageLimitTests(unittest.TestCase): torrent holds.""" def setUp(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_disk_pressure_') + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_disk_pressure_") for attr, filename in ( - ('removed_history_file', 'removed_history.json'), - ('ratio_history_file', 'ratio_history.json'), + ("removed_history_file", "removed_history.json"), + ("ratio_history_file", "ratio_history.json"), ): - patcher = unittest.mock.patch.object(ft, attr, os.path.join(tmp_dir, filename)) + patcher = unittest.mock.patch.object( + ft, attr, os.path.join(tmp_dir, filename) + ) patcher.start() self.addCleanup(patcher.stop) - self.tmp_watch_dir = tempfile.mkdtemp(prefix='fetch_torrents_test_watch_') - patcher = unittest.mock.patch.object(ft, 'watch_dir', self.tmp_watch_dir) + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) patcher.start() self.addCleanup(patcher.stop) def test_disabled_threshold_never_touches_disk_or_rpc(self): - with unittest.mock.patch.object(ft.shutil, 'disk_usage') as mock_disk_usage, \ - unittest.mock.patch.object(ft, 'Client') as mock_client_cls: + with ( + unittest.mock.patch.object(ft.shutil, "disk_usage") as mock_disk_usage, + unittest.mock.patch.object(ft, "Client") as mock_client_cls, + ): ft.enforce_disk_usage_limit(0) mock_disk_usage.assert_not_called() mock_client_cls.assert_not_called() def test_below_threshold_does_not_connect_to_transmission(self): - with unittest.mock.patch.object(ft.shutil, 'disk_usage', return_value=(100, 50, 50)), \ - unittest.mock.patch.object(ft, 'Client') as mock_client_cls: + with ( + unittest.mock.patch.object( + ft.shutil, "disk_usage", return_value=(100, 50, 50) + ), + unittest.mock.patch.object(ft, "Client") as mock_client_cls, + ): ft.enforce_disk_usage_limit(95) mock_client_cls.assert_not_called() def test_removes_lowest_ratio_candidates_until_under_threshold(self): torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0), # latest, never removed + FakeTorrent( + "ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0 + ), # latest, never removed FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.5), FakeTorrent("ubuntu-22.04-desktop-amd64.iso", 3, ratio=0.2), ] @@ -1955,17 +2766,24 @@ def test_removes_lowest_ratio_candidates_until_under_threshold(self): # 96% initially (over threshold); drops to 90% (under threshold) # after the first removal - the second, higher-ratio candidate must # be left alone. - with unittest.mock.patch.object( - ft.shutil, 'disk_usage', side_effect=[(100, 96, 4), (100, 96, 4), (100, 90, 10)], - ), unittest.mock.patch.object(ft, 'Client', return_value=client): + with ( + unittest.mock.patch.object( + ft.shutil, + "disk_usage", + side_effect=[(100, 96, 4), (100, 96, 4), (100, 90, 10)], + ), + unittest.mock.patch.object(ft, "Client", return_value=client), + ): ft.enforce_disk_usage_limit(95) client.remove_torrent.assert_called_once_with(3, delete_data=True) def test_removes_the_watch_file_for_each_removal(self): - watch_path = os.path.join(self.tmp_watch_dir, "ubuntu-22.04-desktop-amd64.iso.torrent") - with open(watch_path, 'w', encoding='utf-8') as f: - f.write('fake torrent bytes') + watch_path = os.path.join( + self.tmp_watch_dir, "ubuntu-22.04-desktop-amd64.iso.torrent" + ) + with open(watch_path, "w", encoding="utf-8") as f: + f.write("fake torrent bytes") torrents = [ FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0), @@ -1975,9 +2793,14 @@ def test_removes_the_watch_file_for_each_removal(self): client = MagicMock() client.get_torrents.return_value = torrents - with unittest.mock.patch.object( - ft.shutil, 'disk_usage', side_effect=[(100, 96, 4), (100, 96, 4), (100, 90, 10)], - ), unittest.mock.patch.object(ft, 'Client', return_value=client): + with ( + unittest.mock.patch.object( + ft.shutil, + "disk_usage", + side_effect=[(100, 96, 4), (100, 96, 4), (100, 90, 10)], + ), + unittest.mock.patch.object(ft, "Client", return_value=client), + ): ft.enforce_disk_usage_limit(95) self.assertFalse(os.path.exists(watch_path)) @@ -1987,8 +2810,12 @@ def test_no_candidates_logs_warning_and_does_not_raise(self): client = MagicMock() client.get_torrents.return_value = torrents - with unittest.mock.patch.object(ft.shutil, 'disk_usage', return_value=(100, 96, 4)), \ - unittest.mock.patch.object(ft, 'Client', return_value=client): + with ( + unittest.mock.patch.object( + ft.shutil, "disk_usage", return_value=(100, 96, 4) + ), + unittest.mock.patch.object(ft, "Client", return_value=client), + ): ft.enforce_disk_usage_limit(95) # must not raise client.remove_torrent.assert_not_called() @@ -2002,14 +2829,21 @@ def test_records_removal_reason_as_disk_pressure(self): client = MagicMock() client.get_torrents.return_value = torrents - with unittest.mock.patch.object( - ft.shutil, 'disk_usage', side_effect=[(100, 96, 4), (100, 96, 4), (100, 90, 10)], - ), unittest.mock.patch.object(ft, 'Client', return_value=client): + with ( + unittest.mock.patch.object( + ft.shutil, + "disk_usage", + side_effect=[(100, 96, 4), (100, 96, 4), (100, 90, 10)], + ), + unittest.mock.patch.object(ft, "Client", return_value=client), + ): ft.enforce_disk_usage_limit(95) history = ft.load_removed_history(ft.removed_history_file) - self.assertEqual(history["ubuntu-22.04-desktop-amd64.iso"]['reason'], 'disk_pressure') - self.assertEqual(history["ubuntu-22.04-desktop-amd64.iso"]['ratio'], 0.2) + self.assertEqual( + history["ubuntu-22.04-desktop-amd64.iso"]["reason"], "disk_pressure" + ) + self.assertEqual(history["ubuntu-22.04-desktop-amd64.iso"]["ratio"], 0.2) if __name__ == "__main__": diff --git a/tests/test_torrent_scrape.py b/tests/test_torrent_scrape.py index fc8b9f2..5813f14 100644 --- a/tests/test_torrent_scrape.py +++ b/tests/test_torrent_scrape.py @@ -6,6 +6,7 @@ Run with: python -m unittest discover -s tests """ + import hashlib import os import socket @@ -26,53 +27,92 @@ def bstr(s): """Hand-rolled bencode byte-string, independent of ts.bencode(), so it serves as an oracle rather than testing the implementation against itself.""" - b = s if isinstance(s, bytes) else s.encode('utf-8') - return str(len(b)).encode() + b':' + b + b = s if isinstance(s, bytes) else s.encode("utf-8") + return str(len(b)).encode() + b":" + b def bint(n): - return b'i' + str(n).encode() + b'e' + return b"i" + str(n).encode() + b"e" class BdecodeBencodeTests(unittest.TestCase): def test_bdecode_integer(self): - value, index = ts.bdecode(b'i12345e') + value, index = ts.bdecode(b"i12345e") self.assertEqual(value, 12345) self.assertEqual(index, 7) def test_bdecode_negative_integer(self): - value, index = ts.bdecode(b'i-42e') + value, index = ts.bdecode(b"i-42e") self.assertEqual(value, -42) def test_bdecode_bytestring(self): - value, index = ts.bdecode(b'4:spam trailing') - self.assertEqual(value, b'spam') + value, index = ts.bdecode(b"4:spam trailing") + self.assertEqual(value, b"spam") self.assertEqual(index, 6) def test_bdecode_empty_bytestring(self): - value, index = ts.bdecode(b'0:') - self.assertEqual(value, b'') + value, index = ts.bdecode(b"0:") + self.assertEqual(value, b"") self.assertEqual(index, 2) def test_bdecode_list(self): - value, index = ts.bdecode(b'l4:spam4:eggse') - self.assertEqual(value, [b'spam', b'eggs']) + value, index = ts.bdecode(b"l4:spam4:eggse") + self.assertEqual(value, [b"spam", b"eggs"]) def test_bdecode_dict(self): - value, index = ts.bdecode(b'd3:cow3:moo4:spam4:eggse') - self.assertEqual(value, {b'cow': b'moo', b'spam': b'eggs'}) + value, index = ts.bdecode(b"d3:cow3:moo4:spam4:eggse") + self.assertEqual(value, {b"cow": b"moo", b"spam": b"eggs"}) def test_bdecode_nested(self): - data = b'd4:infod4:name4:test6:lengthi10eee' + data = b"d4:infod4:name4:test6:lengthi10eee" value, _ = ts.bdecode(data) - self.assertEqual(value, {b'info': {b'name': b'test', b'length': 10}}) + self.assertEqual(value, {b"info": {b"name": b"test", b"length": 10}}) def test_bdecode_invalid_prefix_raises(self): with self.assertRaises(ts.BencodeError): - ts.bdecode(b'x123') + ts.bdecode(b"x123") + + def test_bdecode_malformed_input_raises_bencode_error(self): + # A truncated .torrent must fail loudly: decoding it silently gives + # a wrong info dict, hence a wrong info-hash and a bogus scrape. + cases = { + b"4:ab": "declares 4 bytes, has 2", + b"i12": "integer without terminating e", + b"iabce": "non-numeric integer", + b"i1_0e": "int() accepts underscores, bencode doesn't", + b"i e": "blank integer", + b"2x:ab": "non-numeric length", + b"12": "length without colon", + b"l4:spam": "unterminated list", + b"d3:cow3:moo": "unterminated dict", + } + for data, problem in cases.items(): + with self.subTest(problem, data=data): + with self.assertRaises(ts.BencodeError) as ctx: + ts.bdecode(data) + self.assertRegex(str(ctx.exception), r"index \d+") + + def test_bdecode_truncated_string_reports_declared_and_remaining_bytes(self): + with self.assertRaisesRegex( + ts.BencodeError, r"index 0 declares 4 bytes, only 2 remain" + ): + ts.bdecode(b"4:ab") + + def test_bdecode_unterminated_list_reports_end_of_data(self): + with self.assertRaisesRegex( + ts.BencodeError, r"Unexpected end of data at index 7" + ): + ts.bdecode(b"l4:spam") + + def test_bdecode_truncated_string_inside_container_raises(self): + with self.assertRaises(ts.BencodeError): + ts.bdecode(b"d4:info10:shorte") + + def test_bdecode_string_ending_exactly_at_end_of_data_is_accepted(self): + self.assertEqual(ts.bdecode(b"4:spam"), (b"spam", 6)) def test_bencode_roundtrip(self): - for value in (0, -5, 12345, b'', b'hello', [b'a', b'b'], {b'a': 1, b'b': b'x'}): + for value in (0, -5, 12345, b"", b"hello", [b"a", b"b"], {b"a": 1, b"b": b"x"}): encoded = ts.bencode(value) decoded, index = ts.bdecode(encoded) self.assertEqual(index, len(encoded)) @@ -80,24 +120,32 @@ def test_bencode_roundtrip(self): def test_bencode_dict_sorts_keys(self): # Insertion order deliberately reversed vs. bencode's required sort order. - encoded = ts.bencode({b'zebra': 1, b'apple': 2}) - self.assertEqual(encoded, b'd5:applei2e5:zebrai1ee') + encoded = ts.bencode({b"zebra": 1, b"apple": 2}) + self.assertEqual(encoded, b"d5:applei2e5:zebrai1ee") class ComputeInfoHashTests(unittest.TestCase): def test_matches_sha1_of_raw_info_bytes(self): info_bytes = ( - b'd6:lengthi12345e4:name8:test.iso12:piece lengthi16384e' - b'6:pieces20:' + b'A' * 20 + b'e' + b"d6:lengthi12345e4:name8:test.iso12:piece lengthi16384e" + b"6:pieces20:" + b"A" * 20 + b"e" + ) + torrent_bytes = ( + b"d" + + bstr("announce") + + bstr("http://tracker.example.com:6969/announce") + + bstr("info") + + info_bytes + + b"e" ) - torrent_bytes = b'd' + bstr('announce') + bstr('http://tracker.example.com:6969/announce') \ - + bstr('info') + info_bytes + b'e' expected = hashlib.sha1(info_bytes).digest() self.assertEqual(ts.compute_info_hash(torrent_bytes), expected) def test_raises_when_info_missing(self): - torrent_bytes = b'd' + bstr('announce') + bstr('http://tracker.example.com/announce') + b'e' + torrent_bytes = ( + b"d" + bstr("announce") + bstr("http://tracker.example.com/announce") + b"e" + ) with self.assertRaises(ts.BencodeError): ts.compute_info_hash(torrent_bytes) @@ -105,26 +153,43 @@ def test_raises_when_info_missing(self): class GetAnnounceUrlsTests(unittest.TestCase): def test_combines_announce_list_and_primary_announce(self): torrent_bytes = ( - b'd' - + bstr('announce') + bstr('http://a.example.com/announce') - + bstr('announce-list') - + b'l' - + b'l' + bstr('http://a.example.com/announce') + b'e' - + b'l' + bstr('http://b.example.com/announce') + b'e' - + b'e' - + bstr('info') + b'de' - + b'e' + b"d" + + bstr("announce") + + bstr("http://a.example.com/announce") + + bstr("announce-list") + + b"l" + + b"l" + + bstr("http://a.example.com/announce") + + b"e" + + b"l" + + bstr("http://b.example.com/announce") + + b"e" + + b"e" + + bstr("info") + + b"de" + + b"e" ) urls = ts.get_announce_urls(torrent_bytes) - self.assertEqual(urls, [ - 'http://a.example.com/announce', - 'http://b.example.com/announce', - ]) + self.assertEqual( + urls, + [ + "http://a.example.com/announce", + "http://b.example.com/announce", + ], + ) def test_falls_back_to_bare_announce(self): - torrent_bytes = b'd' + bstr('announce') + bstr('http://only.example.com/announce') \ - + bstr('info') + b'de' + b'e' - self.assertEqual(ts.get_announce_urls(torrent_bytes), ['http://only.example.com/announce']) + torrent_bytes = ( + b"d" + + bstr("announce") + + bstr("http://only.example.com/announce") + + bstr("info") + + b"de" + + b"e" + ) + self.assertEqual( + ts.get_announce_urls(torrent_bytes), ["http://only.example.com/announce"] + ) class DeriveScrapeUrlTests(unittest.TestCase): @@ -133,7 +198,10 @@ def test_examples_from_the_scrape_convention(self): ("http://example.com/announce", "http://example.com/scrape"), ("http://example.com/x/announce", "http://example.com/x/scrape"), ("http://example.com/announce.php", "http://example.com/scrape.php"), - ("http://example.com/a/announce?x2%0644", "http://example.com/a/scrape?x2%0644"), + ( + "http://example.com/a/announce?x2%0644", + "http://example.com/a/scrape?x2%0644", + ), ("http://example.com/a", None), ] for announce_url, expected in cases: @@ -156,7 +224,8 @@ def test_appends_to_existing_query(self): self.assertIn("?x=1&info_hash=", url) def test_percent_encoding_roundtrips(self): - from urllib.parse import urlsplit, parse_qsl, unquote_to_bytes + from urllib.parse import unquote_to_bytes, urlsplit + info_hash = bytes(range(20)) url = ts.build_scrape_request_url("http://example.com/scrape", info_hash) query = urlsplit(url).query @@ -166,19 +235,25 @@ def test_percent_encoding_roundtrips(self): class ParseScrapeResponseTests(unittest.TestCase): def test_extracts_seeders_leechers_completed(self): - info_hash = b'A' * 20 - response = b'd5:filesd' + bstr(info_hash) + b'd8:completei33e10:incompletei73e10:downloadedi9e' + b'ee' + b'e' + info_hash = b"A" * 20 + response = ( + b"d5:filesd" + + bstr(info_hash) + + b"d8:completei33e10:incompletei73e10:downloadedi9e" + + b"ee" + + b"e" + ) stats = ts.parse_scrape_response(response, info_hash) - self.assertEqual(stats, {'seeders': 33, 'leechers': 73, 'completed': 9}) + self.assertEqual(stats, {"seeders": 33, "leechers": 73, "completed": 9}) def test_returns_none_when_hash_absent(self): - response = b'd5:filesdee' + b'e' - self.assertIsNone(ts.parse_scrape_response(response, b'A' * 20)) + response = b"d5:filesdee" + b"e" + self.assertIsNone(ts.parse_scrape_response(response, b"A" * 20)) def test_raises_on_failure_reason(self): - response = b'd14:failure reason7:blockede' + response = b"d14:failure reason7:blockede" with self.assertRaises(ts.BencodeError): - ts.parse_scrape_response(response, b'A' * 20) + ts.parse_scrape_response(response, b"A" * 20) class ParseUdpTrackerUrlTests(unittest.TestCase): @@ -195,10 +270,14 @@ def test_udp_url_without_path_still_parses(self): ) def test_missing_port_is_none(self): - self.assertIsNone(ts.parse_udp_tracker_url("udp://tracker.example.com/announce")) + self.assertIsNone( + ts.parse_udp_tracker_url("udp://tracker.example.com/announce") + ) def test_non_udp_scheme_is_none(self): - self.assertIsNone(ts.parse_udp_tracker_url("http://tracker.example.com:1337/announce")) + self.assertIsNone( + ts.parse_udp_tracker_url("http://tracker.example.com:1337/announce") + ) def _connect_response(transaction_id, connection_id, action=0): @@ -225,7 +304,9 @@ def test_request_matches_bep15_layout(self): self.assertEqual(transaction_id, 0xAABBCCDD) def test_parses_valid_response(self): - response = _connect_response(transaction_id=42, connection_id=0x1122334455667788) + response = _connect_response( + transaction_id=42, connection_id=0x1122334455667788 + ) connection_id = ts.parse_udp_connect_response(response, transaction_id=42) self.assertEqual(connection_id, 0x1122334455667788) @@ -241,7 +322,7 @@ def test_wrong_action_raises(self): def test_short_response_raises(self): with self.assertRaises(ts.UdpTrackerError): - ts.parse_udp_connect_response(b'\x00' * 8, transaction_id=42) + ts.parse_udp_connect_response(b"\x00" * 8, transaction_id=42) def test_tracker_error_action_raises_with_message(self): response = _error_response(transaction_id=42, message="bad request") @@ -254,7 +335,9 @@ class BuildParseUdpScrapeRequestTests(unittest.TestCase): def test_request_matches_bep15_layout(self): info_hash = bytes(range(20)) request = ts.build_udp_scrape_request( - connection_id=0x1122334455667788, transaction_id=7, info_hash=info_hash, + connection_id=0x1122334455667788, + transaction_id=7, + info_hash=info_hash, ) connection_id, action, transaction_id = struct.unpack(">QII", request[:16]) self.assertEqual(connection_id, 0x1122334455667788) @@ -264,26 +347,34 @@ def test_request_matches_bep15_layout(self): def test_wrong_length_info_hash_raises(self): with self.assertRaises(ts.UdpTrackerError): - ts.build_udp_scrape_request(connection_id=1, transaction_id=1, info_hash=b'short') + ts.build_udp_scrape_request( + connection_id=1, transaction_id=1, info_hash=b"short" + ) def test_parses_valid_response(self): - response = _scrape_response(transaction_id=7, seeders=33, completed=9, leechers=73) + response = _scrape_response( + transaction_id=7, seeders=33, completed=9, leechers=73 + ) stats = ts.parse_udp_scrape_response(response, transaction_id=7) - self.assertEqual(stats, {'seeders': 33, 'leechers': 73, 'completed': 9}) + self.assertEqual(stats, {"seeders": 33, "leechers": 73, "completed": 9}) def test_transaction_id_mismatch_raises(self): - response = _scrape_response(transaction_id=7, seeders=1, completed=1, leechers=1) + response = _scrape_response( + transaction_id=7, seeders=1, completed=1, leechers=1 + ) with self.assertRaises(ts.UdpTrackerError): ts.parse_udp_scrape_response(response, transaction_id=8) def test_wrong_action_raises(self): - response = _scrape_response(transaction_id=7, seeders=1, completed=1, leechers=1, action=0) + response = _scrape_response( + transaction_id=7, seeders=1, completed=1, leechers=1, action=0 + ) with self.assertRaises(ts.UdpTrackerError): ts.parse_udp_scrape_response(response, transaction_id=7) def test_short_response_raises(self): with self.assertRaises(ts.UdpTrackerError): - ts.parse_udp_scrape_response(b'\x00' * 4, transaction_id=7) + ts.parse_udp_scrape_response(b"\x00" * 4, transaction_id=7) def test_tracker_error_action_raises_with_message(self): response = _error_response(transaction_id=7, message="scrape not supported") @@ -303,93 +394,119 @@ def _make_fake_socket(self, recvfrom_side_effect): return fake_sock def test_full_round_trip_returns_stats(self): - info_hash = b'A' * 20 - connect_resp = _connect_response(transaction_id=0x00000001, connection_id=0xCAFEBABE) - scrape_resp = _scrape_response(transaction_id=0x00000001, seeders=5, completed=2, leechers=3) - fake_sock = self._make_fake_socket([ - (connect_resp, ('1.2.3.4', 1337)), - (scrape_resp, ('1.2.3.4', 1337)), - ]) - - with unittest.mock.patch.object(ts.socket, 'socket', return_value=fake_sock), \ - unittest.mock.patch.object(ts.os, 'urandom', return_value=b'\x00\x00\x00\x01'): - stats = ts.scrape_udp_tracker('tracker.example.com', 1337, info_hash, timeout=5) - - self.assertEqual(stats, {'seeders': 5, 'leechers': 3, 'completed': 2}) + info_hash = b"A" * 20 + connect_resp = _connect_response( + transaction_id=0x00000001, connection_id=0xCAFEBABE + ) + scrape_resp = _scrape_response( + transaction_id=0x00000001, seeders=5, completed=2, leechers=3 + ) + fake_sock = self._make_fake_socket( + [ + (connect_resp, ("1.2.3.4", 1337)), + (scrape_resp, ("1.2.3.4", 1337)), + ] + ) + + with ( + unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock), + unittest.mock.patch.object( + ts.os, "urandom", return_value=b"\x00\x00\x00\x01" + ), + ): + stats = ts.scrape_udp_tracker( + "tracker.example.com", 1337, info_hash, timeout=5 + ) + + self.assertEqual(stats, {"seeders": 5, "leechers": 3, "completed": 2}) fake_sock.settimeout.assert_called_once_with(5) self.assertEqual(fake_sock.sendto.call_count, 2) # Second (scrape) request must carry the connection_id from the connect response. scrape_request_bytes = fake_sock.sendto.call_args_list[1][0][0] - connection_id_sent, action_sent = struct.unpack(">QI", scrape_request_bytes[:12]) + connection_id_sent, action_sent = struct.unpack( + ">QI", scrape_request_bytes[:12] + ) self.assertEqual(connection_id_sent, 0xCAFEBABE) self.assertEqual(action_sent, 2) fake_sock.close.assert_called_once() def test_connect_transaction_mismatch_raises_and_closes_socket(self): - info_hash = b'A' * 20 + info_hash = b"A" * 20 bad_connect_resp = _connect_response(transaction_id=0x00000002, connection_id=1) - fake_sock = self._make_fake_socket([(bad_connect_resp, ('1.2.3.4', 1337))]) + fake_sock = self._make_fake_socket([(bad_connect_resp, ("1.2.3.4", 1337))]) - with unittest.mock.patch.object(ts.socket, 'socket', return_value=fake_sock), \ - unittest.mock.patch.object(ts.os, 'urandom', return_value=b'\x00\x00\x00\x01'): - with self.assertRaises(ts.UdpTrackerError): - ts.scrape_udp_tracker('tracker.example.com', 1337, info_hash, timeout=5) + with ( + unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock), + unittest.mock.patch.object( + ts.os, "urandom", return_value=b"\x00\x00\x00\x01" + ), + self.assertRaises(ts.UdpTrackerError), + ): + ts.scrape_udp_tracker("tracker.example.com", 1337, info_hash, timeout=5) fake_sock.close.assert_called_once() def test_socket_timeout_propagates(self): - info_hash = b'A' * 20 + info_hash = b"A" * 20 fake_sock = unittest.mock.Mock() - fake_sock.recvfrom.side_effect = socket.timeout("timed out") + fake_sock.recvfrom.side_effect = TimeoutError("timed out") - with unittest.mock.patch.object(ts.socket, 'socket', return_value=fake_sock): + with unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock): with self.assertRaises(socket.timeout): - ts.scrape_udp_tracker('tracker.example.com', 1337, info_hash, timeout=5) + ts.scrape_udp_tracker("tracker.example.com", 1337, info_hash, timeout=5) fake_sock.close.assert_called_once() class IsDisallowedIpTests(unittest.TestCase): def test_public_ipv4_is_allowed(self): - self.assertFalse(ts._is_disallowed_ip('93.184.216.34')) + self.assertFalse(ts._is_disallowed_ip("93.184.216.34")) def test_private_ipv4_ranges_are_disallowed(self): - for ip in ('10.0.0.1', '172.16.0.1', '192.168.1.1'): + for ip in ("10.0.0.1", "172.16.0.1", "192.168.1.1"): with self.subTest(ip=ip): self.assertTrue(ts._is_disallowed_ip(ip)) def test_loopback_is_disallowed(self): - self.assertTrue(ts._is_disallowed_ip('127.0.0.1')) - self.assertTrue(ts._is_disallowed_ip('::1')) + self.assertTrue(ts._is_disallowed_ip("127.0.0.1")) + self.assertTrue(ts._is_disallowed_ip("::1")) def test_link_local_is_disallowed(self): # Cloud metadata endpoint (AWS/GCP/Azure) lives here. - self.assertTrue(ts._is_disallowed_ip('169.254.169.254')) + self.assertTrue(ts._is_disallowed_ip("169.254.169.254")) def test_public_ipv6_is_allowed(self): - self.assertFalse(ts._is_disallowed_ip('2001:4860:4860::8888')) + self.assertFalse(ts._is_disallowed_ip("2001:4860:4860::8888")) class IsSafeScrapeHostTests(unittest.TestCase): def test_public_address_is_safe(self): - with unittest.mock.patch.object(ts, '_resolve_addresses', return_value=['93.184.216.34']): - self.assertTrue(ts._is_safe_scrape_host('tracker.example.com')) + with unittest.mock.patch.object( + ts, "_resolve_addresses", return_value=["93.184.216.34"] + ): + self.assertTrue(ts._is_safe_scrape_host("tracker.example.com")) def test_private_address_is_unsafe(self): - with unittest.mock.patch.object(ts, '_resolve_addresses', return_value=['10.1.2.3']): - self.assertFalse(ts._is_safe_scrape_host('tracker.internal')) + with unittest.mock.patch.object( + ts, "_resolve_addresses", return_value=["10.1.2.3"] + ): + self.assertFalse(ts._is_safe_scrape_host("tracker.internal")) def test_any_disallowed_address_among_several_makes_host_unsafe(self): - with unittest.mock.patch.object(ts, '_resolve_addresses', return_value=['93.184.216.34', '127.0.0.1']): - self.assertFalse(ts._is_safe_scrape_host('multi-homed.example.com')) + with unittest.mock.patch.object( + ts, "_resolve_addresses", return_value=["93.184.216.34", "127.0.0.1"] + ): + self.assertFalse(ts._is_safe_scrape_host("multi-homed.example.com")) def test_resolution_failure_is_unsafe(self): - with unittest.mock.patch.object(ts, '_resolve_addresses', side_effect=OSError('not known')): - self.assertFalse(ts._is_safe_scrape_host('nonexistent.invalid')) + with unittest.mock.patch.object( + ts, "_resolve_addresses", side_effect=OSError("not known") + ): + self.assertFalse(ts._is_safe_scrape_host("nonexistent.invalid")) def test_empty_hostname_is_unsafe(self): self.assertFalse(ts._is_safe_scrape_host(None)) - self.assertFalse(ts._is_safe_scrape_host('')) + self.assertFalse(ts._is_safe_scrape_host("")) class ScrapeTorrentTests(unittest.TestCase): @@ -399,178 +516,240 @@ def setUp(self): # hosts to a public address so existing tracker-selection behavior # doesn't depend on real DNS. Tests of the SSRF guard itself override # this per-case. - patcher = unittest.mock.patch.object(ts, '_resolve_addresses', return_value=['93.184.216.34']) + patcher = unittest.mock.patch.object( + ts, "_resolve_addresses", return_value=["93.184.216.34"] + ) patcher.start() self.addCleanup(patcher.stop) def _write_torrent(self, announce_urls): - tiers = b''.join(b'l' + bstr(u) + b'e' for u in announce_urls) - info_bytes = b'd6:lengthi1e4:name4:teste' + tiers = b"".join(b"l" + bstr(u) + b"e" for u in announce_urls) + info_bytes = b"d6:lengthi1e4:name4:teste" torrent_bytes = ( - b'd' - + bstr('announce') + bstr(announce_urls[0]) - + bstr('announce-list') + b'l' + tiers + b'e' - + bstr('info') + info_bytes - + b'e' + b"d" + + bstr("announce") + + bstr(announce_urls[0]) + + bstr("announce-list") + + b"l" + + tiers + + b"e" + + bstr("info") + + info_bytes + + b"e" ) - fd, path = tempfile.mkstemp(suffix='.torrent') - with os.fdopen(fd, 'wb') as f: + fd, path = tempfile.mkstemp(suffix=".torrent") + self.addCleanup(os.remove, path) + with os.fdopen(fd, "wb") as f: f.write(torrent_bytes) return path def _scrape_response_bytes(self, info_hash, seeders, leechers): - return b'd5:filesd' + bstr(info_hash) + \ - f'd8:completei{seeders}e10:incompletei{leechers}e10:downloadedi0ee'.encode() + b'ee' + b'e' + return ( + b"d5:filesd" + + bstr(info_hash) + + f"d8:completei{seeders}e10:incompletei{leechers}e10:downloadedi0ee".encode() + + b"ee" + + b"e" + ) def test_uses_udp_tracker_when_it_succeeds(self): - path = self._write_torrent([ - 'udp://tracker.example.com:80/announce', - 'http://tracker.example.com:6969/announce', - ]) - with unittest.mock.patch.object(ts, 'scrape_udp_tracker', return_value={ - 'seeders': 5, 'leechers': 7, 'completed': 1, - }) as mock_udp_scrape, unittest.mock.patch.object(ts.requests, 'get') as mock_get: + path = self._write_torrent( + [ + "udp://tracker.example.com:80/announce", + "http://tracker.example.com:6969/announce", + ] + ) + with ( + unittest.mock.patch.object( + ts, + "scrape_udp_tracker", + return_value={ + "seeders": 5, + "leechers": 7, + "completed": 1, + }, + ) as mock_udp_scrape, + unittest.mock.patch.object(ts.requests, "get") as mock_get, + ): result = ts.scrape_torrent(path) mock_udp_scrape.assert_called_once() mock_get.assert_not_called() - self.assertEqual(result['seeders'], 5) - self.assertEqual(result['leechers'], 7) - self.assertEqual(result['tracker'], 'udp://tracker.example.com:80') + self.assertEqual(result["seeders"], 5) + self.assertEqual(result["leechers"], 7) + self.assertEqual(result["tracker"], "udp://tracker.example.com:80") def test_falls_through_from_failing_udp_tracker_to_http_tracker(self): - path = self._write_torrent([ - 'udp://tracker.example.com:80/announce', - 'http://tracker.example.com:6969/announce', - ]) - with open(path, 'rb') as f: + path = self._write_torrent( + [ + "udp://tracker.example.com:80/announce", + "http://tracker.example.com:6969/announce", + ] + ) + with open(path, "rb") as f: info_hash = ts.compute_info_hash(f.read()) fake_response = unittest.mock.Mock() fake_response.content = self._scrape_response_bytes(info_hash, 5, 7) fake_response.raise_for_status = lambda: None - with unittest.mock.patch.object(ts, 'scrape_udp_tracker', side_effect=OSError("unreachable")), \ - unittest.mock.patch.object(ts.requests, 'get', return_value=fake_response) as mock_get: + with ( + unittest.mock.patch.object( + ts, "scrape_udp_tracker", side_effect=OSError("unreachable") + ), + unittest.mock.patch.object( + ts.requests, "get", return_value=fake_response + ) as mock_get, + ): result = ts.scrape_torrent(path) mock_get.assert_called_once() - self.assertIn('tracker.example.com:6969/scrape', mock_get.call_args[0][0]) - self.assertEqual(result['seeders'], 5) - self.assertEqual(result['leechers'], 7) - self.assertEqual(result['tracker'], 'http://tracker.example.com:6969/scrape') + self.assertIn("tracker.example.com:6969/scrape", mock_get.call_args[0][0]) + self.assertEqual(result["seeders"], 5) + self.assertEqual(result["leechers"], 7) + self.assertEqual(result["tracker"], "http://tracker.example.com:6969/scrape") def test_skips_udp_tracker_with_no_port(self): """BEP 15 requires an explicit port; an announce URL without one can't be dialed, so it should be skipped like any other unusable tracker rather than raising.""" - path = self._write_torrent([ - 'udp://tracker.example.com/announce', - 'http://tracker.example.com:6969/announce', - ]) - with open(path, 'rb') as f: + path = self._write_torrent( + [ + "udp://tracker.example.com/announce", + "http://tracker.example.com:6969/announce", + ] + ) + with open(path, "rb") as f: info_hash = ts.compute_info_hash(f.read()) fake_response = unittest.mock.Mock() fake_response.content = self._scrape_response_bytes(info_hash, 5, 7) fake_response.raise_for_status = lambda: None - with unittest.mock.patch.object(ts, 'scrape_udp_tracker') as mock_udp_scrape, \ - unittest.mock.patch.object(ts.requests, 'get', return_value=fake_response): + with ( + unittest.mock.patch.object(ts, "scrape_udp_tracker") as mock_udp_scrape, + unittest.mock.patch.object(ts.requests, "get", return_value=fake_response), + ): result = ts.scrape_torrent(path) mock_udp_scrape.assert_not_called() - self.assertEqual(result['tracker'], 'http://tracker.example.com:6969/scrape') + self.assertEqual(result["tracker"], "http://tracker.example.com:6969/scrape") def test_falls_through_to_next_tracker_on_failure(self): - path = self._write_torrent([ - 'http://dead.example.com/announce', - 'http://alive.example.com/announce', - ]) - with open(path, 'rb') as f: + path = self._write_torrent( + [ + "http://dead.example.com/announce", + "http://alive.example.com/announce", + ] + ) + with open(path, "rb") as f: info_hash = ts.compute_info_hash(f.read()) fake_response = unittest.mock.Mock() fake_response.content = self._scrape_response_bytes(info_hash, 1, 2) fake_response.raise_for_status = lambda: None def fake_get(url, timeout=None): - if 'dead' in url: + if "dead" in url: raise ConnectionError("nope") return fake_response - with unittest.mock.patch.object(ts.requests, 'get', side_effect=fake_get): + with unittest.mock.patch.object(ts.requests, "get", side_effect=fake_get): result = ts.scrape_torrent(path) - self.assertEqual(result['seeders'], 1) - self.assertEqual(result['tracker'], 'http://alive.example.com/scrape') + self.assertEqual(result["seeders"], 1) + self.assertEqual(result["tracker"], "http://alive.example.com/scrape") def test_returns_none_when_no_tracker_usable(self): - path = self._write_torrent(['udp://only.example.com:1337/announce']) - with unittest.mock.patch.object(ts, 'scrape_udp_tracker', side_effect=OSError("unreachable")), \ - unittest.mock.patch.object(ts.requests, 'get') as mock_get: + path = self._write_torrent(["udp://only.example.com:1337/announce"]) + with ( + unittest.mock.patch.object( + ts, "scrape_udp_tracker", side_effect=OSError("unreachable") + ), + unittest.mock.patch.object(ts.requests, "get") as mock_get, + ): result = ts.scrape_torrent(path) mock_get.assert_not_called() self.assertIsNone(result) def test_skips_udp_tracker_resolving_to_a_private_address(self): - path = self._write_torrent([ - 'udp://internal.example.com:1337/announce', - 'http://real-tracker.example.com/announce', - ]) - with open(path, 'rb') as f: + path = self._write_torrent( + [ + "udp://internal.example.com:1337/announce", + "http://real-tracker.example.com/announce", + ] + ) + with open(path, "rb") as f: info_hash = ts.compute_info_hash(f.read()) fake_response = unittest.mock.Mock() fake_response.content = self._scrape_response_bytes(info_hash, 3, 4) fake_response.raise_for_status = lambda: None def fake_resolve(hostname): - if hostname == 'internal.example.com': - return ['127.0.0.1'] - return ['93.184.216.34'] - - with unittest.mock.patch.object(ts, '_resolve_addresses', side_effect=fake_resolve), \ - unittest.mock.patch.object(ts, 'scrape_udp_tracker') as mock_udp_scrape, \ - unittest.mock.patch.object(ts.requests, 'get', return_value=fake_response) as mock_get: + if hostname == "internal.example.com": + return ["127.0.0.1"] + return ["93.184.216.34"] + + with ( + unittest.mock.patch.object( + ts, "_resolve_addresses", side_effect=fake_resolve + ), + unittest.mock.patch.object(ts, "scrape_udp_tracker") as mock_udp_scrape, + unittest.mock.patch.object( + ts.requests, "get", return_value=fake_response + ) as mock_get, + ): result = ts.scrape_torrent(path) mock_udp_scrape.assert_not_called() mock_get.assert_called_once() - self.assertEqual(result['tracker'], 'http://real-tracker.example.com/scrape') + self.assertEqual(result["tracker"], "http://real-tracker.example.com/scrape") def test_skips_tracker_resolving_to_a_private_address(self): """A .torrent whose announce URL resolves to an internal address (e.g. a compromised upstream mirror pointing at cloud metadata or this container's own services) must never be scraped - no request should go out to it at all.""" - path = self._write_torrent([ - 'http://internal.example.com/announce', - 'http://real-tracker.example.com/announce', - ]) - with open(path, 'rb') as f: + path = self._write_torrent( + [ + "http://internal.example.com/announce", + "http://real-tracker.example.com/announce", + ] + ) + with open(path, "rb") as f: info_hash = ts.compute_info_hash(f.read()) fake_response = unittest.mock.Mock() fake_response.content = self._scrape_response_bytes(info_hash, 3, 4) fake_response.raise_for_status = lambda: None def fake_resolve(hostname): - if hostname == 'internal.example.com': - return ['127.0.0.1'] - return ['93.184.216.34'] - - with unittest.mock.patch.object(ts, '_resolve_addresses', side_effect=fake_resolve), \ - unittest.mock.patch.object(ts.requests, 'get', return_value=fake_response) as mock_get: + if hostname == "internal.example.com": + return ["127.0.0.1"] + return ["93.184.216.34"] + + with ( + unittest.mock.patch.object( + ts, "_resolve_addresses", side_effect=fake_resolve + ), + unittest.mock.patch.object( + ts.requests, "get", return_value=fake_response + ) as mock_get, + ): result = ts.scrape_torrent(path) mock_get.assert_called_once() - self.assertIn('real-tracker.example.com', mock_get.call_args[0][0]) - self.assertEqual(result['tracker'], 'http://real-tracker.example.com/scrape') + self.assertIn("real-tracker.example.com", mock_get.call_args[0][0]) + self.assertEqual(result["tracker"], "http://real-tracker.example.com/scrape") def test_returns_none_when_every_tracker_resolves_privately(self): - path = self._write_torrent(['http://internal.example.com/announce']) - with unittest.mock.patch.object(ts, '_resolve_addresses', return_value=['169.254.169.254']), \ - unittest.mock.patch.object(ts.requests, 'get') as mock_get: + path = self._write_torrent(["http://internal.example.com/announce"]) + with ( + unittest.mock.patch.object( + ts, "_resolve_addresses", return_value=["169.254.169.254"] + ), + unittest.mock.patch.object(ts.requests, "get") as mock_get, + ): result = ts.scrape_torrent(path) mock_get.assert_not_called() self.assertIsNone(result) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/torrent_scrape.py b/torrent_scrape.py index f1e460d..b955579 100644 --- a/torrent_scrape.py +++ b/torrent_scrape.py @@ -11,12 +11,14 @@ (BEP 15) are supported. Any other scheme is skipped; callers should just move on to the torrent's next tracker. """ + import hashlib import ipaddress import os +import re import socket import struct -from urllib.parse import urlsplit, urlunsplit, quote_from_bytes +from urllib.parse import quote_from_bytes, urlsplit, urlunsplit import requests @@ -30,7 +32,10 @@ class UdpTrackerError(ValueError): response, transaction ID mismatch, or an explicit tracker error action) - callers should catch it the same as any other scrape failure and move on to the next tracker.""" - pass + + +# Stricter than int(), which also accepts whitespace, '+' and underscores. +_BENCODE_INT = re.compile(rb"-?\d+") def bdecode(data, index=0): @@ -38,33 +43,49 @@ def bdecode(data, index=0): next_index). Byte strings decode to bytes, dict keys must be byte strings (per spec) and dicts decode with keys as bytes.""" if index >= len(data): - raise BencodeError("Unexpected end of data") + raise BencodeError(f"Unexpected end of data at index {index}") - prefix = data[index:index + 1] + prefix = data[index : index + 1] - if prefix == b'i': - end = data.index(b'e', index) - return int(data[index + 1:end]), end + 1 + if prefix == b"i": + end = data.find(b"e", index) + if end == -1: + raise BencodeError(f"Unterminated integer at index {index}") + digits = data[index + 1 : end] + if not _BENCODE_INT.fullmatch(digits): + raise BencodeError(f"Invalid integer {digits!r} at index {index}") + return int(digits), end + 1 if prefix.isdigit(): - colon = data.index(b':', index) - length = int(data[index:colon]) + colon = data.find(b":", index) + if colon == -1: + raise BencodeError(f"Byte string length without ':' at index {index}") + digits = data[index:colon] + if not digits.isdigit(): + raise BencodeError( + f"Invalid byte string length {digits!r} at index {index}" + ) start = colon + 1 - end = start + length + end = start + int(digits) + if end > len(data): + raise BencodeError( + f"Byte string at index {index} declares {int(digits)} bytes, " + f"only {len(data) - start} remain" + ) return data[start:end], end - if prefix == b'l': + if prefix == b"l": index += 1 items = [] - while data[index:index + 1] != b'e': + while data[index : index + 1] != b"e": item, index = bdecode(data, index) items.append(item) return items, index + 1 - if prefix == b'd': + if prefix == b"d": index += 1 result = {} - while data[index:index + 1] != b'e': + while data[index : index + 1] != b"e": key, index = bdecode(data, index) if not isinstance(key, bytes): raise BencodeError("Dict keys must be byte strings") @@ -82,24 +103,26 @@ def bencode(value): if isinstance(value, bool): raise BencodeError("bool is not a bencode type") if isinstance(value, int): - return b'i' + str(value).encode() + b'e' + return b"i" + str(value).encode() + b"e" if isinstance(value, (bytes, bytearray)): - return str(len(value)).encode() + b':' + bytes(value) + return str(len(value)).encode() + b":" + bytes(value) if isinstance(value, str): - encoded = value.encode('utf-8') - return str(len(encoded)).encode() + b':' + encoded + encoded = value.encode("utf-8") + return str(len(encoded)).encode() + b":" + encoded if isinstance(value, list): - return b'l' + b''.join(bencode(v) for v in value) + b'e' + return b"l" + b"".join(bencode(v) for v in value) + b"e" if isinstance(value, dict): + def key_bytes(k): - return k if isinstance(k, bytes) else k.encode('utf-8') + return k if isinstance(k, bytes) else k.encode("utf-8") + items = sorted(value.items(), key=lambda kv: key_bytes(kv[0])) - parts = [b'd'] + parts = [b"d"] for k, v in items: parts.append(bencode(key_bytes(k))) parts.append(bencode(v)) - parts.append(b'e') - return b''.join(parts) + parts.append(b"e") + return b"".join(parts) raise BencodeError(f"Cannot bencode type {type(value)}") @@ -107,9 +130,9 @@ def compute_info_hash(torrent_bytes): """The 20-byte SHA-1 hash that identifies a torrent, computed from the raw bytes of its 'info' dict.""" metainfo, _ = bdecode(torrent_bytes) - if not isinstance(metainfo, dict) or b'info' not in metainfo: + if not isinstance(metainfo, dict) or b"info" not in metainfo: raise BencodeError("Not a valid .torrent file: missing 'info' dictionary") - return hashlib.sha1(bencode(metainfo[b'info'])).digest() + return hashlib.sha1(bencode(metainfo[b"info"])).digest() def get_announce_urls(torrent_bytes): @@ -117,15 +140,15 @@ def get_announce_urls(torrent_bytes): order) then the primary 'announce' if not already present, de-duped.""" metainfo, _ = bdecode(torrent_bytes) urls = [] - for tier in metainfo.get(b'announce-list', []): + for tier in metainfo.get(b"announce-list", []): for url in tier: - decoded = url.decode('utf-8') + decoded = url.decode("utf-8") if decoded not in urls: urls.append(decoded) - primary = metainfo.get(b'announce') + primary = metainfo.get(b"announce") if primary is not None: - decoded = primary.decode('utf-8') + decoded = primary.decode("utf-8") if decoded not in urls: urls.insert(0, decoded) @@ -138,20 +161,24 @@ def derive_scrape_url(announce_url): None if the tracker doesn't support scrape (no 'announce' in its last path segment) or isn't HTTP(S).""" parts = urlsplit(announce_url) - if parts.scheme not in ('http', 'https'): + if parts.scheme not in ("http", "https"): return None - last_slash = parts.path.rfind('/') - segment = parts.path[last_slash + 1:] - if 'announce' not in segment: + last_slash = parts.path.rfind("/") + segment = parts.path[last_slash + 1 :] + if "announce" not in segment: return None - new_segment = segment.replace('announce', 'scrape', 1) - new_path = parts.path[:last_slash + 1] + new_segment - return urlunsplit((parts.scheme, parts.netloc, new_path, parts.query, parts.fragment)) + new_segment = segment.replace("announce", "scrape", 1) + new_path = parts.path[: last_slash + 1] + new_segment + return urlunsplit( + (parts.scheme, parts.netloc, new_path, parts.query, parts.fragment) + ) -UDP_TRACKER_PROTOCOL_ID = 0x41727101980 # BEP 15 magic connection_id for the initial connect request +UDP_TRACKER_PROTOCOL_ID = ( + 0x41727101980 # BEP 15 magic connection_id for the initial connect request +) UDP_ACTION_CONNECT = 0 UDP_ACTION_SCRAPE = 2 UDP_ACTION_ERROR = 3 @@ -162,7 +189,7 @@ def parse_udp_tracker_url(announce_url): isn't udp or no port is present - BEP 15 has no default port, so a tracker URL without one can't be dialed.""" parts = urlsplit(announce_url) - if parts.scheme != 'udp': + if parts.scheme != "udp": return None if not parts.hostname or not parts.port: return None @@ -170,7 +197,9 @@ def parse_udp_tracker_url(announce_url): def build_udp_connect_request(transaction_id): - return struct.pack(">QII", UDP_TRACKER_PROTOCOL_ID, UDP_ACTION_CONNECT, transaction_id) + return struct.pack( + ">QII", UDP_TRACKER_PROTOCOL_ID, UDP_ACTION_CONNECT, transaction_id + ) def _check_for_udp_error_action(data, transaction_id): @@ -180,7 +209,9 @@ def _check_for_udp_error_action(data, transaction_id): if len(data) >= 8: action, resp_transaction_id = struct.unpack(">II", data[:8]) if action == UDP_ACTION_ERROR and resp_transaction_id == transaction_id: - raise UdpTrackerError(f"Tracker error: {data[8:].decode('utf-8', 'replace')}") + raise UdpTrackerError( + f"Tracker error: {data[8:].decode('utf-8', 'replace')}" + ) def parse_udp_connect_response(data, transaction_id): @@ -198,7 +229,10 @@ def parse_udp_connect_response(data, transaction_id): def build_udp_scrape_request(connection_id, transaction_id, info_hash): if len(info_hash) != 20: raise UdpTrackerError(f"info_hash must be 20 bytes, got {len(info_hash)}") - return struct.pack(">QII", connection_id, UDP_ACTION_SCRAPE, transaction_id) + info_hash + return ( + struct.pack(">QII", connection_id, UDP_ACTION_SCRAPE, transaction_id) + + info_hash + ) def parse_udp_scrape_response(data, transaction_id): @@ -213,11 +247,11 @@ def parse_udp_scrape_response(data, transaction_id): if resp_transaction_id != transaction_id: raise UdpTrackerError("Transaction ID mismatch in scrape response") seeders, completed, leechers = struct.unpack(">III", data[8:20]) - return {'seeders': seeders, 'leechers': leechers, 'completed': completed} + return {"seeders": seeders, "leechers": leechers, "completed": completed} def _random_transaction_id(): - return int.from_bytes(os.urandom(4), 'big') + return int.from_bytes(os.urandom(4), "big") def scrape_udp_tracker(host, port, info_hash, timeout=15): @@ -237,7 +271,10 @@ def scrape_udp_tracker(host, port, info_hash, timeout=15): connection_id = parse_udp_connect_response(data, connect_transaction_id) scrape_transaction_id = _random_transaction_id() - sock.sendto(build_udp_scrape_request(connection_id, scrape_transaction_id, info_hash), addr) + sock.sendto( + build_udp_scrape_request(connection_id, scrape_transaction_id, info_hash), + addr, + ) data, _ = sock.recvfrom(2048) return parse_udp_scrape_response(data, scrape_transaction_id) finally: @@ -251,8 +288,12 @@ def _is_disallowed_ip(ip_str): container's own services (127.0.0.1) alongside RFC1918 space.""" ip = ipaddress.ip_address(ip_str) return ( - ip.is_private or ip.is_loopback or ip.is_link_local - or ip.is_multicast or ip.is_reserved or ip.is_unspecified + ip.is_private + or ip.is_loopback + or ip.is_link_local + or ip.is_multicast + or ip.is_reserved + or ip.is_unspecified ) @@ -280,8 +321,8 @@ def _is_safe_scrape_host(hostname): def build_scrape_request_url(scrape_url, info_hash): - encoded_hash = quote_from_bytes(info_hash, safe='') - separator = '&' if '?' in scrape_url else '?' + encoded_hash = quote_from_bytes(info_hash, safe="") + separator = "&" if "?" in scrape_url else "?" return f"{scrape_url}{separator}info_hash={encoded_hash}" @@ -290,17 +331,17 @@ def parse_scrape_response(data, info_hash): 'completed'} for info_hash, or None if the tracker's response doesn't include stats for it.""" parsed, _ = bdecode(data) - if b'failure reason' in parsed: - raise BencodeError(parsed[b'failure reason'].decode('utf-8', 'replace')) + if b"failure reason" in parsed: + raise BencodeError(parsed[b"failure reason"].decode("utf-8", "replace")) - stats = parsed.get(b'files', {}).get(info_hash) + stats = parsed.get(b"files", {}).get(info_hash) if stats is None: return None return { - 'seeders': stats.get(b'complete', 0), - 'leechers': stats.get(b'incomplete', 0), - 'completed': stats.get(b'downloaded', 0), + "seeders": stats.get(b"complete", 0), + "leechers": stats.get(b"incomplete", 0), + "completed": stats.get(b"downloaded", 0), } @@ -310,7 +351,7 @@ def scrape_torrent(torrent_path, timeout=15): until one returns usable stats. Returns {'tracker', 'seeders', 'leechers', 'completed'}, or None if no tracker could be scraped (all UDP-only, all unreachable, or none had stats for this hash).""" - with open(torrent_path, 'rb') as f: + with open(torrent_path, "rb") as f: torrent_bytes = f.read() info_hash = compute_info_hash(torrent_bytes) @@ -325,7 +366,7 @@ def scrape_torrent(torrent_path, timeout=15): stats = scrape_udp_tracker(host, port, info_hash, timeout=timeout) except Exception: continue - return {'tracker': f'udp://{host}:{port}', **stats} + return {"tracker": f"udp://{host}:{port}", **stats} scrape_url = derive_scrape_url(announce_url) if not scrape_url: @@ -342,6 +383,6 @@ def scrape_torrent(torrent_path, timeout=15): continue if stats is not None: - return {'tracker': scrape_url, **stats} + return {"tracker": scrape_url, **stats} return None