diff --git a/docs/remote-scan.md b/docs/remote-scan.md index 5bf3f7d..b2ec4b6 100644 --- a/docs/remote-scan.md +++ b/docs/remote-scan.md @@ -1,7 +1,8 @@ # Remote scan: `cdg scan` on a GitHub repository -Status: design, not implemented. This document is the plan for the pull requests that follow it. -Read against `c1c4d11` (0.12.2). +Status: implemented in the Python build (`src/codigest/core/remote.py`); the Rust build and MCP +are not. Written against `c1c4d11` (0.12.2) as the plan for the pull requests that follow it, and +corrected where the implementation turned out different. ## Goal @@ -114,7 +115,8 @@ The body is what a local scan of a checkout of the same commit gives. The differ Markdown gets a `Source:` line under the heading, JSON a `source` object. A local scan's output stays byte for byte as it is (existing tests and parity tests keep passing), and `prompts.toml` templates receive the three values as extra variables they may ignore. -- The summary on stderr names the repository and the commit: +- The summary on stderr (printed, as for local scans, only when the result is not printed) + names the repository and the commit: `Snapshot NaNIsaac/QoraGeom@main (1a2b3c4) 12 files, ~3,000 tokens`. - Skipped files are listed on stderr like withheld ones, by reason: `Withheld (may contain secrets)`, `Skipped (over 1 MB)`, `Skipped (symlink)`, `Skipped (submodule)`, @@ -156,14 +158,19 @@ parse target ──> resolve ref to commit ──> list tree ──> select ─ 4. **Size check.** The estimate is the sum of the selected sizes over four, as the local scan computes it from `stat`, with the same thresholds (30,000 tokens or 100 files), the same question, the same stop when stdin is not a terminal, and the same `-y`. It runs before any - file is downloaded. + file is downloaded. One difference: a file named with a line range counts with its whole + size, because its lines are not known before it is downloaded (a local scan counts only the + range). 5. **Download** only the selected blobs (see [Requests](#requests)). 6. **Materialize** them into a new temporary folder (`tempfile` / `std::env::temp_dir` with a random name, created with owner-only permissions on Unix) at their repository paths. 7. **Scan** that folder with the existing code: `ProjectContext` gets the folder as a fixed root (a new optional argument; today the root is found by walking up from the current directory, which must not happen here) and `OWNER/REPO` as the project name; `--path` - values become targets inside it. The size check is not asked a second time. + values become targets inside it. The size check is not asked a second time. A named path + with nothing selected under it (ignored, or a withheld secret file) was not downloaded, and + shares nothing, as a local scan of it does; it is not replaced by the whole folder. A line + range on a `--path` that is not a file of the repository is an error. 8. **Clean up.** The folder is removed in a `finally` / `Drop`, also on error and Ctrl+C. Why a temporary folder rather than teaching every stage to read from a listing: the scanner, @@ -235,7 +242,7 @@ to `api.github.com`, and blobs are addressed by their content hash from the list Every request is `GET`. The HTTP layer of both builds has one function, `get(url, headers)`, and no way to send another method or a body. Requests carry `User-Agent: codigest/`, -`X-GitHub-Api-Version: 2022-11-28` on API calls, a 10 s connect and 30 s read timeout, and go +`X-GitHub-Api-Version: 2022-11-28` on API calls, a 30 s socket timeout, and go through the proxy named in `HTTPS_PROXY` when set. Redirects are followed for `GET` only, at most five, and the `Authorization` header is dropped @@ -333,9 +340,10 @@ All tests run without network access unless marked otherwise. ### Fake GitHub -`tests/fixtures/github/` holds small repositories as plain folders. A fake server (Python: -`http.server` on `127.0.0.1:0` in a thread; Rust: a `std::net::TcpListener` thread in -`rust/tests/common`) builds the three API answers and the raw files from such a folder, and +The tests write a small repository into a temporary folder (from a table in the test, so that no +token-shaped string is committed). A fake server (Python: `tests/fake_github.py`, `http.server` +on two ports of `127.0.0.1` for the API and raw hosts; Rust: a `std::net::TcpListener` thread in +`rust/tests/common`) builds the API answers and the raw files from such a folder, and records method, host, path and headers of every request. Both builds take the base URLs from `CODIGEST_GITHUB_API` and `CODIGEST_GITHUB_RAW`; these are for tests and are documented as such. Scenarios are switched per test: rate limited, secondary limit with `Retry-After`, a `500` diff --git a/src/codigest/commands/scan.py b/src/codigest/commands/scan.py index 812aecd..ad58037 100644 --- a/src/codigest/commands/scan.py +++ b/src/codigest/commands/scan.py @@ -17,7 +17,7 @@ lazy from rich.filesize import decimal lazy import json -lazy from ..core import structure, tags, prompts, processor, tokenizer, common, cache +lazy from ..core import structure, tags, prompts, processor, tokenizer, common, cache, remote lazy from ..core.output import OutputHandler, spinner app = typer.Typer() @@ -37,6 +37,8 @@ def _parse_targets(raw: list[str]) -> tuple[list[Path], dict[Path, list[tuple[in span = (int(m["lo"]), int(m["hi"]) if m["hi"] else None) if not path.exists(): hint = " (a line range is FILE:START-END or FILE:LINE)" if ":" in item[2:] else "" + if re.fullmatch(r"[A-Za-z0-9-]+/[A-Za-z0-9._-]+", item): + hint = f"; add --github to scan the GitHub repository {item}" raise typer.BadParameter(f"{item} does not exist{hint}", param_hint="targets") path = path.resolve() if span: @@ -65,26 +67,63 @@ def handle( yes: bool = typer.Option(False, "-y", "--yes", help="Skip confirmation prompt"), resolve: ResolveOption = False, format: FormatOption = Format.XML, + github: bool = typer.Option( + False, + "--github", + help="Read the target OWNER/REPO from GitHub (implied by a github.com URL)", + ), + ref: str | None = typer.Option( + None, + "--ref", + help="GitHub: branch, tag or commit [default: the repository's default branch]", + show_default=False, + ), + paths: list[str] = typer.Option( + None, + "--path", + help="GitHub: only this path of the repository; repeat for more, FILE:40-60 for lines", + show_default=False, + ), + auth: bool = typer.Option( + False, + "--auth", + help="GitHub: send your token (GH_TOKEN, GITHUB_TOKEN or gh) to read private repositories", + ), + max_file_size: str = typer.Option( + "1MB", "--max-file-size", help="GitHub: list larger files as skipped, without downloading" + ), ) -> None: """ Bundle the whole project, or only the given paths, as one block. + A GitHub repository (OWNER/REPO --github, or its URL) is read the same way, without cloning. """ handler = OutputHandler(stdout=stdout, copy=copy, output=output) log_console = handler.log_console - targets, ranges = _parse_targets(raw_targets or []) + raw_targets = raw_targets or [] + + if target := _remote_target(raw_targets, github, ref, paths, resolve): + try: + limit = remote.parse_size(max_file_size) + except ValueError as e: + raise typer.BadParameter(str(e), param_hint="'--max-file-size'") from None + try: + _scan_remote(handler, target, auth, limit, yes, message, line_numbers, format) + except remote.RemoteError as e: + log_console.print(f"[red]Error:[/red] {e}", highlight=False) + raise typer.Exit(1) from None + return + + targets, ranges = _parse_targets(raw_targets) # [1] Context Setup (Centralized) ctx = common.get_context(targets) root_path = ctx.root_path - prompt_engine = prompts.get_engine(root_path) - # [2] File Discovery via Context with spinner(log_console, "[bold blue]Scanning...[/bold blue]"): files = ctx.get_target_files(targets=targets, ignore_config=all, resolve_deps=resolve) # [3] Pre-flight Check - total_files = len(files) # A range of a big file is small: count what will be shared, not the whole file. total_size = sum( sum(len(processor.read_line_range(f, lo, hi)[0]) for lo, hi in ranges[f]) @@ -92,33 +131,146 @@ def handle( else f.stat().st_size for f in files ) - est_tokens = int(total_size / 4) if ctx.withheld: - names = ", ".join(p.relative_to(root_path).as_posix() for p in ctx.withheld) - log_console.print( - f"[yellow]Withheld (may contain secrets):[/yellow] {names}", highlight=False - ) + _report(log_console, "Withheld (may contain secrets)", ctx.withheld, root_path) + if not yes: + _confirm_size(log_console, len(files), total_size) + + _render(handler, root_path, files, ranges, message, line_numbers, format) + +def _report(log_console, label: str, paths, root: Path | None = None) -> None: + names = ", ".join(p.relative_to(root).as_posix() if root else p for p in paths) + log_console.print(f"[yellow]{label}:[/yellow] {names}", highlight=False) + + +def _confirm_size(log_console, total_files: int, total_size: int) -> None: + """Asks before sharing a large context; stops when nobody can answer.""" TOKEN_THRESHOLD = 30000 FILE_COUNT_THRESHOLD = 100 - is_large_context = est_tokens > TOKEN_THRESHOLD or total_files > FILE_COUNT_THRESHOLD - - if yes: - pass - elif is_large_context: - log_console.print( - f"[yellow]Large:[/yellow] {total_files} files, {decimal(total_size)}, about {est_tokens:,} tokens " - f"(asks above {TOKEN_THRESHOLD:,} tokens or {FILE_COUNT_THRESHOLD} files; -y skips this)" + est_tokens = int(total_size / 4) + if est_tokens <= TOKEN_THRESHOLD and total_files <= FILE_COUNT_THRESHOLD: + return + log_console.print( + f"[yellow]Large:[/yellow] {total_files} files, {decimal(total_size)}, about {est_tokens:,} tokens " + f"(asks above {TOKEN_THRESHOLD:,} tokens or {FILE_COUNT_THRESHOLD} files; -y skips this)" + ) + hint = "Share less by naming paths (cdg scan src/), or cdg digest for signatures only." + if not sys.stdin.isatty(): + # Nobody can answer the question: say how to go on instead of a bare "Aborted!". + log_console.print(f"Stopped. {hint} -y takes it all.") + raise typer.Exit(1) + if not typer.confirm("Continue anyway?"): + log_console.print(f"Nothing shared. {hint}") + raise typer.Exit() + + +def _remote_target( + raw: list[str], github: bool, ref: str | None, paths: list[str] | None, resolve: bool +) -> remote.Target | None: + """The repository to read from GitHub, or None for an ordinary local scan.""" + is_remote = github or any(remote.is_url(r) for r in raw) + if not is_remote: + if ref is not None or paths: + raise typer.BadParameter( + "--ref and --path need a GitHub repository (OWNER/REPO --github, or its URL)", + param_hint="targets", + ) + return None + if len(raw) != 1: + raise typer.BadParameter( + "name one GitHub repository and nothing else; choose its paths with --path", + param_hint="targets", + ) + if resolve: + raise typer.BadParameter("-r does not work on a GitHub repository yet", param_hint="'-r'") + try: + target = remote.parse_target(raw[0]) + if ref is not None: + target.ref = ref + if paths: + target.paths = [remote.repo_target(p) for p in paths] + except ValueError as e: + raise typer.BadParameter(str(e), param_hint="'--path'" if paths else "targets") from None + return target + + +def _scan_remote( + handler, + target: remote.Target, + auth: bool, + max_file_size: int, + yes: bool, + message: str, + line_numbers: bool, + format: Format, +) -> None: + log_console = handler.log_console + with spinner(log_console, f"[bold blue]Listing {target.name}...[/bold blue]"): + plan = remote.plan(target, auth=auth, max_file_size=max_file_size) + + for note in plan.notes: + log_console.print(f"[yellow]Note:[/yellow] {note}", highlight=False) + if plan.withheld: + _report(log_console, "Withheld (may contain secrets)", plan.withheld) + for reason, skipped in plan.skipped.items(): + _report(log_console, f"Skipped ({reason})", skipped) + if len(plan.files) > remote.MAX_FILES or plan.size > remote.MAX_BYTES: + raise remote.RemoteError( + f"{len(plan.files)} files, {decimal(plan.size)} selected; a GitHub scan takes at most " + f"{remote.MAX_FILES:,} files or {decimal(remote.MAX_BYTES)}. Name folders with --path." ) - hint = "Share less by naming paths (cdg scan src/), or cdg digest for signatures only." - if not sys.stdin.isatty(): - # Nobody can answer the question: say how to go on instead of a bare "Aborted!". - log_console.print(f"Stopped. {hint} -y takes it all.") - raise typer.Exit(1) - if not typer.confirm("Continue anyway?"): - log_console.print(f"Nothing shared. {hint}") - raise typer.Exit() + if not yes: + _confirm_size(log_console, len(plan.files), plan.size) + + with ( + spinner(log_console, f"[bold blue]Downloading {plan.name}...[/bold blue]"), + remote.materialize(plan) as root, + ): + # Paths named with --path become targets inside the folder, line ranges included. A + # named path with nothing selected under it (ignored, withheld) was not downloaded; the + # local scan of such a path shares nothing, so it is dropped, not scanned as the root. + named = [ + str(root.joinpath(*path.split("/"))) + span + for path, span in plan.targets + if root.joinpath(*path.split("/")).exists() + ] + targets, ranges = _parse_targets(named) + ctx = common.get_context(targets, root=root) + files = [] if plan.targets and not named else ctx.get_target_files(targets=targets) + _render( + handler, + root, + files, + ranges, + message, + line_numbers, + format, + project=plan.name, + source={"source": plan.source, "ref": plan.ref, "commit": plan.commit}, + label=f"{plan.name}@{plan.ref} ({plan.commit[:7]}) ", + ) + + +def _render( + handler, + root_path: Path, + files: list[Path], + ranges, + message: str, + line_numbers: bool, + format: Format, + project: str | None = None, + source: dict[str, str] | None = None, + label: str = "", +) -> None: + """The snapshot of the selected files, handed to the output. `source` (url, ref and commit) + is set only for a repository read from GitHub.""" + log_console = handler.log_console + project = project or root_path.name + source = source or {} + total_files = len(files) if format != Format.XML: with spinner(log_console, "[bold blue]Generating Snapshot...[/bold blue]"): @@ -126,10 +278,12 @@ def handle( # The block cache holds XML blocks, so other formats read the files afresh. shared = _shared_files(files, ranges, root_path, line_numbers) render = _markdown if format == Format.MARKDOWN else _json - snapshot_content = render(root_path.name, message, tree_str, shared) - _deliver(handler, snapshot_content, total_files) + snapshot_content = render(project, message, tree_str, shared, source) + _deliver(handler, snapshot_content, total_files, label) return + prompt_engine = prompts.get_engine(root_path) + # [4] Execution with spinner(log_console, "[bold blue]Generating Snapshot...[/bold blue]"): tree_str = structure.generate_ascii_tree(files, root_path) @@ -178,22 +332,26 @@ def handle( try: snapshot_content = prompt_engine.render( "snapshot", - project_name=root_path.name, + project_name=project, tree_structure=tree_str, source_code=source_code_blob, instruction=message, + **source, ) except Exception as e: log_console.print(f"[red][Error] Template Rendering Failed:[/red] {e}") raise typer.Exit(1) from e - _deliver(handler, snapshot_content, total_files) + _deliver(handler, snapshot_content, total_files, label) -def _deliver(handler, snapshot: str, total_files: int) -> None: +def _deliver(handler, snapshot: str, total_files: int, label: str = "") -> None: def summary(): tokens = tokenizer.estimate_tokens(snapshot) - handler.log_console.print(f"[bold]Snapshot[/bold] {total_files} files, ~{tokens:,} tokens") + handler.log_console.print( + f"[bold]Snapshot[/bold] {label}{total_files} files, ~{tokens:,} tokens", + highlight=False, + ) handler.handle_output(snapshot, summary=summary, item_name="snapshot") @@ -216,8 +374,16 @@ def _shared_files(files, ranges, root: Path, line_numbers: bool) -> list[dict[st return shared -def _markdown(project: str, instruction: str, tree: str, files: list[dict[str, str]]) -> str: +def _markdown( + project: str, + instruction: str, + tree: str, + files: list[dict[str, str]], + source: dict[str, str] | None = None, +) -> str: parts = [f"# {project}"] + if source: + parts.append(f"Source: {source['source']}, {source['ref']} at {source['commit']}") if instruction: parts.append(instruction) parts.append(f"## Structure\n\n{fenced(tree)}") @@ -229,10 +395,27 @@ def _markdown(project: str, instruction: str, tree: str, files: list[dict[str, s return "\n\n".join(parts) + "\n" -def _json(project: str, instruction: str, tree: str, files: list[dict[str, str]]) -> str: +def _json( + project: str, + instruction: str, + tree: str, + files: list[dict[str, str]], + source: dict[str, str] | None = None, +) -> str: return json.dumps( { "project": project, + **( + { + "source": { + "url": source["source"], + "ref": source["ref"], + "commit": source["commit"], + } + } + if source + else {} + ), "instruction": instruction, "structure": tree, "files": [{k: v for k, v in file.items() if v or k != "lines"} for file in files], diff --git a/src/codigest/core/common.py b/src/codigest/core/common.py index 6b3cf10..19db4bd 100644 --- a/src/codigest/core/common.py +++ b/src/codigest/core/common.py @@ -23,24 +23,29 @@ class ProjectContext: # [수정] targets 타입을 list[Path] | Path로 확장 - def __init__(self, targets: Targets | None = None): + def __init__(self, targets: Targets | None = None, root: Path | None = None): # [추가] 단일 Path가 들어오면 리스트로 감싸기 (Safety) if isinstance(targets, Path): targets = [targets] - # 1. 탐색 시작점 결정 - # Paths in different folders of the same project (aps/a.m Auto_run/b.m) share the - # project you are in; starting from the first path alone made its folder the root. - cwd = Path.cwd().resolve() - resolved = [t.resolve() for t in targets or []] - if not resolved or all(t.is_relative_to(cwd) for t in resolved): - self.start_path = cwd + if root is not None: + # A fixed root (a remote repository's files in a temporary folder): never look + # above it for a project marker. + self.start_path = self.root_path = root else: - self.start_path = Path(os.path.commonpath(resolved)) - - # 2. 루트 찾기 - self.root_path = self._find_project_root(self.start_path) + # 1. 탐색 시작점 결정 + # Paths in different folders of the same project (aps/a.m Auto_run/b.m) share the + # project you are in; starting from the first path alone made its folder the root. + cwd = Path.cwd().resolve() + resolved = [t.resolve() for t in targets or []] + if not resolved or all(t.is_relative_to(cwd) for t in resolved): + self.start_path = cwd + else: + self.start_path = Path(os.path.commonpath(resolved)) + + # 2. 루트 찾기 + self.root_path = self._find_project_root(self.start_path) self.withheld: list[Path] = [] self.config_extensions, self.config_ignores = self._load_config_filters() @@ -123,8 +128,8 @@ def get_target_files( # Helper for quick access -def get_context(targets: Targets | None = None) -> ProjectContext: - return ProjectContext(targets) +def get_context(targets: Targets | None = None, root: Path | None = None) -> ProjectContext: + return ProjectContext(targets, root) def remove_tree(path: Path) -> None: diff --git a/src/codigest/core/prompts.py b/src/codigest/core/prompts.py index 328a9f9..fbce900 100644 --- a/src/codigest/core/prompts.py +++ b/src/codigest/core/prompts.py @@ -29,11 +29,18 @@ def _block(root: str, attrs: dict[str, str], instruction: str, *parts: str) -> s # Note: source_code / digest_content arrive already formatted via tags.file(). def _default_snapshot( - project_name: str, tree_structure: str, source_code: str, instruction: str = "" + project_name: str, + tree_structure: str, + source_code: str, + instruction: str = "", + source: str = "", + ref: str = "", + commit: str = "", ) -> str: + # source, ref and commit are set only for a repository read from GitHub. return _block( "project_snapshot", - {"project": project_name}, + {"project": project_name, "source": source, "ref": ref, "commit": commit}, instruction, f"\n{tree_structure}\n", f"\n{source_code}\n", diff --git a/src/codigest/core/remote.py b/src/codigest/core/remote.py new file mode 100644 index 0000000..0d6e661 --- /dev/null +++ b/src/codigest/core/remote.py @@ -0,0 +1,526 @@ +""" +A GitHub repository read through the same pipeline as a local project (docs/remote-scan.md). + +The ref is pinned to one commit, the listing is filtered with the local scanner's rules, and only +the selected files are downloaded, into a temporary folder that the local scan then reads. +Requests are GET only, to the GitHub API and raw hosts; a token is sent only when asked for. +""" + +import contextlib +import os +import re +import sys +from collections.abc import Generator +from dataclasses import dataclass, field +from pathlib import Path +lazy import json +lazy import subprocess +lazy import tempfile +lazy import time +lazy import urllib.error +lazy import urllib.parse +lazy import urllib.request +lazy from concurrent.futures import ThreadPoolExecutor +lazy from importlib.metadata import PackageNotFoundError, version + +lazy from rich.filesize import decimal + +from . import scanner +from .common import remove_tree +from .redact import is_secret_file + +API = "https://api.github.com" +RAW = "https://raw.githubusercontent.com" +API_VERSION = "2022-11-28" +TIMEOUT = 30 +RETRIES = 2 +BACKOFF = (1, 2) +# A Retry-After longer than this is reported instead of waited for. +MAX_WAIT = 10 +DOWNLOADS = 4 +# Bounds a remote scan even with -y (and so through MCP), where the size question is skipped. +MAX_FILES = 5_000 +MAX_BYTES = 50_000_000 + +_OWNER = re.compile(r"[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?") +_REPO = re.compile(r"[A-Za-z0-9._-]+") +_URL = re.compile(r"^(?:https?://)?(?:www\.)?github\.com/(?P.*)$", re.IGNORECASE) +_SHA = re.compile(r"[0-9a-f]{40}") +# As for local targets (commands/scan.py): FILE:40-60 or FILE:42. +_RANGE = re.compile(r"^(?P.+):(?P\d+)(?:-(?P\d+))?$") +_SIZE = re.compile(r"^(\d+)\s*(B|KB|MB|GB)?$", re.IGNORECASE) +_UNITS = {"": 1, "B": 1, "KB": 1_000, "MB": 1_000_000, "GB": 1_000_000_000} +_RESERVED = re.compile(r"^(con|prn|aux|nul|com[0-9¹²³]|lpt[0-9¹²³])(\..*)?$", re.IGNORECASE) +_LOOPBACK = {"localhost", "127.0.0.1", "::1"} + + +class RemoteError(Exception): + """A failure to report as it is (exit 1).""" + + +class HttpError(RemoteError): + def __init__(self, status: int, message: str): + super().__init__(message) + self.status = status + + +@dataclass(slots=True) +class Target: + owner: str + repo: str + ref: str | None = None + paths: list[str] = field(default_factory=list) + + @property + def name(self) -> str: + return f"{self.owner}/{self.repo}" + + +@dataclass(slots=True, frozen=True) +class Entry: + path: str + kind: str # blob, tree, or commit (a submodule) + mode: str + size: int + sha: str + + +@dataclass(slots=True) +class Plan: + """What a remote scan will download, decided before any file is fetched.""" + + client: Client + name: str + ref: str + commit: str + # The paths named with --path and their line ranges (":40-60" or ""). + targets: list[tuple[str, str]] + files: list[Entry] + withheld: list[str] + skipped: dict[str, list[str]] + notes: list[str] + gitignore: bytes | None + + @property + def source(self) -> str: + return f"https://github.com/{self.name}" + + @property + def size(self) -> int: + return sum(e.size for e in self.files) + + +def is_url(raw: str) -> bool: + return bool(_URL.match(raw)) + + +def parse_target(raw: str) -> Target: + """OWNER/REPO, or a github.com URL with an optional /tree/REF/PATH or /blob/REF/PATH.""" + ref, paths = None, [] + if m := _URL.match(raw): + rest = m["rest"].split("#", 1)[0].split("?", 1)[0].strip("/") + parts = [urllib.parse.unquote(p) for p in rest.split("/")] + if len(parts) < 2 or not all(parts[:2]): + raise ValueError(f"{raw} does not name a repository (https://github.com/OWNER/REPO)") + owner, repo, *tail = parts + repo = repo.removesuffix(".git") + if tail: + if tail[0] not in ("tree", "blob") or len(tail) < 2: + raise ValueError(f"{raw} is not a link to a repository, folder or file") + ref = tail[1] + if len(tail) > 2: + paths = [repo_path("/".join(tail[2:]))] + else: + parts = raw.split("/") + if len(parts) != 2: + raise ValueError(f"{raw} is not OWNER/REPO") + owner, repo = parts + if not _OWNER.fullmatch(owner) or not _REPO.fullmatch(repo) or repo in (".", ".."): + raise ValueError(f"{owner}/{repo} is not a GitHub repository name") + return Target(owner, repo, ref, paths) + + +def repo_path(raw: str) -> str: + """A path inside the repository as POSIX relative to its root ("" is the root).""" + if "\\" in raw or raw.startswith("/") or re.match(r"^[A-Za-z]:", raw): + raise ValueError(f"{raw} is not a path inside the repository (use a/b, relative)") + parts = [p for p in raw.split("/") if p not in ("", ".")] + if ".." in parts: + raise ValueError(f"{raw} leaves the repository") + return "/".join(parts) + + +def _split_range(raw: str) -> tuple[str, str]: + """`src/a.py:40-60` -> ("src/a.py", ":40-60"); a plain path has an empty range.""" + if m := _RANGE.match(raw): + return m["path"], raw[m.end("path") :] + return raw, "" + + +def repo_target(raw: str) -> str: + """A --path value, checked and normalized: a path inside the repository, optionally with + a line range as for local targets.""" + path, span = _split_range(raw) + return repo_path(path) + span + + +def parse_size(raw: str) -> int: + if not (m := _SIZE.match(raw.strip())): + raise ValueError(f"{raw} is not a size (1MB, 500KB, 20000)") + return int(m[1]) * _UNITS[(m[2] or "").upper()] + + +def token() -> str: + """The token --auth sends: GH_TOKEN, GITHUB_TOKEN, then `gh auth token`.""" + for name in ("GH_TOKEN", "GITHUB_TOKEN"): + if value := os.environ.get(name, "").strip(): + return value + try: + result = subprocess.run( + ["gh", "auth", "token"], capture_output=True, text=True, timeout=10, check=False + ) + except OSError, subprocess.TimeoutExpired: + result = None + if result and result.returncode == 0 and (value := result.stdout.strip()): + return value + raise RemoteError("--auth needs GH_TOKEN, GITHUB_TOKEN or a logged-in gh.") + + +def _netloc(url: str) -> str: + return urllib.parse.urlsplit(url).netloc.lower() + + +def _user_agent() -> str: + try: + return f"codigest/{version('codigest')}" + except PackageNotFoundError: + return "codigest/dev" + + +def _redirects(client: Client) -> urllib.request.HTTPRedirectHandler: + """Follows redirects only between the hosts codigest talks to, and gives the token back + only to the API host (urllib drops unredirected headers on every redirect). Defined here so + that importing this module does not load urllib.""" + + class Redirects(urllib.request.HTTPRedirectHandler): + max_redirections = 5 + + def redirect_request(self, req, fp, code, msg, headers, newurl): + if (host := _netloc(newurl)) not in client.hosts: + raise RemoteError(f"GitHub redirected to {host}, which codigest does not contact.") + new = super().redirect_request(req, fp, code, msg, headers, newurl) + if new is not None: + client.authorize(new) + return new + + return Redirects() + + +class Client: + """GET requests to the GitHub API and raw hosts, nothing else.""" + + def __init__(self, token: str | None = None): + # Overridable for tests (a fake server on this machine). + self.api = os.environ.get("CODIGEST_GITHUB_API", API).rstrip("/") + self.raw = os.environ.get("CODIGEST_GITHUB_RAW", RAW).rstrip("/") + self.hosts = {_netloc(self.api), _netloc(self.raw)} + api = urllib.parse.urlsplit(self.api) + if token and api.scheme != "https" and api.hostname not in _LOOPBACK: + raise RemoteError(f"--auth sends the token only over https, not to {self.api}.") + self.token = token + self.user_agent = _user_agent() + self._opener = urllib.request.build_opener(_redirects(self)) + + def authorize(self, request: urllib.request.Request) -> None: + if self.token and _netloc(request.full_url) == _netloc(self.api): + request.add_unredirected_header("Authorization", f"Bearer {self.token}") + + def get(self, url: str, accept: str = "application/vnd.github+json") -> tuple[bytes, str]: + """Body and final URL of a GET; HttpError for an answer other than 2xx.""" + request = urllib.request.Request( + url, headers={"Accept": accept, "User-Agent": self.user_agent}, method="GET" + ) + if _netloc(url) == _netloc(self.api): + request.add_header("X-GitHub-Api-Version", API_VERSION) + self.authorize(request) + for attempt in range(RETRIES + 1): + try: + with self._opener.open(request, timeout=TIMEOUT) as response: + return response.read(), response.url + except urllib.error.HTTPError as e: + with e: + status, headers, body = e.code, e.headers, e.read() + wait = _retry_wait(status, headers, attempt) + if wait is None: + raise _http_error(status, headers, body) from None + except (urllib.error.URLError, TimeoutError, ConnectionError) as e: + if attempt == RETRIES: + reason = getattr(e, "reason", None) or e + raise RemoteError(f"Could not reach {_netloc(url)}: {reason}") from None + wait = BACKOFF[attempt] + time.sleep(wait) + raise AssertionError("unreachable") + + def json(self, path: str) -> dict: + body, _ = self.get(f"{self.api}{path}") + return json.loads(body) + + +def _retry_wait(status: int, headers, attempt: int) -> float | None: + """Seconds to wait before trying again, or None to give up.""" + if attempt == RETRIES: + return None + if status in (403, 429): + if headers.get("x-ratelimit-remaining") == "0": + return None + if (after := headers.get("retry-after", "")).isdigit() and int(after) <= MAX_WAIT: + return int(after) + return None + return BACKOFF[attempt] if status >= 500 else None + + +def _http_error(status: int, headers, body: bytes) -> HttpError: + try: + message = str(json.loads(body).get("message", "")) + except ValueError, AttributeError: + message = "" + if status in (403, 429) and headers.get("x-ratelimit-remaining") == "0": + reset = headers.get("x-ratelimit-reset", "") + when = time.strftime("%H:%M", time.localtime(int(reset))) if reset.isdigit() else "later" + return HttpError( + status, + f"GitHub API rate limit reached; resets at {when} (local time). --auth raises it.", + ) + if status in (403, 429) and (after := headers.get("retry-after", "")).isdigit(): + return HttpError( + status, f"GitHub asked to wait {after} s (secondary rate limit). Try again later." + ) + if status == 401: + return HttpError(status, "GitHub rejected the token (401).") + detail = f": {message}" if message else "" + if status == 403: + return HttpError(status, f"GitHub refused the request (403){detail}") + return HttpError(status, f"GitHub answered {status}{detail}") + + +def plan(target: Target, *, auth: bool, max_file_size: int) -> Plan: + """Resolves the ref, lists the tree and selects files, without downloading any but the + root .gitignore.""" + client = Client(token() if auth else None) + notes = [] + try: + info = client.json(f"/repos/{target.name}") + except HttpError as e: + if e.status == 404: + raise RemoteError( + f"{target.name} not found, or the token cannot read it." + if auth + else f"{target.name} not found. If it is private, add --auth." + ) from None + raise + name = str(info["full_name"]) + # The name goes into every later URL, so it must be a name and not a path. + owner, _, repo = name.partition("/") + if not _OWNER.fullmatch(owner) or not _REPO.fullmatch(repo) or repo in (".", ".."): + raise RemoteError(f"GitHub answered with an unexpected repository name for {target.name}.") + if name.lower() != target.name.lower(): + notes.append(f"{target.name} moved to {name}") + default = str(info["default_branch"]) + ref = target.ref or default + try: + body, _ = client.get( + f"{client.api}/repos/{name}/commits/{urllib.parse.quote(ref, safe='/')}", + accept="application/vnd.github.sha", + ) + except HttpError as e: + if e.status in (404, 422): + raise RemoteError( + f"No branch, tag or commit {ref} in {name} (default branch: {default})." + ) from None + raise + if not _SHA.fullmatch(commit := body.decode("ascii", "replace").strip()): + raise RemoteError(f"GitHub gave no commit for {ref}.") + + targets = [_split_range(repo_target(p)) for p in target.paths] + scopes = list(dict.fromkeys(path for path, _ in targets)) + entries = _list(client, name, commit, scopes) + by_path = {e.path: e for e in entries} + for path, span in targets: + if path and path not in by_path: + raise RemoteError(f"{path} is not in {name} at {ref}.") + if span and by_path[path].kind != "blob": + raise RemoteError(f"{path} is not a file in {name}, so {path}{span} is no line range.") + + gitignore = None + if (e := by_path.get(".gitignore")) and e.kind == "blob" and e.mode != "120000": + gitignore = _download(client, name, commit, e) + files, withheld, skipped = _select(entries, gitignore, scopes, max_file_size) + return Plan(client, name, ref, commit, targets, files, withheld, skipped, notes, gitignore) + + +def _entries(data: dict, prefix: str = "") -> list[Entry]: + return [ + Entry( + f"{prefix}{item['path']}", + str(item["type"]), + str(item.get("mode", "")), + int(item.get("size") or 0), + str(item["sha"]), + ) + for item in data.get("tree", []) + ] + + +def _list(client: Client, name: str, commit: str, scopes: list[str]) -> list[Entry]: + """Every entry under the scopes. A recursive listing GitHub had to cut short is replaced by + a walk down to each scope and a listing of that scope alone.""" + data = client.json(f"/repos/{name}/git/trees/{commit}?recursive=1") + if not data.get("truncated"): + return _entries(data) + # The root is the listing that was cut short, so it cannot be a scope here, even next to others. + if not scopes or "" in scopes: + raise RemoteError(f"{name} is too large to list at once; name folders with --path.") + root = _entries(client.json(f"/repos/{name}/git/trees/{commit}")) + found = {e.path: e for e in root if e.path == ".gitignore"} + for scope in scopes: + level, parts = root, scope.split("/") + for depth in range(len(parts)): + here = "/".join(parts[: depth + 1]) + entry = next((e for e in level if e.path == here), None) + if entry is None: + break + found[here] = entry + if entry.kind != "tree": + break + listing = client.json( + f"/repos/{name}/git/trees/{entry.sha}" + + ("?recursive=1" if depth == len(parts) - 1 else "") + ) + if listing.get("truncated"): + raise RemoteError(f"{scope} in {name} is too large to list; name a smaller folder.") + level = _entries(listing, f"{here}/") + if depth == len(parts) - 1: + found |= {e.path: e for e in level} + return list(found.values()) + + +def _scopes_of(path: str, scopes: list[str]) -> list[str]: + """The named paths whose walk reaches this path ("" is the root).""" + if not scopes: + return [""] + return [s for s in scopes if not s or path == s or path.startswith(s + "/")] + + +def _kept(path: str, scope: str, rules: scanner.ProjectScanner) -> bool: + """The local scanner's decision: a folder it skips hides everything below it.""" + if path == scope: + return not rules.is_ignored_rel(path) + parts = path.split("/") + first = len(scope.split("/")) if scope else 0 + for depth in range(first + 1, len(parts)): + if rules.is_ignored_rel("/".join(parts[:depth]), is_dir=True): + return False + return not rules.is_ignored_rel(path) + + +def valid_here(path: str) -> bool: + """Whether the path can be written under a folder on this system as it is named.""" + parts = path.split("/") + if any(p in ("", ".", "..") or "\\" in p or "\0" in p for p in parts): + return False + if os.name == "nt": + return not any( + _RESERVED.match(p) or p[-1] in ". " or any(c in '<>:"|?*' or ord(c) < 32 for c in p) + for p in parts + ) + return True + + +def _select( + entries: list[Entry], gitignore: bytes | None, scopes: list[str], max_file_size: int +) -> tuple[list[Entry], list[str], dict[str, list[str]]]: + with tempfile.TemporaryDirectory(prefix="cdg-rules-") as tmp: + root = Path(tmp) + if gitignore is not None: + (root / ".gitignore").write_bytes(gitignore) + rules = scanner.ProjectScanner(root) + + files, withheld = [], [] + skipped: dict[str, list[str]] = {} + seen: set[str] = set() + fold = os.name == "nt" or sys.platform == "darwin" + for entry in sorted(entries, key=lambda e: e.path): + # Named paths are walked one by one and the results joined, as the local scanner does. + if entry.kind == "tree" or not any( + _kept(entry.path, scope, rules) for scope in _scopes_of(entry.path, scopes) + ): + continue + reason = None + if entry.kind == "commit": + reason = "submodule" + elif entry.mode == "120000": + reason = "symlink" + elif is_secret_file(entry.path.rsplit("/", 1)[-1]): + withheld.append(entry.path) + continue + elif not valid_here(entry.path): + reason = "path not valid on this system" + elif entry.size > max_file_size: + reason = f"over {_size_label(max_file_size)}" + elif (key := entry.path.casefold() if fold else entry.path) in seen: + reason = "same name as another file except for case" + else: + seen.add(key) + files.append(entry) + continue + skipped.setdefault(reason, []).append(entry.path) + return files, withheld, skipped + + +def _size_label(size: int) -> str: + return decimal(size) + + +def _download(client: Client, name: str, commit: str, entry: Entry) -> bytes: + if client.token: + # Blobs by content hash from the API, so the token never goes to another host. + url = f"{client.api}/repos/{name}/git/blobs/{entry.sha}" + body, _ = client.get(url, accept="application/vnd.github.raw+json") + else: + url = f"{client.raw}/{name}/{commit}/{urllib.parse.quote(entry.path)}" + body, _ = client.get(url, accept="*/*") + return body + + +@contextlib.contextmanager +def materialize(plan: Plan) -> Generator[Path]: + """The selected files (and the root .gitignore) in a new temporary folder, removed after.""" + root = Path(tempfile.mkdtemp(prefix="cdg-remote-")).resolve() + try: + if plan.gitignore is not None: + (root / ".gitignore").write_bytes(plan.gitignore) + pending = [ + e for e in plan.files if not (e.path == ".gitignore" and plan.gitignore is not None) + ] + with ThreadPoolExecutor(DOWNLOADS) as pool: + futures = [ + (e, pool.submit(_download, plan.client, plan.name, plan.commit, e)) for e in pending + ] + try: + for entry, future in futures: + _write(root, entry.path, future.result()) + except BaseException: + pool.shutdown(cancel_futures=True) + raise + yield root + finally: + remove_tree(root) + + +def _write(root: Path, path: str, data: bytes) -> None: + target = root.joinpath(*path.split("/")) + # The listing comes from the network: whatever it names must land inside the folder. + if not target.resolve().is_relative_to(root): + raise RemoteError(f"{path} would be written outside the temporary folder.") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) diff --git a/src/codigest/core/scanner.py b/src/codigest/core/scanner.py index 1e8e5da..4a51b67 100644 --- a/src/codigest/core/scanner.py +++ b/src/codigest/core/scanner.py @@ -62,12 +62,14 @@ def is_ignored(self, path: Path) -> bool: try: # pathspec requires relative paths (e.g., "src/main.py") rel_path = path.relative_to(self.root_path).as_posix() - if path.is_dir(): - rel_path += "/" except ValueError: return False - return self.ignore_spec.match_file(rel_path) + return self.is_ignored_rel(rel_path, path.is_dir()) + + def is_ignored_rel(self, rel_path: str, is_dir: bool = False) -> bool: + """The same check for a POSIX path relative to the root, which need not exist on disk.""" + return self.ignore_spec.match_file(rel_path + "/" if is_dir else rel_path) def _is_included(self, entry: Path) -> bool: """파일이 지정된 include_paths 범위 안에 있는지 확인""" diff --git a/tests/fake_github.py b/tests/fake_github.py new file mode 100644 index 0000000..826b5ed --- /dev/null +++ b/tests/fake_github.py @@ -0,0 +1,192 @@ +"""A fake GitHub for remote scan tests: the API and raw hosts on two local ports, serving one +repository from a folder and recording every request.""" + +import contextlib +import hashlib +import json +import threading +import urllib.parse +from collections.abc import Generator +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + + +def blob_sha(data: bytes) -> str: + return hashlib.sha1(b"blob %d\0" % len(data) + data).hexdigest() + + +def fake_sha(text: str) -> str: + return hashlib.sha1(text.encode()).hexdigest() + + +@dataclass(slots=True) +class Request: + method: str + host: str # "api" or "raw" + path: str + headers: dict[str, str] + + +@dataclass +class FakeGitHub: + root: Path + name: str = "octo/sample" + default_branch: str = "main" + # Repository is private: answers 404 unless this token is sent. + private_token: str | None = None + # A former name that redirects to `name`. + moved_from: str | None = None + truncate: bool = False + # Listing entries without a file behind them (symlinks, submodules, hostile paths). + extra: list[dict] = field(default_factory=list) + # Canned answers, by request path with query, used before the normal one. + queue: dict[str, list[tuple[int, dict[str, str], bytes]]] = field(default_factory=dict) + requests: list[Request] = field(default_factory=list) + api: str = "" + raw: str = "" + + @property + def commit(self) -> str: + return fake_sha(f"{self.name}@commit") + + def files(self) -> dict[str, bytes]: + return { + p.relative_to(self.root).as_posix(): p.read_bytes() + for p in sorted(self.root.rglob("*")) + if p.is_file() + } + + def entries(self, folder: str, recursive: bool) -> list[dict]: + """Tree entries under `folder` ("" for the root), paths relative to it.""" + prefix = f"{folder}/" if folder else "" + files = {p: d for p, d in self.files().items() if p.startswith(prefix)} + dirs = {"/".join(p.split("/")[:i]) for p in files for i in range(1, p.count("/") + 1)} + items = [ + {"path": d, "type": "tree", "mode": "040000", "sha": fake_sha(f"tree:{d}")} + for d in dirs + if d.startswith(prefix) + ] + items += [ + {"path": p, "type": "blob", "mode": "100644", "size": len(d), "sha": blob_sha(d)} + for p, d in files.items() + ] + if not folder: + items += self.extra + items = [{**i, "path": i["path"].removeprefix(prefix)} for i in items] + if not recursive: + items = [i for i in items if "/" not in i["path"]] + return sorted(items, key=lambda i: i["path"]) + + def folder_of(self, sha: str) -> str | None: + if sha in (self.commit, fake_sha("tree:")): + return "" + dirs = { + "/".join(p.split("/")[:i]) for p in self.files() for i in range(1, p.count("/") + 1) + } + return next((d for d in dirs if fake_sha(f"tree:{d}") == sha), None) + + def answer(self, host: str, path: str, headers: dict[str, str]) -> tuple[int, dict, bytes]: + if canned := self.queue.get(path): + return canned.pop(0) + parsed = urllib.parse.urlsplit(path) + parts = [urllib.parse.unquote(p) for p in parsed.path.strip("/").split("/")] + authorized = self.private_token is None or headers.get("authorization") == ( + f"Bearer {self.private_token}" + ) + not_found = (404, {}, json.dumps({"message": "Not Found"}).encode()) + if host == "raw": + owner, repo, commit, *rest = parts + data = self.files().get("/".join(rest)) + if ( + not authorized + or f"{owner}/{repo}" != self.name + or commit != self.commit + or data is None + ): + return not_found + return 200, {}, data + if parts[0] != "repos" or len(parts) < 3: + return not_found + repo = f"{parts[1]}/{parts[2]}" + if repo == self.moved_from: + rest = "/".join(parts[3:]) + target = f"{self.api}/repos/{self.name}" + (f"/{rest}" if rest else "") + return 301, {"Location": target}, b"" + if repo != self.name or not authorized: + return not_found + match parts[3:]: + case []: + body = {"full_name": self.name, "default_branch": self.default_branch} + return 200, {}, json.dumps(body).encode() + case ["commits", *ref]: + if "/".join(ref) in (self.default_branch, self.commit): + return 200, {}, self.commit.encode() + return 422, {}, json.dumps({"message": "No commit found"}).encode() + case ["git", "trees", sha]: + if (folder := self.folder_of(sha)) is None: + return not_found + recursive = "recursive=1" in parsed.query + items = self.entries(folder, recursive) + truncated = self.truncate and recursive and folder == "" + if truncated: + items = items[:1] + body = {"sha": sha, "tree": items, "truncated": truncated} + return 200, {}, json.dumps(body).encode() + case ["git", "blobs", sha]: + data = next((d for d in self.files().values() if blob_sha(d) == sha), None) + return not_found if data is None else (200, {}, data) + return not_found + + def fetched(self) -> list[str]: + """Repository paths whose content was requested, from either host.""" + files = self.files() + by_sha = {blob_sha(d): p for p, d in files.items()} + paths = [] + for r in self.requests: + parts = [urllib.parse.unquote(p) for p in r.path.split("?")[0].strip("/").split("/")] + if r.host == "raw": + paths.append("/".join(parts[3:])) + elif parts[3:5] == ["git", "blobs"]: + paths.append(by_sha.get(parts[5], parts[5])) + return paths + + @contextlib.contextmanager + def running(self) -> Generator[FakeGitHub]: + servers = [ThreadingHTTPServer(("127.0.0.1", 0), _handler(self, h)) for h in ("api", "raw")] + self.api, self.raw = (f"http://127.0.0.1:{s.server_address[1]}" for s in servers) + threads = [ + threading.Thread(target=s.serve_forever, args=(0.02,), daemon=True) for s in servers + ] + for t in threads: + t.start() + try: + yield self + finally: + for s in servers: + s.shutdown() + s.server_close() + + +def _handler(fake: FakeGitHub, host: str) -> type[BaseHTTPRequestHandler]: + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args) -> None: + pass + + def _serve(self) -> None: + headers = {k.lower(): v for k, v in self.headers.items()} + fake.requests.append(Request(self.command, host, self.path, headers)) + if self.command != "GET": + status, extra, body = 405, {}, b"" + else: + status, extra, body = fake.answer(host, self.path, headers) + self.send_response(status) + for key, value in extra.items(): + self.send_header(key, value) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + do_GET = do_POST = do_PUT = do_PATCH = do_DELETE = _serve + + return Handler diff --git a/tests/test_remote_scan.py b/tests/test_remote_scan.py new file mode 100644 index 0000000..6999092 --- /dev/null +++ b/tests/test_remote_scan.py @@ -0,0 +1,563 @@ +"""`cdg scan OWNER/REPO --github` against a fake GitHub on this machine (no network).""" + +import json +import os +import re +import tempfile +import time + +import pytest +from fake_github import FakeGitHub +from typer.testing import CliRunner + +from codigest.core import remote +from codigest.main import app + +TOKEN = "fixture-token-not-a-secret" + +SAMPLE = { + ".gitignore": b"build/\n*.log\n!keep.log\n", + "README.md": b"# sample\n", + "src/app.py": b"from util.helpers import twice\n\n\ndef main():\n return twice(2)\n", + "src/util/helpers.py": b'def twice(x):\n """Double."""\n return 2 * x\n', + "src/data.bin": b"\x00\xff\x10binary", + "src/crlf.txt": b"one\r\ntwo\r\n", + "docs/한글.md": "제목\n".encode(), + "build/out.py": b"generated = True\n", + "debug.log": b"noise\n", + "keep.log": b"kept by negation\n", + "node_modules/x/index.js": b"module.exports = 1\n", + ".env": b"PASSWORD=fixture\n", + "keys/id_rsa": b"not a key\n", + # Built here so no token-shaped string sits in the repository. + "config.py": b'GITHUB = "ghp_' + b"A" * 36 + b'"\n', +} + + +def write(root, files): + for name, data in files.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + +@pytest.fixture +def github(tmp_path, monkeypatch): + repo = tmp_path / "sample" + write(repo, SAMPLE) + work = tmp_path / "work" + work.mkdir() + temp = tmp_path / "temp" + temp.mkdir() + monkeypatch.chdir(work) + monkeypatch.setattr(tempfile, "tempdir", str(temp)) + for name in ("GH_TOKEN", "GITHUB_TOKEN"): + monkeypatch.delenv(name, raising=False) + fake = FakeGitHub(repo) + with fake.running(): + monkeypatch.setenv("CODIGEST_GITHUB_API", fake.api) + monkeypatch.setenv("CODIGEST_GITHUB_RAW", fake.raw) + yield fake + # Nothing left behind: no download folder, nothing in the working directory. + assert list(temp.iterdir()) == [] + assert list(work.iterdir()) == [] + + +@pytest.fixture +def sleeps(monkeypatch): + waited = [] + monkeypatch.setattr(remote.time, "sleep", waited.append) + return waited + + +def run(*args): + return CliRunner().invoke(app, ["scan", *args]) + + +def local(repo, monkeypatch, *args): + """The same scan on the folder the fake serves, as a local project.""" + with monkeypatch.context() as m: + m.chdir(repo) + return run(*args) + + +def body(stdout: str) -> str: + """Everything but the root element's opening line, which names the project.""" + return stdout.split("\n", 1)[1] + + +def uncolor(text: str) -> str: + """Without the colors rich adds where they are forced (CI).""" + return re.sub(r"\x1b\[[0-9;]*m", "", text) + + +def plain(text: str) -> str: + """Messages as words: no colors, box lines or wrapping.""" + return " ".join(uncolor(text).replace("│", " ").split()) + + +def test_the_whole_repository_comes_out_as_a_local_scan_of_it(github, monkeypatch): + remote_result = run("octo/sample", "--github", "-y") + local_result = local(github.root, monkeypatch, "-y") + + assert remote_result.exit_code == 0, remote_result.output + assert body(remote_result.stdout) == body(local_result.stdout) + assert remote_result.stdout.split("\n", 1)[0] == ( + '' + ) + assert "Withheld (may contain secrets): .env, keys/id_rsa" in plain(remote_result.stderr) + assert "Withheld (may contain secrets): .env, keys/id_rsa" in plain(local_result.stderr) + assert "ghp_" not in remote_result.stdout + assert "<>" in remote_result.stdout + + +@pytest.mark.parametrize( + "paths", + [ + ["src/"], + ["src/util", "keep.log"], + ["src/app.py:4-5", "README.md"], + ["build/out.py"], + ["build", ".env", "src/app.py"], + ["src", "src/util/helpers.py", "keep.log", "debug.log"], + [".env"], + ["docs/한글.md"], + ], +) +def test_paths_select_what_local_targets_select(github, monkeypatch, paths): + remote_result = run("octo/sample", "--github", "-y", *(a for p in paths for a in ("--path", p))) + local_result = local(github.root, monkeypatch, "-y", *paths) + + assert remote_result.exit_code == 0, remote_result.output + assert body(remote_result.stdout) == body(local_result.stdout) + + +def test_url_forms_name_the_ref_and_path(github): + by_url = run("https://github.com/octo/sample/tree/main/src/util", "-y") + by_option = run("octo/sample", "--github", "--ref", "main", "--path", "src/util", "-y") + + assert by_url.exit_code == 0, by_url.output + assert by_url.stdout == by_option.stdout + assert ( + by_url.stdout + == run("https://github.com/octo/sample.git", "-y", "--path", "src/util").stdout + ) + + +def test_max_file_size_is_a_remote_rule_the_local_scan_does_not_have(github, monkeypatch): + # Outside the "same as local" tests: a local scan has no size limit per file. A remote scan + # equals the local scan of the same folder without the files over the limit. + write(github.root, {"src/big.py": b"x = 1\n" * 400}) + result = run("octo/sample", "--github", "-y", "--max-file-size", "2KB") + (github.root / "src" / "big.py").unlink() + without_big = local(github.root, monkeypatch, "-y") + + assert result.exit_code == 0, result.output + assert "Skipped (over 2.0 kB): src/big.py" in plain(result.stderr) + assert "src/big.py" not in github.fetched() + assert body(result.stdout) == body(without_big.stdout) + + +def test_files_that_are_not_shared_are_never_downloaded(github): + result = run("octo/sample", "--github", "-y", "--max-file-size", "12") + + assert result.exit_code == 0, result.output + fetched = set(github.fetched()) + assert not fetched & {".env", "keys/id_rsa", "build/out.py", "debug.log"} + assert not any(p.startswith("node_modules/") for p in fetched) + assert "src/app.py" not in fetched # over 12 bytes + assert "Skipped (over 12 bytes): " in plain(result.stderr) + assert "src/app.py" in plain(result.stderr).split("Skipped (over 12 bytes): ")[1] + assert "README.md" in fetched + + +def test_paths_outside_the_selection_are_never_downloaded(github): + result = run("octo/sample", "--github", "-y", "--path", "src/util") + + assert result.exit_code == 0, result.output + # The root .gitignore is read for its rules, as a local scan of src/util reads it. + assert sorted(github.fetched()) == [".gitignore", "src/util/helpers.py"] + + +def test_the_repository_codigest_folder_is_neither_read_nor_applied(github): + write( + github.root, + { + ".codigest/prompts.toml": b'[prompts]\nsnapshot = "HIJACKED {project_name}"\n', + ".codigest/config.toml": b'[filter]\nextensions = [".md"]\n', + }, + ) + result = run("octo/sample", "--github", "-y") + + assert result.exit_code == 0, result.output + assert "HIJACKED" not in result.stdout + assert '' in result.stdout + assert not any(p.startswith(".codigest/") for p in github.fetched()) + + +def test_only_get_requests_to_the_two_hosts(github): + result = run("octo/sample", "--github", "-y") + + assert result.exit_code == 0, result.output + assert {r.method for r in github.requests} == {"GET"} + assert {r.host for r in github.requests} == {"api", "raw"} + + +def test_no_token_is_sent_without_auth_even_when_one_is_set(github, monkeypatch): + monkeypatch.setenv("GH_TOKEN", TOKEN) + result = run("octo/sample", "--github", "-y") + + assert result.exit_code == 0, result.output + assert not any("authorization" in r.headers for r in github.requests) + + +def test_a_private_repository_needs_auth(github, monkeypatch): + github.private_token = TOKEN + monkeypatch.setenv("GH_TOKEN", TOKEN) + + refused = run("octo/sample", "--github", "-y") + assert refused.exit_code == 1 + assert "octo/sample not found. If it is private, add --auth." in plain(refused.stderr) + assert refused.stdout == "" + + github.requests.clear() + allowed = run("octo/sample", "--github", "-y", "--auth") + assert allowed.exit_code == 0, allowed.output + assert '' in allowed.stdout + # Files come from the API by content hash; the token goes nowhere else. + assert {r.host for r in github.requests} == {"api"} + assert all(r.headers.get("authorization") == f"Bearer {TOKEN}" for r in github.requests) + assert TOKEN not in allowed.stdout + allowed.stderr + + +def test_a_token_that_cannot_read_the_repository(github, monkeypatch): + github.private_token = TOKEN + monkeypatch.setenv("GITHUB_TOKEN", "another-token") + result = run("octo/sample", "--github", "-y", "--auth") + + assert result.exit_code == 1 + assert "octo/sample not found, or the token cannot read it." in plain(result.stderr) + assert "another-token" not in result.output + + +def test_a_rejected_token_is_not_repeated_in_the_message(github, monkeypatch): + monkeypatch.setenv("GH_TOKEN", TOKEN) + github.queue["/repos/octo/sample"] = [(401, {}, b'{"message": "Bad credentials"}')] + result = run("octo/sample", "--github", "-y", "--auth") + + assert result.exit_code == 1 + assert "GitHub rejected the token (401)." in plain(result.stderr) + assert TOKEN not in result.output + + +def test_auth_without_any_token_says_where_one_comes_from(github, monkeypatch, tmp_path): + monkeypatch.setenv("PATH", str(tmp_path / "no-bin")) + result = run("octo/sample", "--github", "-y", "--auth") + + assert result.exit_code == 1 + assert "--auth needs GH_TOKEN, GITHUB_TOKEN or a logged-in gh." in plain(result.stderr) + assert github.requests == [] + + +def test_a_token_is_not_sent_in_clear_text_to_another_machine(monkeypatch): + monkeypatch.setenv("CODIGEST_GITHUB_API", "http://example.com") + with pytest.raises(remote.RemoteError, match="only over https"): + remote.Client(TOKEN) + + +def test_a_moved_repository_is_followed_and_named(github): + github.moved_from = "octo/old" + result = run("octo/old", "--github", "-y", "--path", "README.md") + + assert result.exit_code == 0, result.output + assert "octo/old moved to octo/sample" in plain(result.stderr) + assert 'project="octo/sample"' in result.stdout + + +def test_a_redirect_to_another_host_is_refused(github): + github.queue["/repos/octo/sample"] = [(302, {"Location": "http://192.0.2.1/x"}, b"")] + result = run("octo/sample", "--github", "-y") + + assert result.exit_code == 1 + assert "redirected to 192.0.2.1, which codigest does not contact" in plain(result.stderr) + assert result.stdout == "" + + +def test_an_unknown_ref_names_the_default_branch(github): + result = run("octo/sample", "--github", "--ref", "gone") + + assert result.exit_code == 1 + assert "No branch, tag or commit gone in octo/sample (default branch: main)." in plain( + result.stderr + ) + + +def test_a_renamed_default_branch_is_read_afresh(github): + assert 'ref="main"' in run("octo/sample", "--github", "-y").stdout + github.default_branch = "trunk" + + result = run("octo/sample", "--github", "-y") + assert result.exit_code == 0, result.output + assert 'ref="trunk"' in result.stdout + + +def test_a_ref_pins_every_download_to_one_commit(github): + result = run("octo/sample", "--github", "--ref", "main", "-y") + + assert result.exit_code == 0, result.output + raw = [r.path for r in github.requests if r.host == "raw"] + assert raw and all(p.startswith(f"/octo/sample/{github.commit}/") for p in raw) + + +def test_the_rate_limit_is_reported_with_its_reset_time(github, sleeps): + reset = int(time.time()) + 600 + github.queue["/repos/octo/sample"] = [ + (403, {"x-ratelimit-remaining": "0", "x-ratelimit-reset": str(reset)}, b"{}") + ] + result = run("octo/sample", "--github", "-y") + + when = time.strftime("%H:%M", time.localtime(reset)) + assert result.exit_code == 1 + assert f"GitHub API rate limit reached; resets at {when} (local time)." in plain(result.stderr) + assert sleeps == [] + + +def test_a_short_retry_after_is_waited_for(github, sleeps): + github.queue["/repos/octo/sample"] = [(429, {"retry-after": "3"}, b"")] + result = run("octo/sample", "--github", "-y") + + assert result.exit_code == 0, result.output + assert sleeps == [3] + + +def test_a_long_retry_after_is_reported(github, sleeps): + github.queue["/repos/octo/sample"] = [(403, {"retry-after": "60"}, b"")] + result = run("octo/sample", "--github", "-y") + + assert result.exit_code == 1 + assert "GitHub asked to wait 60 s (secondary rate limit)." in plain(result.stderr) + assert sleeps == [] + + +def test_server_errors_are_retried_twice(github, sleeps): + github.queue["/repos/octo/sample"] = [(502, {}, b""), (503, {}, b"")] + assert run("octo/sample", "--github", "-y").exit_code == 0 + assert sleeps == [1, 2] + + github.queue["/repos/octo/sample"] = [(500, {}, b"")] * 3 + result = run("octo/sample", "--github", "-y") + assert result.exit_code == 1 + assert "GitHub answered 500" in plain(result.stderr) + + +def test_a_failed_download_gives_no_partial_snapshot(github, sleeps): + github.queue[f"/octo/sample/{github.commit}/README.md"] = [(500, {}, b"")] * 3 + result = run("octo/sample", "--github", "-y") + + assert result.exit_code == 1 + assert result.stdout == "" + + +def test_an_unreachable_host_is_named(github, monkeypatch, sleeps): + monkeypatch.setenv("CODIGEST_GITHUB_API", "http://127.0.0.1:9") + result = run("octo/sample", "--github", "-y") + + assert result.exit_code == 1 + assert "Could not reach 127.0.0.1:9" in plain(result.stderr) + assert sleeps == [1, 2] + + +def test_a_truncated_listing_walks_down_to_the_named_paths(github): + expected = run("octo/sample", "--github", "-y", "--path", "src/util").stdout + github.truncate = True + + whole = run("octo/sample", "--github", "-y") + assert whole.exit_code == 1 + assert "too large to list at once; name folders with --path" in plain(whole.stderr) + + scoped = run("octo/sample", "--github", "-y", "--path", "src/util") + assert scoped.exit_code == 0, scoped.output + assert scoped.stdout == expected + + +@pytest.mark.parametrize("root", [".", "./", ""]) +def test_a_truncated_listing_cannot_give_the_root_by_its_name(github, root): + # The root is the one listing GitHub already cut short: walking to it gave only the + # other paths, so root files went missing with exit 0 (review on #33). + github.truncate = True + result = run("octo/sample", "--github", "-y", "--path", root, "--path", "src") + + assert result.exit_code == 1 + assert "too large to list at once; name folders with --path" in plain(result.stderr) + assert result.stdout == "" + + +def test_entries_that_are_not_plain_files_are_skipped_and_named(github): + odd = {"type": "blob", "mode": "100644", "size": 3, "sha": "0" * 40} + github.extra = [ + {**odd, "path": "../evil.py"}, + {**odd, "path": "link.py", "mode": "120000"}, + {"path": "vendor", "type": "commit", "mode": "160000", "sha": "1" * 40}, + ] + result = run("octo/sample", "--github", "-y") + + assert result.exit_code == 0, result.output + assert "Skipped (path not valid on this system): ../evil.py" in plain(result.stderr) + assert "Skipped (symlink): link.py" in plain(result.stderr) + assert "Skipped (submodule): vendor" in plain(result.stderr) + assert not {"../evil.py", "link.py", "vendor"} & set(github.fetched()) + + +def test_a_listed_path_cannot_be_written_outside_the_folder(tmp_path): + with pytest.raises(remote.RemoteError, match="outside the temporary folder"): + remote._write(tmp_path, "a/../../x.py", b"") + + +@pytest.mark.skipif(os.name != "nt", reason="names reserved on Windows") +def test_names_windows_cannot_hold_are_skipped(): + assert not remote.valid_here("src/aux.py") + assert not remote.valid_here("a:b") + assert not remote.valid_here("dot.") + assert remote.valid_here("src/auxiliary.py") + + +def test_the_size_question_comes_before_any_download(github, monkeypatch): + write(github.root, {f"many/{i:03}.py": b"x = 1\n" for i in range(101)}) + remote_result = run("octo/sample", "--github") + local_result = local(github.root, monkeypatch) + + assert remote_result.exit_code == 1 + large = [ + line for line in uncolor(remote_result.stderr).splitlines() if line.startswith("Large:") + ] + assert large == [ + line for line in uncolor(local_result.stderr).splitlines() if line.startswith("Large:") + ] + assert "Stopped." in plain(remote_result.stderr) + assert github.fetched() == [".gitignore"] + + +def test_a_hard_ceiling_applies_even_with_yes(github, monkeypatch): + monkeypatch.setattr(remote, "MAX_FILES", 2) + result = run("octo/sample", "--github", "-y") + + assert result.exit_code == 1 + assert "a GitHub scan takes at most 2 files" in plain(result.stderr) + assert github.fetched() == [".gitignore"] + + +def test_markdown_and_json_name_the_source(github): + markdown = run("octo/sample", "--github", "-y", "--format", "markdown", "--path", "README.md") + assert markdown.stdout.startswith( + f"# octo/sample\n\nSource: https://github.com/octo/sample, main at {github.commit}\n" + ) + data = json.loads(run("octo/sample", "--github", "-y", "--format", "json").stdout) + assert data["project"] == "octo/sample" + assert data["source"] == { + "url": "https://github.com/octo/sample", + "ref": "main", + "commit": github.commit, + } + + +@pytest.mark.parametrize( + ("raw", "owner", "repo", "ref", "paths"), + [ + ("octo/sample", "octo", "sample", None, []), + ("https://github.com/octo/sample", "octo", "sample", None, []), + ("https://github.com/octo/sample/", "octo", "sample", None, []), + ("https://github.com/octo/sample.git", "octo", "sample", None, []), + ("http://github.com/octo/sample", "octo", "sample", None, []), + ("github.com/octo/sample", "octo", "sample", None, []), + ("https://github.com/octo/sample/tree/v1.0", "octo", "sample", "v1.0", []), + ("https://github.com/octo/sample/tree/main/src/app", "octo", "sample", "main", ["src/app"]), + ("https://github.com/octo/sample/blob/main/a%20b.py", "octo", "sample", "main", ["a b.py"]), + ("https://github.com/octo/my.repo-1?tab=readme", "octo", "my.repo-1", None, []), + ], +) +def test_accepted_targets(raw, owner, repo, ref, paths): + assert remote.parse_target(raw) == remote.Target(owner, repo, ref, paths) + + +@pytest.mark.parametrize( + "raw", + [ + "octo", + "octo/sample/extra", + "-octo/sample", + "octo/..", + "https://github.com/octo", + "https://github.com/octo/sample/issues/1", + "https://github.com/octo/sample/tree/main/../../x", + ], +) +def test_rejected_targets(raw): + with pytest.raises(ValueError): + remote.parse_target(raw) + + +@pytest.mark.parametrize( + ("args", "message"), + [ + (["--ref", "main"], "--ref and --path need a GitHub repository"), + (["octo/sample", "src", "--github"], "name one GitHub repository and nothing else"), + (["octo/sample", "--github", "-r"], "-r does not work on a GitHub repository yet"), + (["octo/sample/extra", "--github"], "is not OWNER/REPO"), + (["octo/sample", "--github", "--path", "../x"], "leaves the repository"), + (["octo/sample", "--github", "--path", "/etc"], "not a path inside the repository"), + (["octo/sample", "--github", "--max-file-size", "big"], "is not a size"), + ], +) +def test_usage_errors_stop_before_any_request(github, args, message): + result = run(*args) + + assert result.exit_code == 2 + assert message in plain(result.stderr) + assert github.requests == [] + + +def test_a_missing_local_path_that_looks_like_a_repository_suggests_github(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + result = run("octo/sample") + + assert result.exit_code == 2 + assert "add --github to scan the GitHub repository octo/sample" in plain(result.stderr) + + +@pytest.mark.parametrize( + ("raw", "size"), + [("1MB", 1_000_000), ("500KB", 500_000), ("20000", 20_000), ("2 gb", 2 * 10**9)], +) +def test_sizes(raw, size): + assert remote.parse_size(raw) == size + + +@pytest.mark.skipif( + not os.environ.get("CODIGEST_NETWORK"), reason="reads github.com; CODIGEST_NETWORK=1" +) +def test_github_gives_what_a_checkout_of_the_same_tag_gives(tmp_path, monkeypatch): + import io + import subprocess + import tarfile + from pathlib import Path + + tag, folder = "v0.9.0", "src/codigest/core" + archive = subprocess.run( + ["git", "archive", "--format=tar", tag, ".gitignore", folder], + cwd=Path(__file__).parent.parent, + capture_output=True, + check=True, + ).stdout + checkout = tmp_path / "checkout" + with tarfile.open(fileobj=io.BytesIO(archive)) as tar: + tar.extractall(checkout, filter="data") + for name in ("CODIGEST_GITHUB_API", "CODIGEST_GITHUB_RAW", "GH_TOKEN", "GITHUB_TOKEN"): + monkeypatch.delenv(name, raising=False) + monkeypatch.chdir(tmp_path) + + on_github = run("NaNIsaac/codigest", "--github", "--ref", tag, "--path", folder, "-y") + on_disk = local(checkout, monkeypatch, folder, "-y") + + assert on_github.exit_code == 0, on_github.output + assert body(on_github.stdout) == body(on_disk.stdout)