From dcb2fab095ffa606a6ff015cc507502beb86c6d2 Mon Sep 17 00:00:00 2001 From: nagisml Date: Wed, 18 Nov 2026 21:38:32 +0100 Subject: [PATCH] Publish a macro index with versions, sizes and SHA-256 hashes Add scripts/build_index.py and the Index workflow. After every push to main, the workflow builds index.json from main's git history and publishes it on the "index" branch: https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK-Macros/index/index.json OpenSAK's Macro manager can read this file to list, install and update macros without walking the repository through the GitHub API. Per macro the index holds: - the header, read with OpenSAK's own parser (from .github/opensak-ref) - every file in its folder, with size, SHA-256 and a download URL pinned to the commit - every version that was on main: publish date, SHA-256, size, Requires-API and permissions of the macro file - the changelog from the "## Changes" list in its README Macros removed from main are listed under "withdrawn" with the hashes of all their versions, so an installed copy can be recognised. The script only reads git objects and never runs macro code. The same commit always gives the same file, and a run that changes nothing pushes nothing. MAINTAINERS.md describes the format (schema 1). CONTRIBUTING.md and the template README now ask for a "## Changes" line per version. --- .github/workflows/index.yml | 97 ++++++++++ CONTRIBUTING.md | 13 ++ MAINTAINERS.md | 43 +++++ scripts/build_index.py | 341 ++++++++++++++++++++++++++++++++++++ template/my-macro/README.md | 3 +- tests/test_build_index.py | 256 +++++++++++++++++++++++++++ 6 files changed, 752 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/index.yml create mode 100644 scripts/build_index.py create mode 100644 tests/test_build_index.py diff --git a/.github/workflows/index.yml b/.github/workflows/index.yml new file mode 100644 index 0000000..1facc33 --- /dev/null +++ b/.github/workflows/index.yml @@ -0,0 +1,97 @@ +# Builds index.json — the list of macros with versions, file sizes and +# SHA-256 hashes that OpenSAK's Macro manager reads — after every push to +# main, and publishes it on the "index" branch: +# +# https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK-Macros/index/index.json +# +# The index branch has no history of main: it holds only index.json (and a +# README saying where it comes from), one commit per change. Nobody edits it +# by hand; a run that gives the same file pushes nothing. +# +# Runs scripts/build_index.py from main (only pushes to main trigger it), so +# what lands in the index went through the pull request checks first. The +# script reads git objects only; no macro code is run. + +name: Index + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: {} + +concurrency: + group: index + cancel-in-progress: false + +env: + INDEX_BRANCH: index + PARSER_URL: https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/%s/src/opensak/macro/metadata.py + +jobs: + index: + name: Build and publish index.json + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 0 # the versions come from main's history + persist-credentials: false + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Get OpenSAK header parser + run: | + mkdir -p "$RUNNER_TEMP/tools" + ref=$(grep -v '^#' .github/opensak-ref | head -n 1 | tr -d '[:space:]') + curl -fsSL -o "$RUNNER_TEMP/tools/macro_metadata.py" "$(printf "$PARSER_URL" "$ref")" + + - name: Build index.json + run: | + mkdir -p "$RUNNER_TEMP/out" + python -I scripts/build_index.py --repo . --ref HEAD \ + --metadata "$RUNNER_TEMP/tools/macro_metadata.py" \ + --repository "$GITHUB_REPOSITORY" \ + --output "$RUNNER_TEMP/out/index.json" + + - name: Publish on the index branch + env: + GH_TOKEN: ${{ github.token }} + run: | + out="$RUNNER_TEMP/out" + cat > "$out/README.md" <<'EOF' + # Macro index + + `index.json` lists the macros on `main` with their versions, files, + sizes and SHA-256 hashes, for OpenSAK's Macro manager. It is built by + `.github/workflows/index.yml` after every push to `main`; don't edit + this branch by hand. The format is described in MAINTAINERS.md on + `main`. + EOF + + remote="https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + work="$RUNNER_TEMP/index-branch" + if git ls-remote --exit-code --heads "$remote" "$INDEX_BRANCH" >/dev/null; then + git clone -q --depth 1 --branch "$INDEX_BRANCH" "$remote" "$work" + else + git init -q -b "$INDEX_BRANCH" "$work" + git -C "$work" remote add origin "$remote" + fi + cp "$out/index.json" "$out/README.md" "$work/" + cd "$work" + git add index.json README.md + if git diff --cached --quiet; then + echo "index.json is unchanged." + exit 0 + fi + git -c user.name="github-actions[bot]" \ + -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \ + commit -q -m "Index for ${GITHUB_SHA::7}" + git push -q origin "HEAD:$INDEX_BRANCH" + echo "Published index.json for ${GITHUB_SHA}." >> "$GITHUB_STEP_SUMMARY" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 56a3106..34e2b2d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -138,6 +138,19 @@ maintainer too. Every change to the `.lua` file needs a higher `Version`. A change to only the README or example files doesn't — but if in doubt, bump it. +Add a line for the new version to the `## Changes` section of the README, +starting with the version: + +```markdown +## Changes + +- 1.1 — lists the last 10 runs instead of 5 +- 1.0 — first version +``` + +OpenSAK's Macro manager shows these lines when it offers the update (see +[The macro index](MAINTAINERS.md#the-macro-index)). + ## License By opening a pull request you agree that your contribution is published under diff --git a/MAINTAINERS.md b/MAINTAINERS.md index ae0a249..c96f437 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -71,6 +71,47 @@ The checker also uses two internal helpers of the parser (`_header_lines`, `_KEY_LINE`) to see which keys are actually written. If "Test checker" fails after a bump, look there first. +## The macro index + +After every push to `main`, **Index** (`.github/workflows/index.yml`) runs +`scripts/build_index.py` and publishes `index.json` on the `index` branch: + + https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK-Macros/index/index.json + +OpenSAK's Macro manager reads it to list, install and update macros, so it +doesn't have to walk the repository through the GitHub API. Everything in it +comes from `main`'s git history, so the same commit always gives the same +file; a push that changes nothing in it (e.g. to MAINTAINERS.md) leaves the +`index` branch alone. Run it by hand from the Actions tab ("Run workflow") +after changing the script or `.github/opensak-ref`. + +Locally (with the header parser in `.tools/`, see `tests/`): + + python scripts/build_index.py --metadata .tools/macro_metadata.py --ref main + +Format (`schema` 1; fields are only ever added, a breaking change raises +`schema`): + +| Field | | +|---|---| +| `schema`, `commit`, `generated` | format version; the `main` commit indexed and its date (UTC) | +| `macros[]` | one entry per folder under `macros/` that has `/.lua` | +| `.id`, `.folder`, `.file`, `.readme`, `.page` | folder name, paths in the repository, link to the folder on GitHub | +| `.name`, `.author`, `.version`, `.description`, `.requires_api`, `.permissions`, `.network`, `.headless` | the header, read with OpenSAK's parser (`headless` is null with a parser from before API 4) | +| `.files[]` | every file in the folder: `path`, `size`, `sha256`, and a download `url` pinned to `commit` | +| `.versions[]` | every version that was on `main`, newest first: `version`, `published` (date it first reached `main`), `commit`, `requires_api`, `permissions`, `network`, and `size`, `sha256`, `url` of the macro file | +| `.published`, `.updated` | when the first and the current version reached `main` | +| `.changelog[]` | `version` and `text` from the `## Changes` list in the README, newest first | +| `.error` | only when the header couldn't be read | +| `withdrawn[]` | macros that were on `main` and were removed: `id`, `folder`, `name`, `last_version`, `withdrawn` (date), and `versions[]` with the `sha256` of each, so OpenSAK can recognise an installed copy | + +Hashes are SHA-256 of the bytes in git, which are the bytes the `url`s +serve. A macro is identified by its folder name, so renaming a folder lists +the old name as withdrawn and the new one as a new macro. + +The `index` branch must stay writable by GitHub Actions: don't put it under +the "Protect main" ruleset or any other ruleset that blocks pushes. + ## Repository settings this relies on - Ruleset "Protect main" on the default branch: pull request required (0 @@ -80,3 +121,5 @@ fails after a bump, look there first. - Settings → General → Pull Requests → **Allow auto-merge** on. - Labels `macro problem`, `auto-merge` and `needs review`. - Discussions on. +- The `index` branch is created by the first **Index** run and left + unprotected (see [The macro index](#the-macro-index)). diff --git a/scripts/build_index.py b/scripts/build_index.py new file mode 100644 index 0000000..3117777 --- /dev/null +++ b/scripts/build_index.py @@ -0,0 +1,341 @@ +""" +scripts/build_index.py — builds index.json, the list of macros OpenSAK's +Macro manager reads to browse, install and update macros. + +Run by .github/workflows/index.yml after every push to main; the workflow +publishes the result on the "index" branch: + + https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK-Macros/index/index.json + +Like check_macro_pr.py it only reads git objects (git ls-tree / git show / +git log) and parses headers with OpenSAK's own src/opensak/macro/metadata.py +(--metadata). It never runs Lua code. + +Everything comes from the git history of one commit (--ref, normally main), +so building twice from the same commit gives the same file: + + - macros: every folder under macros/ that has /.lua, with + its header, every file in the folder (size, SHA-256 and a download URL + pinned to the commit, so the bytes behind a URL never change), the + versions it had on main (when each was published, the SHA-256 of the + macro file) and the changelog from the "## Changes" section of its + README. + - withdrawn: folders that were on main once and are not any more, with + their last version and the hashes of all their versions, so OpenSAK can + tell a user that an installed macro was withdrawn. + +Hashes are taken over the bytes git stores, which are the bytes +raw.githubusercontent.com serves (no CRLF conversion). + +A macro whose header can't be read is still listed, with "error" set; the +pull request checks keep that from happening on main. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import hashlib +import importlib.util +import json +import re +import subprocess +import sys +from pathlib import Path, PurePosixPath +from types import ModuleType +from typing import Any + +SCHEMA = 1 +MACROS_DIR = "macros" +DEFAULT_REPOSITORY = "OpenSAK-Org/OpenSAK-Macros" +RAW_URL = "https://raw.githubusercontent.com/{repo}/{commit}/{path}" +PAGE_URL = "https://github.com/{repo}/tree/{ref}/{path}" +VERSION = re.compile(r"^\d+(?:\.\d+)*$") + +# "## Changes" (also "## Changelog", "## Change log", "## History") +_CHANGES_HEADING = re.compile(r"^##\s+(?:changes|change\s*log|history)\s*$", re.I) +# "- 1.1 — text", also with "-", "–", ":" or "(date)" after the version +_CHANGE_ITEM = re.compile( + r"^[-*]\s+v?(\d+(?:\.\d+)*)\b\s*(?:\([^)]*\))?\s*(?:—|–|-|:)?\s*(.*)$") + + +# --- git --------------------------------------------------------------------- + +class Git: + def __init__(self, repo: Path): + self.repo = repo + + def _run(self, *args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run(["git", "-C", str(self.repo), *args], + capture_output=True, check=check) + + def rev_parse(self, rev: str) -> str: + return self._run("rev-parse", "--verify", f"{rev}^{{commit}}" + ).stdout.decode().strip() + + def commit_date(self, rev: str) -> str: + return utc(self._run("show", "-s", "--format=%ct", rev).stdout.decode()) + + def ls_tree(self, rev: str, folder: str) -> list[tuple[str, int]]: + """(path, size) of every file under *folder* at *rev*.""" + out = self._run("ls-tree", "-r", "-z", "--long", rev, "--", + folder + "/").stdout.decode("utf-8") + files = [] + for entry in out.split("\0"): + if not entry: + continue + meta, path = entry.split("\t", 1) + kind, size = meta.split()[1], meta.split()[3] + if kind == "blob": + files.append((path, int(size))) + return sorted(files) + + def read(self, rev: str, path: str) -> bytes | None: + proc = self._run("show", f"{rev}:{path}", check=False) + return proc.stdout if proc.returncode == 0 else None + + def folders_ever(self, rev: str) -> set[str]: + """Names of all folders under macros/ that ever had a file on the + first-parent history of *rev*.""" + out = self._run("log", "--first-parent", "--format=", "--name-only", + "-z", rev, "--", MACROS_DIR + "/").stdout.decode("utf-8") + names = set() + for path in out.replace("\n", "\0").split("\0"): + parts = PurePosixPath(path).parts + if len(parts) >= 3 and parts[0] == MACROS_DIR: + names.add(parts[1]) + return names + + def history(self, rev: str, path: str) -> list[tuple[str, str]]: + """(commit, date) of every commit on the first-parent history of + *rev* that changed *path*, oldest first. On main these are the merge + commits, i.e. when a change was published.""" + out = self._run("log", "--first-parent", "--format=%H %ct", rev, + "--", path).stdout.decode() + found = [] + for line in reversed(out.splitlines()): + if line: + commit, stamp = line.split(" ", 1) + found.append((commit, utc(stamp))) + return found + + +# --- helpers ----------------------------------------------------------------- + +def load_metadata_module(path: Path) -> ModuleType: + spec = importlib.util.spec_from_file_location("macro_metadata", path) + if spec is None or spec.loader is None: + raise SystemExit(f"Cannot load the OpenSAK header parser from {path}") + module = importlib.util.module_from_spec(spec) + sys.modules["macro_metadata"] = module # dataclasses look the module up + spec.loader.exec_module(module) + return module + + +def utc(stamp: str) -> str: + """A git Unix timestamp as ISO 8601 in UTC, e.g. 2026-11-18T12:14:51Z.""" + when = datetime.fromtimestamp(int(stamp.strip()), timezone.utc) + return when.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def version_key(version: str) -> tuple[int, ...]: + """Dot-separated numbers as a comparable tuple; trailing zeros dropped + so that 1.0 and 1.0.0 are the same version.""" + parts = [int(x) for x in version.split(".")] + while len(parts) > 1 and parts[-1] == 0: + parts.pop() + return tuple(parts) + + +def same_version(a: str, b: str) -> bool: + if VERSION.match(a) and VERSION.match(b): + return version_key(a) == version_key(b) + return a == b + + +def header(md: ModuleType, data: bytes | None, path: str) -> dict[str, Any]: + """The header fields OpenSAK's Macro manager shows, or {"error": ...}.""" + if data is None: + return {"error": "missing"} + try: + text = data.decode("utf-8") + except UnicodeDecodeError: + return {"error": "not UTF-8 text"} + info = md.parse_header(text, Path(path)) + api = getattr(info, "requires_api_number", None) + return { + "name": info.name, + "author": info.author, + "version": info.version, + "description": info.description, + "requires_api": api if api is not None else (info.requires_api or None), + "permissions": list(info.permissions) + or ([md.NO_PERMISSIONS] if info.permissions_declared else []), + "network": list(info.network), + # null with a parser from before the Headless key (API 4) + "headless": getattr(info, "runs_headless", None), + } + + +def changelog(readme: bytes | None) -> list[dict[str, str]]: + """The entries of the "## Changes" section of a macro README: one per + list item starting with a version ("- 1.1 — what changed"), newest + first. Indented lines below an item continue it.""" + if not readme: + return [] + try: + text = readme.decode("utf-8") + except UnicodeDecodeError: + return [] + entries: list[dict[str, str]] = [] + inside = False + for raw in text.splitlines(): + line = raw.strip() + if raw.startswith("#"): + inside = bool(_CHANGES_HEADING.match(line)) + continue + if not inside or not line: + continue + m = _CHANGE_ITEM.match(line) + if m: + entries.append({"version": m.group(1), "text": m.group(2).strip()}) + elif entries and raw[:1] in (" ", "\t"): + entries[-1]["text"] = (entries[-1]["text"] + " " + line).strip() + good = [e for e in entries if VERSION.match(e["version"])] + good.sort(key=lambda e: version_key(e["version"]), reverse=True) + return good + + +def versions(git: Git, md: ModuleType, ref: str, main: str, + repository: str) -> list[dict[str, Any]]: + """Every version of the macro file *main* on the history of *ref*, + newest first: when it was first published and the file of its last + commit (a version republished without a bump keeps the newest bytes).""" + found: list[dict[str, Any]] = [] + for commit, date in git.history(ref, main): + data = git.read(commit, main) + if data is None: # the file was removed in this commit + continue + h = header(md, data, main) + version = h.get("version", "") + entry = { + "version": version, + "published": date, + "commit": commit, + "requires_api": h.get("requires_api"), + "permissions": h.get("permissions", []), + "network": h.get("network", []), + "size": len(data), + "sha256": sha256(data), + "url": RAW_URL.format(repo=repository, commit=commit, path=main), + } + if found and same_version(found[-1]["version"], version): + entry["published"] = found[-1]["published"] + found[-1] = entry + else: + found.append(entry) + return list(reversed(found)) + + +# --- the index --------------------------------------------------------------- + +def macro_entry(git: Git, md: ModuleType, commit: str, name: str, + repository: str) -> dict[str, Any] | None: + folder = f"{MACROS_DIR}/{name}" + main = f"{folder}/{name}.lua" + files = git.ls_tree(commit, folder) + if main not in {p for p, _ in files}: + return None + entry: dict[str, Any] = {"id": name, "folder": folder} + entry.update(header(md, git.read(commit, main), main)) + entry["file"] = main + entry["readme"] = f"{folder}/README.md" if any( + p == f"{folder}/README.md" for p, _ in files) else None + entry["page"] = PAGE_URL.format(repo=repository, ref="main", path=folder) + entry["files"] = [ + {"path": p, "size": size, "sha256": sha256(git.read(commit, p) or b""), + "url": RAW_URL.format(repo=repository, commit=commit, path=p)} + for p, size in files + ] + entry["versions"] = versions(git, md, commit, main, repository) + if entry["versions"]: + entry["published"] = entry["versions"][-1]["published"] + entry["updated"] = entry["versions"][0]["published"] + entry["changelog"] = changelog( + git.read(commit, entry["readme"]) if entry["readme"] else None) + return entry + + +def withdrawn_entry(git: Git, md: ModuleType, commit: str, name: str, + repository: str) -> dict[str, Any] | None: + main = f"{MACROS_DIR}/{name}/{name}.lua" + history = versions(git, md, commit, main, repository) + if not history: + return None # never had a macro file (e.g. a typo folder) + removed = git.history(commit, main)[-1] + last = history[0] + return { + "id": name, + "folder": f"{MACROS_DIR}/{name}", + "name": header(md, git.read(last["commit"], main), main).get("name", name), + "last_version": last["version"], + "withdrawn": removed[1], + "versions": [{"version": v["version"], "sha256": v["sha256"]} + for v in history], + } + + +def build(repo: Path, ref: str, md: ModuleType, + repository: str = DEFAULT_REPOSITORY) -> dict[str, Any]: + git = Git(repo) + commit = git.rev_parse(ref) + current = {p.split("/")[1] for p, _ in git.ls_tree(commit, MACROS_DIR) + if len(PurePosixPath(p).parts) >= 3} + macros = [e for name in sorted(current) + if (e := macro_entry(git, md, commit, name, repository))] + listed = {m["id"] for m in macros} + withdrawn = [e for name in sorted(git.folders_ever(commit) - listed) + if (e := withdrawn_entry(git, md, commit, name, repository))] + return { + "schema": SCHEMA, + "repository": f"https://github.com/{repository}", + "commit": commit, + "generated": git.commit_date(commit), + "macros": macros, + "withdrawn": withdrawn, + } + + +def dumps(index: dict[str, Any]) -> str: + return json.dumps(index, indent=2, ensure_ascii=False) + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--repo", type=Path, default=Path(".")) + parser.add_argument("--ref", default="HEAD", help="commit to index (main)") + parser.add_argument("--metadata", type=Path, required=True, + help="path to OpenSAK's src/opensak/macro/metadata.py") + parser.add_argument("--repository", default=DEFAULT_REPOSITORY, + help="owner/name on GitHub, for the download URLs") + parser.add_argument("--output", type=Path, help="write here (default: stdout)") + args = parser.parse_args(argv) + + index = build(args.repo, args.ref, load_metadata_module(args.metadata), + args.repository) + text = dumps(index) + if args.output: + args.output.write_text(text, encoding="utf-8", newline="\n") + print(f"{len(index['macros'])} macro(s), {len(index['withdrawn'])} " + f"withdrawn, from {index['commit']}") + else: + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/template/my-macro/README.md b/template/my-macro/README.md index 9b1e7ef..132301a 100644 --- a/template/my-macro/README.md +++ b/template/my-macro/README.md @@ -26,6 +26,7 @@ Optional: a screenshot (put it in `screenshots/`) or an example file ## Changes -Optional: a short note per version. +Optional: a short note per version, one line each starting with the version. +OpenSAK's Macro manager shows these notes when it offers an update. - 1.0 — first version diff --git a/tests/test_build_index.py b/tests/test_build_index.py new file mode 100644 index 0000000..b7ec828 --- /dev/null +++ b/tests/test_build_index.py @@ -0,0 +1,256 @@ +""" +Tests for scripts/build_index.py. + +They need OpenSAK's header parser, like tests/test_check_macro_pr.py (see +there how to get it locally). Set MACRO_METADATA to use one somewhere else. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) + +import build_index as bi + +METADATA = Path(os.environ.get("MACRO_METADATA", ROOT / ".tools" / "macro_metadata.py")) +if not METADATA.is_file(): + pytest.skip(f"OpenSAK header parser not found at {METADATA} (see " + "tests/test_check_macro_pr.py)", allow_module_level=True) + +MD = bi.load_metadata_module(METADATA) +REPO = "Example/Macros" + + +def macro(version="1.0", permissions="none", network=None, api="3", name="Test macro", + body='opensak.log("hi")\n'): + header = {"Name": name, "Author": "Tester", "Version": version, + "Requires-API": api, "Description": "Does a test.", + "Permissions": permissions} + if network is not None: + header["Network"] = network + lines = ["-- OpenSAK Macro"] + [f"-- {k}: {v}" for k, v in header.items()] + return "\n".join(lines) + "\n\n" + body + + +def readme(*changes: str) -> str: + return "# Test\n\nWhat it does.\n\n## Changes\n\n" + "".join( + f"{c}\n" for c in changes) + "\n## License\n\n- 9.9 — not a change\n" + + +class Repo: + """A repository whose main gets one merge commit per publish(), like + pull requests merged with "Create a merge commit".""" + + def __init__(self, path: Path): + self.path = path + self.n = 0 + self.git("init", "-q", "-b", "main") + self.git("config", "user.email", "t@example.com") + self.git("config", "user.name", "t") + self.git("config", "core.autocrlf", "false") + self.write({"README.md": "repo\n"}) + self.git("add", "-A") + self.git("commit", "-q", "-m", "base") + + def git(self, *args): + return subprocess.run(["git", "-C", str(self.path), *args], check=True, + capture_output=True, text=True).stdout.strip() + + def write(self, files: dict[str, str | bytes | None]): + for name, content in files.items(): + p = self.path / name + if content is None: + p.unlink() + continue + p.parent.mkdir(parents=True, exist_ok=True) + if isinstance(content, bytes): + p.write_bytes(content) + else: + p.write_bytes(content.encode("utf-8")) + + def publish(self, files: dict[str, str | bytes | None]) -> str: + """Commit *files* on a branch and merge it into main with --no-ff; + returns the merge commit.""" + self.n += 1 + when = f"2026-01-{self.n:02d}T12:00:00+01:00" + env = {**os.environ, "GIT_AUTHOR_DATE": when, "GIT_COMMITTER_DATE": when} + self.git("checkout", "-q", "-B", "pr", "main") + self.write(files) + self.git("add", "-A") + subprocess.run(["git", "-C", str(self.path), "commit", "-q", "-m", "pr"], + check=True, capture_output=True, env=env) + self.git("checkout", "-q", "main") + subprocess.run(["git", "-C", str(self.path), "merge", "-q", "--no-ff", + "-m", f"Merge {self.n}", "pr"], + check=True, capture_output=True, env=env) + return self.git("rev-parse", "HEAD") + + def index(self) -> dict: + return bi.build(self.path, "main", MD, REPO) + + +@pytest.fixture +def repo(tmp_path): + return Repo(tmp_path) + + +def by_id(index: dict) -> dict[str, dict]: + return {m["id"]: m for m in index["macros"]} + + +def test_lists_header_files_and_hashes(repo): + lua = macro(permissions="write-files", network="api.example.com") + head = repo.publish({"macros/a/a.lua": lua, "macros/a/README.md": readme(), + "macros/a/screenshots/x.png": b"\x89PNG\r\n\0"}) + index = repo.index() + assert index["schema"] == bi.SCHEMA + assert index["commit"] == head + assert index["repository"] == f"https://github.com/{REPO}" + m = by_id(index)["a"] + assert (m["name"], m["author"], m["version"], m["requires_api"]) == \ + ("Test macro", "Tester", "1.0", 3) + assert m["permissions"] == ["write-files"] + assert m["network"] == ["api.example.com"] + assert m["file"] == "macros/a/a.lua" + assert m["readme"] == "macros/a/README.md" + files = {f["path"]: f for f in m["files"]} + assert set(files) == {"macros/a/a.lua", "macros/a/README.md", + "macros/a/screenshots/x.png"} + png = files["macros/a/screenshots/x.png"] + assert png["size"] == 7 + assert png["sha256"] == hashlib.sha256(b"\x89PNG\r\n\0").hexdigest() + assert png["url"] == (f"https://raw.githubusercontent.com/{REPO}/{head}/" + "macros/a/screenshots/x.png") + + +def test_hash_is_of_the_bytes_in_git(repo): + lua = macro().replace("\n", "\r\n") + repo.publish({"macros/a/a.lua": lua, "macros/a/README.md": readme()}) + f = by_id(repo.index())["a"]["files"] + main = next(x for x in f if x["path"] == "macros/a/a.lua") + assert main["sha256"] == hashlib.sha256(lua.encode()).hexdigest() + assert main["size"] == len(lua.encode()) + + +def test_permissions_none(repo): + repo.publish({"macros/a/a.lua": macro(), "macros/a/README.md": readme()}) + assert by_id(repo.index())["a"]["permissions"] == ["none"] + + +def test_versions_newest_first_with_publish_dates(repo): + first = repo.publish({"macros/a/a.lua": macro("1.0"), "macros/a/README.md": readme()}) + repo.publish({"macros/a/README.md": readme("- 1.0 — first")}) # README only + second = repo.publish({"macros/a/a.lua": macro("1.1", api="4")}) + m = by_id(repo.index())["a"] + assert [v["version"] for v in m["versions"]] == ["1.1", "1.0"] + new, old = m["versions"] + assert new["commit"] == second and old["commit"] == first + assert new["requires_api"] == 4 and old["requires_api"] == 3 + assert old["sha256"] == hashlib.sha256(macro("1.0").encode()).hexdigest() + assert old["url"].endswith(f"/{first}/macros/a/a.lua") + assert old["published"] == "2026-01-01T11:00:00Z" + assert new["published"] == "2026-01-03T11:00:00Z" + assert (m["published"], m["updated"]) == (old["published"], new["published"]) + + +def test_same_version_republished_keeps_first_date_and_newest_bytes(repo): + repo.publish({"macros/a/a.lua": macro("1.0"), "macros/a/README.md": readme()}) + fix = macro("1.0.0", body="-- fixed\n") + last = repo.publish({"macros/a/a.lua": fix}) + (only,) = by_id(repo.index())["a"]["versions"] + assert only["commit"] == last + assert only["sha256"] == hashlib.sha256(fix.encode()).hexdigest() + assert only["published"] == "2026-01-01T11:00:00Z" + + +def test_changelog_from_readme(repo): + text = readme("- 1.0 — first version", + "- 1.10 – longer list", + "\t\tand a second line", + "- 1.2: fix", + "- not a version", + "* v2.0 (2026-02-01) - big one") + repo.publish({"macros/a/a.lua": macro(), "macros/a/README.md": text}) + assert by_id(repo.index())["a"]["changelog"] == [ + {"version": "2.0", "text": "big one"}, + {"version": "1.10", "text": "longer list and a second line"}, + {"version": "1.2", "text": "fix"}, + {"version": "1.0", "text": "first version"}, + ] + + +def test_no_changes_section(repo): + repo.publish({"macros/a/a.lua": macro(), "macros/a/README.md": "# A\n\n- 1.0 — x\n"}) + assert by_id(repo.index())["a"]["changelog"] == [] + + +def test_folder_without_macro_file_is_skipped(repo): + repo.publish({"macros/a/other.lua": macro(), "macros/a/README.md": readme()}) + index = repo.index() + assert index["macros"] == [] and index["withdrawn"] == [] + + +def test_withdrawn_macro(repo): + repo.publish({"macros/a/a.lua": macro("1.0", name="Old one"), + "macros/a/README.md": readme()}) + repo.publish({"macros/a/a.lua": macro("1.1", name="Old one")}) + repo.publish({"macros/b/b.lua": macro(), "macros/b/README.md": readme()}) + repo.publish({"macros/a/a.lua": None, "macros/a/README.md": None}) + index = repo.index() + assert [m["id"] for m in index["macros"]] == ["b"] + (w,) = index["withdrawn"] + assert (w["id"], w["name"], w["last_version"]) == ("a", "Old one", "1.1") + assert w["withdrawn"] == "2026-01-04T11:00:00Z" + assert [v["version"] for v in w["versions"]] == ["1.1", "1.0"] + assert w["versions"][1]["sha256"] == hashlib.sha256( + macro("1.0", name="Old one").encode()).hexdigest() + + +def test_macro_back_after_withdrawal_is_listed_not_withdrawn(repo): + repo.publish({"macros/a/a.lua": macro("1.0"), "macros/a/README.md": readme()}) + repo.publish({"macros/a/a.lua": None, "macros/a/README.md": None}) + repo.publish({"macros/a/a.lua": macro("2.0"), "macros/a/README.md": readme()}) + index = repo.index() + assert index["withdrawn"] == [] + assert [v["version"] for v in by_id(index)["a"]["versions"]] == ["2.0", "1.0"] + + +def test_unreadable_header_is_listed_with_error(repo): + repo.publish({"macros/a/a.lua": b"\xff\xfe-- broken\n", "macros/a/README.md": readme()}) + assert by_id(repo.index())["a"]["error"] == "not UTF-8 text" + + +def test_same_commit_gives_same_file(repo, tmp_path): + repo.publish({"macros/a/a.lua": macro(), "macros/a/README.md": readme()}) + out1, out2 = tmp_path / "1.json", tmp_path / "2.json" + for out in (out1, out2): + assert bi.main(["--repo", str(repo.path), "--ref", "main", "--metadata", + str(METADATA), "--repository", REPO, + "--output", str(out)]) == 0 + assert out1.read_bytes() == out2.read_bytes() + assert json.loads(out1.read_text(encoding="utf-8"))["macros"][0]["id"] == "a" + + +def test_this_repository(tmp_path): + """The real repository builds, and every listed file hashes to what git + stores.""" + if not (ROOT / ".git").exists(): + pytest.skip("not a git checkout") + index = bi.build(ROOT, "HEAD", MD) + git = bi.Git(ROOT) + for m in index["macros"]: + assert "error" not in m, m["id"] + assert m["versions"], m["id"] + for f in m["files"]: + data = git.read(index["commit"], f["path"]) + assert data is not None + assert hashlib.sha256(data).hexdigest() == f["sha256"]