diff --git a/.github/opensak-ref b/.github/opensak-ref index e330fa6..61a0b64 100644 --- a/.github/opensak-ref +++ b/.github/opensak-ref @@ -1,5 +1,5 @@ -# The OpenSAK commit whose src/opensak/macro/metadata.py the checks use to -# parse macro headers. Change it deliberately when the header rules change +# The OpenSAK commit whose src/opensak/macro/metadata.py (and the +# deprecation.py it imports) the checks use to parse macro headers. Change it deliberately when the header rules change # in OpenSAK (see MAINTAINERS.md). The first line without "#" is used. -# OpenSAK v1.22.0-beta.9 -7f2bf6daafb91aee9357c9b5363655af8424a4fc +# OpenSAK v1.22.0 +2f05c474b1dd7e688919c8f3070b8e2dec2b8be3 diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index 9ce4e14..6b2090c 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -70,16 +70,21 @@ jobs: run: | pip install "lupa>=2.0,<3" ref=$(grep -v '^#' .github/opensak-ref | head -n 1 | tr -d '[:space:]') - mkdir -p "$RUNNER_TEMP/tools" - curl -fsSL -o "$RUNNER_TEMP/tools/macro_metadata.py" \ - "https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/${ref}/src/opensak/macro/metadata.py" + dir="$RUNNER_TEMP/opensak-src/opensak/macro" + url="https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/${ref}/src/opensak/macro" + mkdir -p "$dir" + curl -fsSL -o "$dir/metadata.py" "$url/metadata.py" + # metadata.py's imports, as PARSER_DEPS in check.yml + for f in deprecation.py; do + curl -fsSL -o "$dir/$f" "$url/$f" || echo "::notice::No $f at $ref (not needed there)" + done - name: Check the pull request if: steps.fetch.outputs.current == 'true' run: | python -I scripts/check_macro_pr.py \ --repo . --base "$(git rev-parse HEAD)" --head "$HEAD_SHA" \ - --metadata "$RUNNER_TEMP/tools/macro_metadata.py" \ + --metadata "$RUNNER_TEMP/opensak-src/opensak/macro/metadata.py" \ --json "$RUNNER_TEMP/check.json" || true - name: Decide diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index bfc51d4..70195cb 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -10,8 +10,9 @@ # "Test checker" runs the tests with the pull request's version of the # scripts, for pull requests that change them. # -# The header parser is OpenSAK's src/opensak/macro/metadata.py, downloaded -# from the commit in .github/opensak-ref. +# The header parser is OpenSAK's src/opensak/macro/metadata.py (with the +# deprecation.py it imports), downloaded from the commit in +# .github/opensak-ref into an opensak/macro/ folder. name: Check @@ -28,7 +29,11 @@ concurrency: env: LUPA: "lupa>=2.0,<3" - PARSER_URL: https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/%s/src/opensak/macro/metadata.py + PARSER_URL: https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/%s/src/opensak/macro + # The modules metadata.py imports (deprecation.py from v1.22.0). An older + # ref doesn't have them and doesn't need them; if one is missing that is + # needed, loading the parser fails with ModuleNotFoundError. + PARSER_DEPS: deprecation.py jobs: check-macro: @@ -55,7 +60,13 @@ jobs: from_main() { git show "$BASE:$1" 2>/dev/null || cat "$1"; } from_main scripts/check_macro_pr.py > "$RUNNER_TEMP/tools/check_macro_pr.py" ref=$(from_main .github/opensak-ref | grep -v '^#' | head -n 1 | tr -d '[:space:]') - curl -fsSL -o "$RUNNER_TEMP/tools/macro_metadata.py" "$(printf "$PARSER_URL" "$ref")" + dir="$RUNNER_TEMP/opensak-src/opensak/macro" + url="$(printf "$PARSER_URL" "$ref")" + mkdir -p "$dir" + curl -fsSL -o "$dir/metadata.py" "$url/metadata.py" + for f in $PARSER_DEPS; do + curl -fsSL -o "$dir/$f" "$url/$f" || echo "::notice::No $f at $ref (not needed there)" + done - name: Check macro env: @@ -64,7 +75,7 @@ jobs: run: | python -I "$RUNNER_TEMP/tools/check_macro_pr.py" \ --repo . --base "$BASE" --head "$HEAD" \ - --metadata "$RUNNER_TEMP/tools/macro_metadata.py" + --metadata "$RUNNER_TEMP/opensak-src/opensak/macro/metadata.py" test-checker: name: Test checker @@ -83,9 +94,14 @@ jobs: - name: Get OpenSAK header parser run: | - mkdir -p .tools ref=$(grep -v '^#' .github/opensak-ref | head -n 1 | tr -d '[:space:]') - curl -fsSL -o .tools/macro_metadata.py "$(printf "$PARSER_URL" "$ref")" + dir=".tools/opensak/macro" + url="$(printf "$PARSER_URL" "$ref")" + mkdir -p "$dir" + curl -fsSL -o "$dir/metadata.py" "$url/metadata.py" + for f in $PARSER_DEPS; do + curl -fsSL -o "$dir/$f" "$url/$f" || echo "::notice::No $f at $ref (not needed there)" + done - name: Run tests run: pytest -v tests/ diff --git a/.github/workflows/index.yml b/.github/workflows/index.yml new file mode 100644 index 0000000..f90e18c --- /dev/null +++ b/.github/workflows/index.yml @@ -0,0 +1,106 @@ +# Builds index.json — the list of macros with versions, file sizes and +# SHA-256 hashes that OpenSAK's Macro manager reads — after every push to +# main, and publishes it on the "index" branch: +# +# https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK-Macros/index/index.json +# +# The index branch has no history of main: it holds only index.json (and a +# README saying where it comes from), one commit per change. Nobody edits it +# by hand; a run that gives the same file pushes nothing. +# +# Runs scripts/build_index.py from main (only pushes to main trigger it), so +# what lands in the index went through the pull request checks first. The +# script reads git objects only; no macro code is run. + +name: Index + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: {} + +concurrency: + group: index + cancel-in-progress: false + +env: + INDEX_BRANCH: index + PARSER_URL: https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/%s/src/opensak/macro + # The modules metadata.py imports (deprecation.py from v1.22.0). An older + # ref doesn't have them and doesn't need them; if one is missing that is + # needed, loading the parser fails with ModuleNotFoundError. + PARSER_DEPS: deprecation.py + +jobs: + index: + name: Build and publish index.json + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 0 # the versions come from main's history + persist-credentials: false + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Get OpenSAK header parser + run: | + ref=$(grep -v '^#' .github/opensak-ref | head -n 1 | tr -d '[:space:]') + dir="$RUNNER_TEMP/opensak-src/opensak/macro" + url="$(printf "$PARSER_URL" "$ref")" + mkdir -p "$dir" + curl -fsSL -o "$dir/metadata.py" "$url/metadata.py" + for f in $PARSER_DEPS; do + curl -fsSL -o "$dir/$f" "$url/$f" || echo "::notice::No $f at $ref (not needed there)" + done + + - name: Build index.json + run: | + mkdir -p "$RUNNER_TEMP/out" + python -I scripts/build_index.py --repo . --ref HEAD \ + --metadata "$RUNNER_TEMP/opensak-src/opensak/macro/metadata.py" \ + --repository "$GITHUB_REPOSITORY" \ + --output "$RUNNER_TEMP/out/index.json" + + - name: Publish on the index branch + env: + GH_TOKEN: ${{ github.token }} + run: | + out="$RUNNER_TEMP/out" + cat > "$out/README.md" <<'EOF' + # Macro index + + `index.json` lists the macros on `main` with their versions, files, + sizes and SHA-256 hashes, for OpenSAK's Macro manager. It is built by + `.github/workflows/index.yml` after every push to `main`; don't edit + this branch by hand. The format is described in MAINTAINERS.md on + `main`. + EOF + + remote="https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + work="$RUNNER_TEMP/index-branch" + if git ls-remote --exit-code --heads "$remote" "$INDEX_BRANCH" >/dev/null; then + git clone -q --depth 1 --branch "$INDEX_BRANCH" "$remote" "$work" + else + git init -q -b "$INDEX_BRANCH" "$work" + git -C "$work" remote add origin "$remote" + fi + cp "$out/index.json" "$out/README.md" "$work/" + cd "$work" + git add index.json README.md + if git diff --cached --quiet; then + echo "index.json is unchanged." + exit 0 + fi + git -c user.name="github-actions[bot]" \ + -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \ + commit -q -m "Index for ${GITHUB_SHA::7}" + git push -q origin "HEAD:$INDEX_BRANCH" + echo "Published index.json for ${GITHUB_SHA}." >> "$GITHUB_STEP_SUMMARY" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 56a3106..a9ccde0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,8 +64,10 @@ Declare everything the macro can do, not just what it usually does. List plain host names (or IP addresses): `api.example.com`, not `https://api.example.com/path`, no ports and no wildcards. -`geocaching.com`, `coord.info` and their subdomains are not allowed — the -Geocaching.com terms of use don't permit automated access outside their API, +Geocaching HQ's sites — `geocaching.com`, `coord.info`, `groundspeak.com`, +`waymarking.com`, `wherigo.com` and related domains — and their +subdomains are not allowed: the Geocaching.com terms of use don't permit +automated access outside their API, so OpenSAK will never let a macro contact them. ## Automatic checks @@ -138,6 +140,19 @@ maintainer too. Every change to the `.lua` file needs a higher `Version`. A change to only the README or example files doesn't — but if in doubt, bump it. +Add a line for the new version to the `## Changes` section of the README, +starting with the version: + +```markdown +## Changes + +- 1.1 — lists the last 10 runs instead of 5 +- 1.0 — first version +``` + +OpenSAK's Macro manager shows these lines when it offers the update (see +[The macro index](MAINTAINERS.md#the-macro-index)). + ## License By opening a pull request you agree that your contribution is published under diff --git a/MAINTAINERS.md b/MAINTAINERS.md index ae0a249..3f950c2 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -57,7 +57,9 @@ the README explains why. ## Changing the header rules The checks parse headers with OpenSAK's `src/opensak/macro/metadata.py` from -the commit in `.github/opensak-ref`. When OpenSAK changes the header rules +the commit in `.github/opensak-ref`, together with the `deprecation.py` it +imports (from v1.22.0; OpenSAK keeps that module free of other OpenSAK +imports for this repository's sake). When OpenSAK changes the header rules (new permission words, new keys, new blocked hosts): 1. Put the new OpenSAK commit (ideally a tagged release) in @@ -69,7 +71,51 @@ the commit in `.github/opensak-ref`. When OpenSAK changes the header rules The checker also uses two internal helpers of the parser (`_header_lines`, `_KEY_LINE`) to see which keys are actually written. If "Test checker" -fails after a bump, look there first. +fails after a bump, look there first. If it fails with `ModuleNotFoundError`, +`metadata.py` has gained another top-level import: add that file to +`PARSER_DEPS` in `check.yml` and `index.yml`, to the list in +`auto-merge.yml` and to the download in `tests/test_check_macro_pr.py`. + +## The macro index + +After every push to `main`, **Index** (`.github/workflows/index.yml`) runs +`scripts/build_index.py` and publishes `index.json` on the `index` branch: + + https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK-Macros/index/index.json + +OpenSAK's Macro manager reads it to list, install and update macros, so it +doesn't have to walk the repository through the GitHub API. Everything in it +comes from `main`'s git history, so the same commit always gives the same +file; a push that changes nothing in it (e.g. to MAINTAINERS.md) leaves the +`index` branch alone. Run it by hand from the Actions tab ("Run workflow") +after changing the script or `.github/opensak-ref`. + +Locally (with the header parser in `.tools/`, see `tests/`): + + python scripts/build_index.py --metadata .tools/opensak/macro/metadata.py --ref main + +Format (`schema` 1; fields are only ever added, a breaking change raises +`schema`): + +| Field | | +|---|---| +| `schema`, `commit`, `generated` | format version; the `main` commit indexed and its date (UTC) | +| `macros[]` | one entry per folder under `macros/` that has `/.lua` | +| `.id`, `.folder`, `.file`, `.readme`, `.page` | folder name, paths in the repository, link to the folder on GitHub | +| `.name`, `.author`, `.version`, `.description`, `.requires_api`, `.permissions`, `.network`, `.headless` | the header, read with OpenSAK's parser (`headless` stays null until OpenSAK's parser has a Headless key; v1.22.0 doesn't) | +| `.files[]` | every file in the folder: `path`, `size`, `sha256`, and a download `url` pinned to `commit` | +| `.versions[]` | every version that was on `main`, newest first: `version`, `published` (date it first reached `main`), `commit`, `requires_api`, `permissions`, `network`, and `size`, `sha256`, `url` of the macro file | +| `.published`, `.updated` | when the first and the current version reached `main` | +| `.changelog[]` | `version` and `text` from the `## Changes` list in the README, newest first | +| `.error` | only when the header couldn't be read | +| `withdrawn[]` | macros that were on `main` and were removed: `id`, `folder`, `name`, `last_version`, `withdrawn` (date), and `versions[]` with the `sha256` of each, so OpenSAK can recognise an installed copy | + +Hashes are SHA-256 of the bytes in git, which are the bytes the `url`s +serve. A macro is identified by its folder name, so renaming a folder lists +the old name as withdrawn and the new one as a new macro. + +The `index` branch must stay writable by GitHub Actions: don't put it under +the "Protect main" ruleset or any other ruleset that blocks pushes. ## Repository settings this relies on @@ -80,3 +126,5 @@ fails after a bump, look there first. - Settings → General → Pull Requests → **Allow auto-merge** on. - Labels `macro problem`, `auto-merge` and `needs review`. - Discussions on. +- The `index` branch is created by the first **Index** run and left + unprotected (see [The macro index](#the-macro-index)). diff --git a/docs/github-website-guide.md b/docs/github-website-guide.md index 322284d..df0f503 100644 --- a/docs/github-website-guide.md +++ b/docs/github-website-guide.md @@ -48,8 +48,9 @@ the GitHub website — no git, no command line. The rules themselves are in `read-files`, `write-files`, `open`, `run-macros`). If it only reads data, write `none`. - **Network:** add this line only if the macro contacts web sites. Give - plain host names such as `api.example.com`. `geocaching.com` and - `coord.info` are not allowed. + plain host names such as `api.example.com`. Geocaching HQ's sites + (`geocaching.com`, `coord.info`, `waymarking.com`, `wherigo.com` and + related domains) are not allowed. 4. Write the `README.md`: what the macro does, how to use it, and what it needs. diff --git a/scripts/build_index.py b/scripts/build_index.py new file mode 100644 index 0000000..c8ca764 --- /dev/null +++ b/scripts/build_index.py @@ -0,0 +1,356 @@ +""" +scripts/build_index.py — builds index.json, the list of macros OpenSAK's +Macro manager reads to browse, install and update macros. + +Run by .github/workflows/index.yml after every push to main; the workflow +publishes the result on the "index" branch: + + https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK-Macros/index/index.json + +Like check_macro_pr.py it only reads git objects (git ls-tree / git show / +git log) and parses headers with OpenSAK's own src/opensak/macro/metadata.py +(--metadata). It never runs Lua code. + +Everything comes from the git history of one commit (--ref, normally main), +so building twice from the same commit gives the same file: + + - macros: every folder under macros/ that has /.lua, with + its header, every file in the folder (size, SHA-256 and a download URL + pinned to the commit, so the bytes behind a URL never change), the + versions it had on main (when each was published, the SHA-256 of the + macro file) and the changelog from the "## Changes" section of its + README. + - withdrawn: folders that were on main once and are not any more, with + their last version and the hashes of all their versions, so OpenSAK can + tell a user that an installed macro was withdrawn. + +Hashes are taken over the bytes git stores, which are the bytes +raw.githubusercontent.com serves (no CRLF conversion). + +A macro whose header can't be read is still listed, with "error" set; the +pull request checks keep that from happening on main. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import hashlib +import importlib.util +import json +import re +import subprocess +import sys +from pathlib import Path, PurePosixPath +from types import ModuleType +from typing import Any + +SCHEMA = 1 +MACROS_DIR = "macros" +DEFAULT_REPOSITORY = "OpenSAK-Org/OpenSAK-Macros" +RAW_URL = "https://raw.githubusercontent.com/{repo}/{commit}/{path}" +PAGE_URL = "https://github.com/{repo}/tree/{ref}/{path}" +VERSION = re.compile(r"^\d+(?:\.\d+)*$") + +# "## Changes" (also "## Changelog", "## Change log", "## History") +_CHANGES_HEADING = re.compile(r"^##\s+(?:changes|change\s*log|history)\s*$", re.I) +# "- 1.1 — text", also with "-", "–", ":" or "(date)" after the version +_CHANGE_ITEM = re.compile( + r"^[-*]\s+v?(\d+(?:\.\d+)*)\b\s*(?:\([^)]*\))?\s*(?:—|–|-|:)?\s*(.*)$") + + +# --- git --------------------------------------------------------------------- + +class Git: + def __init__(self, repo: Path): + self.repo = repo + + def _run(self, *args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run(["git", "-C", str(self.repo), *args], + capture_output=True, check=check) + + def rev_parse(self, rev: str) -> str: + return self._run("rev-parse", "--verify", f"{rev}^{{commit}}" + ).stdout.decode().strip() + + def commit_date(self, rev: str) -> str: + return utc(self._run("show", "-s", "--format=%ct", rev).stdout.decode()) + + def ls_tree(self, rev: str, folder: str) -> list[tuple[str, int]]: + """(path, size) of every file under *folder* at *rev*.""" + out = self._run("ls-tree", "-r", "-z", "--long", rev, "--", + folder + "/").stdout.decode("utf-8") + files = [] + for entry in out.split("\0"): + if not entry: + continue + meta, path = entry.split("\t", 1) + kind, size = meta.split()[1], meta.split()[3] + if kind == "blob": + files.append((path, int(size))) + return sorted(files) + + def read(self, rev: str, path: str) -> bytes | None: + proc = self._run("show", f"{rev}:{path}", check=False) + return proc.stdout if proc.returncode == 0 else None + + def folders_ever(self, rev: str) -> set[str]: + """Names of all folders under macros/ that ever had a file on the + first-parent history of *rev*.""" + out = self._run("log", "--first-parent", "--format=", "--name-only", + "-z", rev, "--", MACROS_DIR + "/").stdout.decode("utf-8") + names = set() + for path in out.replace("\n", "\0").split("\0"): + parts = PurePosixPath(path).parts + if len(parts) >= 3 and parts[0] == MACROS_DIR: + names.add(parts[1]) + return names + + def history(self, rev: str, path: str) -> list[tuple[str, str]]: + """(commit, date) of every commit on the first-parent history of + *rev* that changed *path*, oldest first. On main these are the merge + commits, i.e. when a change was published.""" + out = self._run("log", "--first-parent", "--format=%H %ct", rev, + "--", path).stdout.decode() + found = [] + for line in reversed(out.splitlines()): + if line: + commit, stamp = line.split(" ", 1) + found.append((commit, utc(stamp))) + return found + + +# --- helpers ----------------------------------------------------------------- + +def load_metadata_module(path: Path) -> ModuleType: + """Load OpenSAK's metadata.py from *path*. + + From v1.22.0 on it imports opensak.macro.deprecation, which has no + OpenSAK imports of its own, so the two files work outside OpenSAK as a + pair. With *path* at /opensak/macro/metadata.py and deprecation.py + next to it (as the workflows download them), is put first on + sys.path so that import finds it. A parser in any other place (an older + one, which needs nothing else) is loaded as it is. + """ + path = path.resolve() + if path.parent.name == "macro" and path.parent.parent.name == "opensak": + src = str(path.parents[2]) + if src not in sys.path: + sys.path.insert(0, src) + spec = importlib.util.spec_from_file_location("macro_metadata", path) + if spec is None or spec.loader is None: + raise SystemExit(f"Cannot load the OpenSAK header parser from {path}") + module = importlib.util.module_from_spec(spec) + sys.modules["macro_metadata"] = module # dataclasses look the module up + spec.loader.exec_module(module) + return module + + +def utc(stamp: str) -> str: + """A git Unix timestamp as ISO 8601 in UTC, e.g. 2026-11-18T12:14:51Z.""" + when = datetime.fromtimestamp(int(stamp.strip()), timezone.utc) + return when.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def version_key(version: str) -> tuple[int, ...]: + """Dot-separated numbers as a comparable tuple; trailing zeros dropped + so that 1.0 and 1.0.0 are the same version.""" + parts = [int(x) for x in version.split(".")] + while len(parts) > 1 and parts[-1] == 0: + parts.pop() + return tuple(parts) + + +def same_version(a: str, b: str) -> bool: + if VERSION.match(a) and VERSION.match(b): + return version_key(a) == version_key(b) + return a == b + + +def header(md: ModuleType, data: bytes | None, path: str) -> dict[str, Any]: + """The header fields OpenSAK's Macro manager shows, or {"error": ...}.""" + if data is None: + return {"error": "missing"} + try: + text = data.decode("utf-8") + except UnicodeDecodeError: + return {"error": "not UTF-8 text"} + info = md.parse_header(text, Path(path)) + api = getattr(info, "requires_api_number", None) + return { + "name": info.name, + "author": info.author, + "version": info.version, + "description": info.description, + "requires_api": api if api is not None else (info.requires_api or None), + "permissions": list(info.permissions) + or ([md.NO_PERMISSIONS] if info.permissions_declared else []), + "network": list(info.network), + # null until OpenSAK's parser has the Headless key (planned, OpenSAK + # #1195; v1.22.0 doesn't have it yet) + "headless": getattr(info, "runs_headless", None), + } + + +def changelog(readme: bytes | None) -> list[dict[str, str]]: + """The entries of the "## Changes" section of a macro README: one per + list item starting with a version ("- 1.1 — what changed"), newest + first. Indented lines below an item continue it.""" + if not readme: + return [] + try: + text = readme.decode("utf-8") + except UnicodeDecodeError: + return [] + entries: list[dict[str, str]] = [] + inside = False + for raw in text.splitlines(): + line = raw.strip() + if raw.startswith("#"): + inside = bool(_CHANGES_HEADING.match(line)) + continue + if not inside or not line: + continue + m = _CHANGE_ITEM.match(line) + if m: + entries.append({"version": m.group(1), "text": m.group(2).strip()}) + elif entries and raw[:1] in (" ", "\t"): + entries[-1]["text"] = (entries[-1]["text"] + " " + line).strip() + good = [e for e in entries if VERSION.match(e["version"])] + good.sort(key=lambda e: version_key(e["version"]), reverse=True) + return good + + +def versions(git: Git, md: ModuleType, ref: str, main: str, + repository: str) -> list[dict[str, Any]]: + """Every version of the macro file *main* on the history of *ref*, + newest first: when it was first published and the file of its last + commit (a version republished without a bump keeps the newest bytes).""" + found: list[dict[str, Any]] = [] + for commit, date in git.history(ref, main): + data = git.read(commit, main) + if data is None: # the file was removed in this commit + continue + h = header(md, data, main) + version = h.get("version", "") + entry = { + "version": version, + "published": date, + "commit": commit, + "requires_api": h.get("requires_api"), + "permissions": h.get("permissions", []), + "network": h.get("network", []), + "size": len(data), + "sha256": sha256(data), + "url": RAW_URL.format(repo=repository, commit=commit, path=main), + } + if found and same_version(found[-1]["version"], version): + entry["published"] = found[-1]["published"] + found[-1] = entry + else: + found.append(entry) + return list(reversed(found)) + + +# --- the index --------------------------------------------------------------- + +def macro_entry(git: Git, md: ModuleType, commit: str, name: str, + repository: str) -> dict[str, Any] | None: + folder = f"{MACROS_DIR}/{name}" + main = f"{folder}/{name}.lua" + files = git.ls_tree(commit, folder) + if main not in {p for p, _ in files}: + return None + entry: dict[str, Any] = {"id": name, "folder": folder} + entry.update(header(md, git.read(commit, main), main)) + entry["file"] = main + entry["readme"] = f"{folder}/README.md" if any( + p == f"{folder}/README.md" for p, _ in files) else None + entry["page"] = PAGE_URL.format(repo=repository, ref="main", path=folder) + entry["files"] = [ + {"path": p, "size": size, "sha256": sha256(git.read(commit, p) or b""), + "url": RAW_URL.format(repo=repository, commit=commit, path=p)} + for p, size in files + ] + entry["versions"] = versions(git, md, commit, main, repository) + if entry["versions"]: + entry["published"] = entry["versions"][-1]["published"] + entry["updated"] = entry["versions"][0]["published"] + entry["changelog"] = changelog( + git.read(commit, entry["readme"]) if entry["readme"] else None) + return entry + + +def withdrawn_entry(git: Git, md: ModuleType, commit: str, name: str, + repository: str) -> dict[str, Any] | None: + main = f"{MACROS_DIR}/{name}/{name}.lua" + history = versions(git, md, commit, main, repository) + if not history: + return None # never had a macro file (e.g. a typo folder) + removed = git.history(commit, main)[-1] + last = history[0] + return { + "id": name, + "folder": f"{MACROS_DIR}/{name}", + "name": header(md, git.read(last["commit"], main), main).get("name", name), + "last_version": last["version"], + "withdrawn": removed[1], + "versions": [{"version": v["version"], "sha256": v["sha256"]} + for v in history], + } + + +def build(repo: Path, ref: str, md: ModuleType, + repository: str = DEFAULT_REPOSITORY) -> dict[str, Any]: + git = Git(repo) + commit = git.rev_parse(ref) + current = {p.split("/")[1] for p, _ in git.ls_tree(commit, MACROS_DIR) + if len(PurePosixPath(p).parts) >= 3} + macros = [e for name in sorted(current) + if (e := macro_entry(git, md, commit, name, repository))] + listed = {m["id"] for m in macros} + withdrawn = [e for name in sorted(git.folders_ever(commit) - listed) + if (e := withdrawn_entry(git, md, commit, name, repository))] + return { + "schema": SCHEMA, + "repository": f"https://github.com/{repository}", + "commit": commit, + "generated": git.commit_date(commit), + "macros": macros, + "withdrawn": withdrawn, + } + + +def dumps(index: dict[str, Any]) -> str: + return json.dumps(index, indent=2, ensure_ascii=False) + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--repo", type=Path, default=Path(".")) + parser.add_argument("--ref", default="HEAD", help="commit to index (main)") + parser.add_argument("--metadata", type=Path, required=True, + help="path to OpenSAK's src/opensak/macro/metadata.py") + parser.add_argument("--repository", default=DEFAULT_REPOSITORY, + help="owner/name on GitHub, for the download URLs") + parser.add_argument("--output", type=Path, help="write here (default: stdout)") + args = parser.parse_args(argv) + + index = build(args.repo, args.ref, load_metadata_module(args.metadata), + args.repository) + text = dumps(index) + if args.output: + args.output.write_text(text, encoding="utf-8", newline="\n") + print(f"{len(index['macros'])} macro(s), {len(index['withdrawn'])} " + f"withdrawn, from {index['commit']}") + else: + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_macro_pr.py b/scripts/check_macro_pr.py index ba8441d..d22fa7a 100644 --- a/scripts/check_macro_pr.py +++ b/scripts/check_macro_pr.py @@ -120,6 +120,20 @@ def read(self, rev: str, path: str) -> bytes | None: # --- helpers ----------------------------------------------------------------- def load_metadata_module(path: Path) -> ModuleType: + """Load OpenSAK's metadata.py from *path*. + + From v1.22.0 on it imports opensak.macro.deprecation, which has no + OpenSAK imports of its own, so the two files work outside OpenSAK as a + pair. With *path* at /opensak/macro/metadata.py and deprecation.py + next to it (as the workflows download them), is put first on + sys.path so that import finds it. A parser in any other place (an older + one, which needs nothing else) is loaded as it is. + """ + path = path.resolve() + if path.parent.name == "macro" and path.parent.parent.name == "opensak": + src = str(path.parents[2]) + if src not in sys.path: + sys.path.insert(0, src) spec = importlib.util.spec_from_file_location("macro_metadata", path) if spec is None or spec.loader is None: raise SystemExit(f"Cannot load the OpenSAK header parser from {path}") diff --git a/template/my-macro/README.md b/template/my-macro/README.md index 9b1e7ef..132301a 100644 --- a/template/my-macro/README.md +++ b/template/my-macro/README.md @@ -26,6 +26,7 @@ Optional: a screenshot (put it in `screenshots/`) or an example file ## Changes -Optional: a short note per version. +Optional: a short note per version, one line each starting with the version. +OpenSAK's Macro manager shows these notes when it offers an update. - 1.0 — first version diff --git a/tests/test_build_index.py b/tests/test_build_index.py new file mode 100644 index 0000000..1f5fe46 --- /dev/null +++ b/tests/test_build_index.py @@ -0,0 +1,256 @@ +""" +Tests for scripts/build_index.py. + +They need OpenSAK's header parser, like tests/test_check_macro_pr.py (see +there how to get it locally). Set MACRO_METADATA to use one somewhere else. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) + +import build_index as bi + +METADATA = Path(os.environ.get("MACRO_METADATA", ROOT / ".tools" / "opensak" / "macro" / "metadata.py")) +if not METADATA.is_file(): + pytest.skip(f"OpenSAK header parser not found at {METADATA} (see " + "tests/test_check_macro_pr.py)", allow_module_level=True) + +MD = bi.load_metadata_module(METADATA) +REPO = "Example/Macros" + + +def macro(version="1.0", permissions="none", network=None, api="3", name="Test macro", + body='opensak.log("hi")\n'): + header = {"Name": name, "Author": "Tester", "Version": version, + "Requires-API": api, "Description": "Does a test.", + "Permissions": permissions} + if network is not None: + header["Network"] = network + lines = ["-- OpenSAK Macro"] + [f"-- {k}: {v}" for k, v in header.items()] + return "\n".join(lines) + "\n\n" + body + + +def readme(*changes: str) -> str: + return "# Test\n\nWhat it does.\n\n## Changes\n\n" + "".join( + f"{c}\n" for c in changes) + "\n## License\n\n- 9.9 — not a change\n" + + +class Repo: + """A repository whose main gets one merge commit per publish(), like + pull requests merged with "Create a merge commit".""" + + def __init__(self, path: Path): + self.path = path + self.n = 0 + self.git("init", "-q", "-b", "main") + self.git("config", "user.email", "t@example.com") + self.git("config", "user.name", "t") + self.git("config", "core.autocrlf", "false") + self.write({"README.md": "repo\n"}) + self.git("add", "-A") + self.git("commit", "-q", "-m", "base") + + def git(self, *args): + return subprocess.run(["git", "-C", str(self.path), *args], check=True, + capture_output=True, text=True).stdout.strip() + + def write(self, files: dict[str, str | bytes | None]): + for name, content in files.items(): + p = self.path / name + if content is None: + p.unlink() + continue + p.parent.mkdir(parents=True, exist_ok=True) + if isinstance(content, bytes): + p.write_bytes(content) + else: + p.write_bytes(content.encode("utf-8")) + + def publish(self, files: dict[str, str | bytes | None]) -> str: + """Commit *files* on a branch and merge it into main with --no-ff; + returns the merge commit.""" + self.n += 1 + when = f"2026-01-{self.n:02d}T12:00:00+01:00" + env = {**os.environ, "GIT_AUTHOR_DATE": when, "GIT_COMMITTER_DATE": when} + self.git("checkout", "-q", "-B", "pr", "main") + self.write(files) + self.git("add", "-A") + subprocess.run(["git", "-C", str(self.path), "commit", "-q", "-m", "pr"], + check=True, capture_output=True, env=env) + self.git("checkout", "-q", "main") + subprocess.run(["git", "-C", str(self.path), "merge", "-q", "--no-ff", + "-m", f"Merge {self.n}", "pr"], + check=True, capture_output=True, env=env) + return self.git("rev-parse", "HEAD") + + def index(self) -> dict: + return bi.build(self.path, "main", MD, REPO) + + +@pytest.fixture +def repo(tmp_path): + return Repo(tmp_path) + + +def by_id(index: dict) -> dict[str, dict]: + return {m["id"]: m for m in index["macros"]} + + +def test_lists_header_files_and_hashes(repo): + lua = macro(permissions="write-files", network="api.example.com") + head = repo.publish({"macros/a/a.lua": lua, "macros/a/README.md": readme(), + "macros/a/screenshots/x.png": b"\x89PNG\r\n\0"}) + index = repo.index() + assert index["schema"] == bi.SCHEMA + assert index["commit"] == head + assert index["repository"] == f"https://github.com/{REPO}" + m = by_id(index)["a"] + assert (m["name"], m["author"], m["version"], m["requires_api"]) == \ + ("Test macro", "Tester", "1.0", 3) + assert m["permissions"] == ["write-files"] + assert m["network"] == ["api.example.com"] + assert m["file"] == "macros/a/a.lua" + assert m["readme"] == "macros/a/README.md" + files = {f["path"]: f for f in m["files"]} + assert set(files) == {"macros/a/a.lua", "macros/a/README.md", + "macros/a/screenshots/x.png"} + png = files["macros/a/screenshots/x.png"] + assert png["size"] == 7 + assert png["sha256"] == hashlib.sha256(b"\x89PNG\r\n\0").hexdigest() + assert png["url"] == (f"https://raw.githubusercontent.com/{REPO}/{head}/" + "macros/a/screenshots/x.png") + + +def test_hash_is_of_the_bytes_in_git(repo): + lua = macro().replace("\n", "\r\n") + repo.publish({"macros/a/a.lua": lua, "macros/a/README.md": readme()}) + f = by_id(repo.index())["a"]["files"] + main = next(x for x in f if x["path"] == "macros/a/a.lua") + assert main["sha256"] == hashlib.sha256(lua.encode()).hexdigest() + assert main["size"] == len(lua.encode()) + + +def test_permissions_none(repo): + repo.publish({"macros/a/a.lua": macro(), "macros/a/README.md": readme()}) + assert by_id(repo.index())["a"]["permissions"] == ["none"] + + +def test_versions_newest_first_with_publish_dates(repo): + first = repo.publish({"macros/a/a.lua": macro("1.0"), "macros/a/README.md": readme()}) + repo.publish({"macros/a/README.md": readme("- 1.0 — first")}) # README only + second = repo.publish({"macros/a/a.lua": macro("1.1", api="4")}) + m = by_id(repo.index())["a"] + assert [v["version"] for v in m["versions"]] == ["1.1", "1.0"] + new, old = m["versions"] + assert new["commit"] == second and old["commit"] == first + assert new["requires_api"] == 4 and old["requires_api"] == 3 + assert old["sha256"] == hashlib.sha256(macro("1.0").encode()).hexdigest() + assert old["url"].endswith(f"/{first}/macros/a/a.lua") + assert old["published"] == "2026-01-01T11:00:00Z" + assert new["published"] == "2026-01-03T11:00:00Z" + assert (m["published"], m["updated"]) == (old["published"], new["published"]) + + +def test_same_version_republished_keeps_first_date_and_newest_bytes(repo): + repo.publish({"macros/a/a.lua": macro("1.0"), "macros/a/README.md": readme()}) + fix = macro("1.0.0", body="-- fixed\n") + last = repo.publish({"macros/a/a.lua": fix}) + (only,) = by_id(repo.index())["a"]["versions"] + assert only["commit"] == last + assert only["sha256"] == hashlib.sha256(fix.encode()).hexdigest() + assert only["published"] == "2026-01-01T11:00:00Z" + + +def test_changelog_from_readme(repo): + text = readme("- 1.0 — first version", + "- 1.10 – longer list", + "\t\tand a second line", + "- 1.2: fix", + "- not a version", + "* v2.0 (2026-02-01) - big one") + repo.publish({"macros/a/a.lua": macro(), "macros/a/README.md": text}) + assert by_id(repo.index())["a"]["changelog"] == [ + {"version": "2.0", "text": "big one"}, + {"version": "1.10", "text": "longer list and a second line"}, + {"version": "1.2", "text": "fix"}, + {"version": "1.0", "text": "first version"}, + ] + + +def test_no_changes_section(repo): + repo.publish({"macros/a/a.lua": macro(), "macros/a/README.md": "# A\n\n- 1.0 — x\n"}) + assert by_id(repo.index())["a"]["changelog"] == [] + + +def test_folder_without_macro_file_is_skipped(repo): + repo.publish({"macros/a/other.lua": macro(), "macros/a/README.md": readme()}) + index = repo.index() + assert index["macros"] == [] and index["withdrawn"] == [] + + +def test_withdrawn_macro(repo): + repo.publish({"macros/a/a.lua": macro("1.0", name="Old one"), + "macros/a/README.md": readme()}) + repo.publish({"macros/a/a.lua": macro("1.1", name="Old one")}) + repo.publish({"macros/b/b.lua": macro(), "macros/b/README.md": readme()}) + repo.publish({"macros/a/a.lua": None, "macros/a/README.md": None}) + index = repo.index() + assert [m["id"] for m in index["macros"]] == ["b"] + (w,) = index["withdrawn"] + assert (w["id"], w["name"], w["last_version"]) == ("a", "Old one", "1.1") + assert w["withdrawn"] == "2026-01-04T11:00:00Z" + assert [v["version"] for v in w["versions"]] == ["1.1", "1.0"] + assert w["versions"][1]["sha256"] == hashlib.sha256( + macro("1.0", name="Old one").encode()).hexdigest() + + +def test_macro_back_after_withdrawal_is_listed_not_withdrawn(repo): + repo.publish({"macros/a/a.lua": macro("1.0"), "macros/a/README.md": readme()}) + repo.publish({"macros/a/a.lua": None, "macros/a/README.md": None}) + repo.publish({"macros/a/a.lua": macro("2.0"), "macros/a/README.md": readme()}) + index = repo.index() + assert index["withdrawn"] == [] + assert [v["version"] for v in by_id(index)["a"]["versions"]] == ["2.0", "1.0"] + + +def test_unreadable_header_is_listed_with_error(repo): + repo.publish({"macros/a/a.lua": b"\xff\xfe-- broken\n", "macros/a/README.md": readme()}) + assert by_id(repo.index())["a"]["error"] == "not UTF-8 text" + + +def test_same_commit_gives_same_file(repo, tmp_path): + repo.publish({"macros/a/a.lua": macro(), "macros/a/README.md": readme()}) + out1, out2 = tmp_path / "1.json", tmp_path / "2.json" + for out in (out1, out2): + assert bi.main(["--repo", str(repo.path), "--ref", "main", "--metadata", + str(METADATA), "--repository", REPO, + "--output", str(out)]) == 0 + assert out1.read_bytes() == out2.read_bytes() + assert json.loads(out1.read_text(encoding="utf-8"))["macros"][0]["id"] == "a" + + +def test_this_repository(tmp_path): + """The real repository builds, and every listed file hashes to what git + stores.""" + if not (ROOT / ".git").exists(): + pytest.skip("not a git checkout") + index = bi.build(ROOT, "HEAD", MD) + git = bi.Git(ROOT) + for m in index["macros"]: + assert "error" not in m, m["id"] + assert m["versions"], m["id"] + for f in m["files"]: + data = git.read(index["commit"], f["path"]) + assert data is not None + assert hashlib.sha256(data).hexdigest() == f["sha256"] diff --git a/tests/test_check_macro_pr.py b/tests/test_check_macro_pr.py index 989b07c..a84d8a4 100644 --- a/tests/test_check_macro_pr.py +++ b/tests/test_check_macro_pr.py @@ -3,13 +3,17 @@ They need OpenSAK's header parser. In CI the workflow downloads it; locally: - ref=$(grep -v '^#' .github/opensak-ref | head -n 1) - mkdir -p .tools && curl -fsSL -o .tools/macro_metadata.py \ - https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/$ref/src/opensak/macro/metadata.py + ref=$(grep -v '^#' .github/opensak-ref | head -n 1 | tr -d '[:space:]') + mkdir -p .tools/opensak/macro + for f in metadata.py deprecation.py; do + curl -fsSL -o .tools/opensak/macro/$f \ + https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/$ref/src/opensak/macro/$f + done pip install "lupa>=2.0,<3" pytest pytest -v tests/ -Set MACRO_METADATA to use a parser somewhere else. +Set MACRO_METADATA to use a parser somewhere else (e.g. an OpenSAK +checkout's src/opensak/macro/metadata.py). """ from __future__ import annotations @@ -26,7 +30,7 @@ import check_macro_pr as cm -METADATA = Path(os.environ.get("MACRO_METADATA", ROOT / ".tools" / "macro_metadata.py")) +METADATA = Path(os.environ.get("MACRO_METADATA", ROOT / ".tools" / "opensak" / "macro" / "metadata.py")) if not METADATA.is_file(): pytest.skip(f"OpenSAK header parser not found at {METADATA} (see module docstring)", allow_module_level=True)