From dcb2fab095ffa606a6ff015cc507502beb86c6d2 Mon Sep 17 00:00:00 2001 From: nagisml Date: Wed, 18 Nov 2026 21:38:32 +0100 Subject: [PATCH 1/3] Publish a macro index with versions, sizes and SHA-256 hashes Add scripts/build_index.py and the Index workflow. After every push to main, the workflow builds index.json from main's git history and publishes it on the "index" branch: https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK-Macros/index/index.json OpenSAK's Macro manager can read this file to list, install and update macros without walking the repository through the GitHub API. Per macro the index holds: - the header, read with OpenSAK's own parser (from .github/opensak-ref) - every file in its folder, with size, SHA-256 and a download URL pinned to the commit - every version that was on main: publish date, SHA-256, size, Requires-API and permissions of the macro file - the changelog from the "## Changes" list in its README Macros removed from main are listed under "withdrawn" with the hashes of all their versions, so an installed copy can be recognised. The script only reads git objects and never runs macro code. The same commit always gives the same file, and a run that changes nothing pushes nothing. MAINTAINERS.md describes the format (schema 1). CONTRIBUTING.md and the template README now ask for a "## Changes" line per version. --- .github/workflows/index.yml | 97 ++++++++++ CONTRIBUTING.md | 13 ++ MAINTAINERS.md | 43 +++++ scripts/build_index.py | 341 ++++++++++++++++++++++++++++++++++++ template/my-macro/README.md | 3 +- tests/test_build_index.py | 256 +++++++++++++++++++++++++++ 6 files changed, 752 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/index.yml create mode 100644 scripts/build_index.py create mode 100644 tests/test_build_index.py diff --git a/.github/workflows/index.yml b/.github/workflows/index.yml new file mode 100644 index 0000000..1facc33 --- /dev/null +++ b/.github/workflows/index.yml @@ -0,0 +1,97 @@ +# Builds index.json — the list of macros with versions, file sizes and +# SHA-256 hashes that OpenSAK's Macro manager reads — after every push to +# main, and publishes it on the "index" branch: +# +# https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK-Macros/index/index.json +# +# The index branch has no history of main: it holds only index.json (and a +# README saying where it comes from), one commit per change. Nobody edits it +# by hand; a run that gives the same file pushes nothing. +# +# Runs scripts/build_index.py from main (only pushes to main trigger it), so +# what lands in the index went through the pull request checks first. The +# script reads git objects only; no macro code is run. + +name: Index + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: {} + +concurrency: + group: index + cancel-in-progress: false + +env: + INDEX_BRANCH: index + PARSER_URL: https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/%s/src/opensak/macro/metadata.py + +jobs: + index: + name: Build and publish index.json + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 0 # the versions come from main's history + persist-credentials: false + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Get OpenSAK header parser + run: | + mkdir -p "$RUNNER_TEMP/tools" + ref=$(grep -v '^#' .github/opensak-ref | head -n 1 | tr -d '[:space:]') + curl -fsSL -o "$RUNNER_TEMP/tools/macro_metadata.py" "$(printf "$PARSER_URL" "$ref")" + + - name: Build index.json + run: | + mkdir -p "$RUNNER_TEMP/out" + python -I scripts/build_index.py --repo . --ref HEAD \ + --metadata "$RUNNER_TEMP/tools/macro_metadata.py" \ + --repository "$GITHUB_REPOSITORY" \ + --output "$RUNNER_TEMP/out/index.json" + + - name: Publish on the index branch + env: + GH_TOKEN: ${{ github.token }} + run: | + out="$RUNNER_TEMP/out" + cat > "$out/README.md" <<'EOF' + # Macro index + + `index.json` lists the macros on `main` with their versions, files, + sizes and SHA-256 hashes, for OpenSAK's Macro manager. It is built by + `.github/workflows/index.yml` after every push to `main`; don't edit + this branch by hand. The format is described in MAINTAINERS.md on + `main`. + EOF + + remote="https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + work="$RUNNER_TEMP/index-branch" + if git ls-remote --exit-code --heads "$remote" "$INDEX_BRANCH" >/dev/null; then + git clone -q --depth 1 --branch "$INDEX_BRANCH" "$remote" "$work" + else + git init -q -b "$INDEX_BRANCH" "$work" + git -C "$work" remote add origin "$remote" + fi + cp "$out/index.json" "$out/README.md" "$work/" + cd "$work" + git add index.json README.md + if git diff --cached --quiet; then + echo "index.json is unchanged." + exit 0 + fi + git -c user.name="github-actions[bot]" \ + -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \ + commit -q -m "Index for ${GITHUB_SHA::7}" + git push -q origin "HEAD:$INDEX_BRANCH" + echo "Published index.json for ${GITHUB_SHA}." >> "$GITHUB_STEP_SUMMARY" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 56a3106..34e2b2d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -138,6 +138,19 @@ maintainer too. Every change to the `.lua` file needs a higher `Version`. A change to only the README or example files doesn't — but if in doubt, bump it. +Add a line for the new version to the `## Changes` section of the README, +starting with the version: + +```markdown +## Changes + +- 1.1 — lists the last 10 runs instead of 5 +- 1.0 — first version +``` + +OpenSAK's Macro manager shows these lines when it offers the update (see +[The macro index](MAINTAINERS.md#the-macro-index)). + ## License By opening a pull request you agree that your contribution is published under diff --git a/MAINTAINERS.md b/MAINTAINERS.md index ae0a249..c96f437 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -71,6 +71,47 @@ The checker also uses two internal helpers of the parser (`_header_lines`, `_KEY_LINE`) to see which keys are actually written. If "Test checker" fails after a bump, look there first. +## The macro index + +After every push to `main`, **Index** (`.github/workflows/index.yml`) runs +`scripts/build_index.py` and publishes `index.json` on the `index` branch: + + https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK-Macros/index/index.json + +OpenSAK's Macro manager reads it to list, install and update macros, so it +doesn't have to walk the repository through the GitHub API. Everything in it +comes from `main`'s git history, so the same commit always gives the same +file; a push that changes nothing in it (e.g. to MAINTAINERS.md) leaves the +`index` branch alone. Run it by hand from the Actions tab ("Run workflow") +after changing the script or `.github/opensak-ref`. + +Locally (with the header parser in `.tools/`, see `tests/`): + + python scripts/build_index.py --metadata .tools/macro_metadata.py --ref main + +Format (`schema` 1; fields are only ever added, a breaking change raises +`schema`): + +| Field | | +|---|---| +| `schema`, `commit`, `generated` | format version; the `main` commit indexed and its date (UTC) | +| `macros[]` | one entry per folder under `macros/` that has `/.lua` | +| `.id`, `.folder`, `.file`, `.readme`, `.page` | folder name, paths in the repository, link to the folder on GitHub | +| `.name`, `.author`, `.version`, `.description`, `.requires_api`, `.permissions`, `.network`, `.headless` | the header, read with OpenSAK's parser (`headless` is null with a parser from before API 4) | +| `.files[]` | every file in the folder: `path`, `size`, `sha256`, and a download `url` pinned to `commit` | +| `.versions[]` | every version that was on `main`, newest first: `version`, `published` (date it first reached `main`), `commit`, `requires_api`, `permissions`, `network`, and `size`, `sha256`, `url` of the macro file | +| `.published`, `.updated` | when the first and the current version reached `main` | +| `.changelog[]` | `version` and `text` from the `## Changes` list in the README, newest first | +| `.error` | only when the header couldn't be read | +| `withdrawn[]` | macros that were on `main` and were removed: `id`, `folder`, `name`, `last_version`, `withdrawn` (date), and `versions[]` with the `sha256` of each, so OpenSAK can recognise an installed copy | + +Hashes are SHA-256 of the bytes in git, which are the bytes the `url`s +serve. A macro is identified by its folder name, so renaming a folder lists +the old name as withdrawn and the new one as a new macro. + +The `index` branch must stay writable by GitHub Actions: don't put it under +the "Protect main" ruleset or any other ruleset that blocks pushes. + ## Repository settings this relies on - Ruleset "Protect main" on the default branch: pull request required (0 @@ -80,3 +121,5 @@ fails after a bump, look there first. - Settings → General → Pull Requests → **Allow auto-merge** on. - Labels `macro problem`, `auto-merge` and `needs review`. - Discussions on. +- The `index` branch is created by the first **Index** run and left + unprotected (see [The macro index](#the-macro-index)). diff --git a/scripts/build_index.py b/scripts/build_index.py new file mode 100644 index 0000000..3117777 --- /dev/null +++ b/scripts/build_index.py @@ -0,0 +1,341 @@ +""" +scripts/build_index.py — builds index.json, the list of macros OpenSAK's +Macro manager reads to browse, install and update macros. + +Run by .github/workflows/index.yml after every push to main; the workflow +publishes the result on the "index" branch: + + https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK-Macros/index/index.json + +Like check_macro_pr.py it only reads git objects (git ls-tree / git show / +git log) and parses headers with OpenSAK's own src/opensak/macro/metadata.py +(--metadata). It never runs Lua code. + +Everything comes from the git history of one commit (--ref, normally main), +so building twice from the same commit gives the same file: + + - macros: every folder under macros/ that has /.lua, with + its header, every file in the folder (size, SHA-256 and a download URL + pinned to the commit, so the bytes behind a URL never change), the + versions it had on main (when each was published, the SHA-256 of the + macro file) and the changelog from the "## Changes" section of its + README. + - withdrawn: folders that were on main once and are not any more, with + their last version and the hashes of all their versions, so OpenSAK can + tell a user that an installed macro was withdrawn. + +Hashes are taken over the bytes git stores, which are the bytes +raw.githubusercontent.com serves (no CRLF conversion). + +A macro whose header can't be read is still listed, with "error" set; the +pull request checks keep that from happening on main. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import hashlib +import importlib.util +import json +import re +import subprocess +import sys +from pathlib import Path, PurePosixPath +from types import ModuleType +from typing import Any + +SCHEMA = 1 +MACROS_DIR = "macros" +DEFAULT_REPOSITORY = "OpenSAK-Org/OpenSAK-Macros" +RAW_URL = "https://raw.githubusercontent.com/{repo}/{commit}/{path}" +PAGE_URL = "https://github.com/{repo}/tree/{ref}/{path}" +VERSION = re.compile(r"^\d+(?:\.\d+)*$") + +# "## Changes" (also "## Changelog", "## Change log", "## History") +_CHANGES_HEADING = re.compile(r"^##\s+(?:changes|change\s*log|history)\s*$", re.I) +# "- 1.1 — text", also with "-", "–", ":" or "(date)" after the version +_CHANGE_ITEM = re.compile( + r"^[-*]\s+v?(\d+(?:\.\d+)*)\b\s*(?:\([^)]*\))?\s*(?:—|–|-|:)?\s*(.*)$") + + +# --- git --------------------------------------------------------------------- + +class Git: + def __init__(self, repo: Path): + self.repo = repo + + def _run(self, *args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run(["git", "-C", str(self.repo), *args], + capture_output=True, check=check) + + def rev_parse(self, rev: str) -> str: + return self._run("rev-parse", "--verify", f"{rev}^{{commit}}" + ).stdout.decode().strip() + + def commit_date(self, rev: str) -> str: + return utc(self._run("show", "-s", "--format=%ct", rev).stdout.decode()) + + def ls_tree(self, rev: str, folder: str) -> list[tuple[str, int]]: + """(path, size) of every file under *folder* at *rev*.""" + out = self._run("ls-tree", "-r", "-z", "--long", rev, "--", + folder + "/").stdout.decode("utf-8") + files = [] + for entry in out.split("\0"): + if not entry: + continue + meta, path = entry.split("\t", 1) + kind, size = meta.split()[1], meta.split()[3] + if kind == "blob": + files.append((path, int(size))) + return sorted(files) + + def read(self, rev: str, path: str) -> bytes | None: + proc = self._run("show", f"{rev}:{path}", check=False) + return proc.stdout if proc.returncode == 0 else None + + def folders_ever(self, rev: str) -> set[str]: + """Names of all folders under macros/ that ever had a file on the + first-parent history of *rev*.""" + out = self._run("log", "--first-parent", "--format=", "--name-only", + "-z", rev, "--", MACROS_DIR + "/").stdout.decode("utf-8") + names = set() + for path in out.replace("\n", "\0").split("\0"): + parts = PurePosixPath(path).parts + if len(parts) >= 3 and parts[0] == MACROS_DIR: + names.add(parts[1]) + return names + + def history(self, rev: str, path: str) -> list[tuple[str, str]]: + """(commit, date) of every commit on the first-parent history of + *rev* that changed *path*, oldest first. On main these are the merge + commits, i.e. when a change was published.""" + out = self._run("log", "--first-parent", "--format=%H %ct", rev, + "--", path).stdout.decode() + found = [] + for line in reversed(out.splitlines()): + if line: + commit, stamp = line.split(" ", 1) + found.append((commit, utc(stamp))) + return found + + +# --- helpers ----------------------------------------------------------------- + +def load_metadata_module(path: Path) -> ModuleType: + spec = importlib.util.spec_from_file_location("macro_metadata", path) + if spec is None or spec.loader is None: + raise SystemExit(f"Cannot load the OpenSAK header parser from {path}") + module = importlib.util.module_from_spec(spec) + sys.modules["macro_metadata"] = module # dataclasses look the module up + spec.loader.exec_module(module) + return module + + +def utc(stamp: str) -> str: + """A git Unix timestamp as ISO 8601 in UTC, e.g. 2026-11-18T12:14:51Z.""" + when = datetime.fromtimestamp(int(stamp.strip()), timezone.utc) + return when.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def version_key(version: str) -> tuple[int, ...]: + """Dot-separated numbers as a comparable tuple; trailing zeros dropped + so that 1.0 and 1.0.0 are the same version.""" + parts = [int(x) for x in version.split(".")] + while len(parts) > 1 and parts[-1] == 0: + parts.pop() + return tuple(parts) + + +def same_version(a: str, b: str) -> bool: + if VERSION.match(a) and VERSION.match(b): + return version_key(a) == version_key(b) + return a == b + + +def header(md: ModuleType, data: bytes | None, path: str) -> dict[str, Any]: + """The header fields OpenSAK's Macro manager shows, or {"error": ...}.""" + if data is None: + return {"error": "missing"} + try: + text = data.decode("utf-8") + except UnicodeDecodeError: + return {"error": "not UTF-8 text"} + info = md.parse_header(text, Path(path)) + api = getattr(info, "requires_api_number", None) + return { + "name": info.name, + "author": info.author, + "version": info.version, + "description": info.description, + "requires_api": api if api is not None else (info.requires_api or None), + "permissions": list(info.permissions) + or ([md.NO_PERMISSIONS] if info.permissions_declared else []), + "network": list(info.network), + # null with a parser from before the Headless key (API 4) + "headless": getattr(info, "runs_headless", None), + } + + +def changelog(readme: bytes | None) -> list[dict[str, str]]: + """The entries of the "## Changes" section of a macro README: one per + list item starting with a version ("- 1.1 — what changed"), newest + first. Indented lines below an item continue it.""" + if not readme: + return [] + try: + text = readme.decode("utf-8") + except UnicodeDecodeError: + return [] + entries: list[dict[str, str]] = [] + inside = False + for raw in text.splitlines(): + line = raw.strip() + if raw.startswith("#"): + inside = bool(_CHANGES_HEADING.match(line)) + continue + if not inside or not line: + continue + m = _CHANGE_ITEM.match(line) + if m: + entries.append({"version": m.group(1), "text": m.group(2).strip()}) + elif entries and raw[:1] in (" ", "\t"): + entries[-1]["text"] = (entries[-1]["text"] + " " + line).strip() + good = [e for e in entries if VERSION.match(e["version"])] + good.sort(key=lambda e: version_key(e["version"]), reverse=True) + return good + + +def versions(git: Git, md: ModuleType, ref: str, main: str, + repository: str) -> list[dict[str, Any]]: + """Every version of the macro file *main* on the history of *ref*, + newest first: when it was first published and the file of its last + commit (a version republished without a bump keeps the newest bytes).""" + found: list[dict[str, Any]] = [] + for commit, date in git.history(ref, main): + data = git.read(commit, main) + if data is None: # the file was removed in this commit + continue + h = header(md, data, main) + version = h.get("version", "") + entry = { + "version": version, + "published": date, + "commit": commit, + "requires_api": h.get("requires_api"), + "permissions": h.get("permissions", []), + "network": h.get("network", []), + "size": len(data), + "sha256": sha256(data), + "url": RAW_URL.format(repo=repository, commit=commit, path=main), + } + if found and same_version(found[-1]["version"], version): + entry["published"] = found[-1]["published"] + found[-1] = entry + else: + found.append(entry) + return list(reversed(found)) + + +# --- the index --------------------------------------------------------------- + +def macro_entry(git: Git, md: ModuleType, commit: str, name: str, + repository: str) -> dict[str, Any] | None: + folder = f"{MACROS_DIR}/{name}" + main = f"{folder}/{name}.lua" + files = git.ls_tree(commit, folder) + if main not in {p for p, _ in files}: + return None + entry: dict[str, Any] = {"id": name, "folder": folder} + entry.update(header(md, git.read(commit, main), main)) + entry["file"] = main + entry["readme"] = f"{folder}/README.md" if any( + p == f"{folder}/README.md" for p, _ in files) else None + entry["page"] = PAGE_URL.format(repo=repository, ref="main", path=folder) + entry["files"] = [ + {"path": p, "size": size, "sha256": sha256(git.read(commit, p) or b""), + "url": RAW_URL.format(repo=repository, commit=commit, path=p)} + for p, size in files + ] + entry["versions"] = versions(git, md, commit, main, repository) + if entry["versions"]: + entry["published"] = entry["versions"][-1]["published"] + entry["updated"] = entry["versions"][0]["published"] + entry["changelog"] = changelog( + git.read(commit, entry["readme"]) if entry["readme"] else None) + return entry + + +def withdrawn_entry(git: Git, md: ModuleType, commit: str, name: str, + repository: str) -> dict[str, Any] | None: + main = f"{MACROS_DIR}/{name}/{name}.lua" + history = versions(git, md, commit, main, repository) + if not history: + return None # never had a macro file (e.g. a typo folder) + removed = git.history(commit, main)[-1] + last = history[0] + return { + "id": name, + "folder": f"{MACROS_DIR}/{name}", + "name": header(md, git.read(last["commit"], main), main).get("name", name), + "last_version": last["version"], + "withdrawn": removed[1], + "versions": [{"version": v["version"], "sha256": v["sha256"]} + for v in history], + } + + +def build(repo: Path, ref: str, md: ModuleType, + repository: str = DEFAULT_REPOSITORY) -> dict[str, Any]: + git = Git(repo) + commit = git.rev_parse(ref) + current = {p.split("/")[1] for p, _ in git.ls_tree(commit, MACROS_DIR) + if len(PurePosixPath(p).parts) >= 3} + macros = [e for name in sorted(current) + if (e := macro_entry(git, md, commit, name, repository))] + listed = {m["id"] for m in macros} + withdrawn = [e for name in sorted(git.folders_ever(commit) - listed) + if (e := withdrawn_entry(git, md, commit, name, repository))] + return { + "schema": SCHEMA, + "repository": f"https://github.com/{repository}", + "commit": commit, + "generated": git.commit_date(commit), + "macros": macros, + "withdrawn": withdrawn, + } + + +def dumps(index: dict[str, Any]) -> str: + return json.dumps(index, indent=2, ensure_ascii=False) + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--repo", type=Path, default=Path(".")) + parser.add_argument("--ref", default="HEAD", help="commit to index (main)") + parser.add_argument("--metadata", type=Path, required=True, + help="path to OpenSAK's src/opensak/macro/metadata.py") + parser.add_argument("--repository", default=DEFAULT_REPOSITORY, + help="owner/name on GitHub, for the download URLs") + parser.add_argument("--output", type=Path, help="write here (default: stdout)") + args = parser.parse_args(argv) + + index = build(args.repo, args.ref, load_metadata_module(args.metadata), + args.repository) + text = dumps(index) + if args.output: + args.output.write_text(text, encoding="utf-8", newline="\n") + print(f"{len(index['macros'])} macro(s), {len(index['withdrawn'])} " + f"withdrawn, from {index['commit']}") + else: + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/template/my-macro/README.md b/template/my-macro/README.md index 9b1e7ef..132301a 100644 --- a/template/my-macro/README.md +++ b/template/my-macro/README.md @@ -26,6 +26,7 @@ Optional: a screenshot (put it in `screenshots/`) or an example file ## Changes -Optional: a short note per version. +Optional: a short note per version, one line each starting with the version. +OpenSAK's Macro manager shows these notes when it offers an update. - 1.0 — first version diff --git a/tests/test_build_index.py b/tests/test_build_index.py new file mode 100644 index 0000000..b7ec828 --- /dev/null +++ b/tests/test_build_index.py @@ -0,0 +1,256 @@ +""" +Tests for scripts/build_index.py. + +They need OpenSAK's header parser, like tests/test_check_macro_pr.py (see +there how to get it locally). Set MACRO_METADATA to use one somewhere else. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) + +import build_index as bi + +METADATA = Path(os.environ.get("MACRO_METADATA", ROOT / ".tools" / "macro_metadata.py")) +if not METADATA.is_file(): + pytest.skip(f"OpenSAK header parser not found at {METADATA} (see " + "tests/test_check_macro_pr.py)", allow_module_level=True) + +MD = bi.load_metadata_module(METADATA) +REPO = "Example/Macros" + + +def macro(version="1.0", permissions="none", network=None, api="3", name="Test macro", + body='opensak.log("hi")\n'): + header = {"Name": name, "Author": "Tester", "Version": version, + "Requires-API": api, "Description": "Does a test.", + "Permissions": permissions} + if network is not None: + header["Network"] = network + lines = ["-- OpenSAK Macro"] + [f"-- {k}: {v}" for k, v in header.items()] + return "\n".join(lines) + "\n\n" + body + + +def readme(*changes: str) -> str: + return "# Test\n\nWhat it does.\n\n## Changes\n\n" + "".join( + f"{c}\n" for c in changes) + "\n## License\n\n- 9.9 — not a change\n" + + +class Repo: + """A repository whose main gets one merge commit per publish(), like + pull requests merged with "Create a merge commit".""" + + def __init__(self, path: Path): + self.path = path + self.n = 0 + self.git("init", "-q", "-b", "main") + self.git("config", "user.email", "t@example.com") + self.git("config", "user.name", "t") + self.git("config", "core.autocrlf", "false") + self.write({"README.md": "repo\n"}) + self.git("add", "-A") + self.git("commit", "-q", "-m", "base") + + def git(self, *args): + return subprocess.run(["git", "-C", str(self.path), *args], check=True, + capture_output=True, text=True).stdout.strip() + + def write(self, files: dict[str, str | bytes | None]): + for name, content in files.items(): + p = self.path / name + if content is None: + p.unlink() + continue + p.parent.mkdir(parents=True, exist_ok=True) + if isinstance(content, bytes): + p.write_bytes(content) + else: + p.write_bytes(content.encode("utf-8")) + + def publish(self, files: dict[str, str | bytes | None]) -> str: + """Commit *files* on a branch and merge it into main with --no-ff; + returns the merge commit.""" + self.n += 1 + when = f"2026-01-{self.n:02d}T12:00:00+01:00" + env = {**os.environ, "GIT_AUTHOR_DATE": when, "GIT_COMMITTER_DATE": when} + self.git("checkout", "-q", "-B", "pr", "main") + self.write(files) + self.git("add", "-A") + subprocess.run(["git", "-C", str(self.path), "commit", "-q", "-m", "pr"], + check=True, capture_output=True, env=env) + self.git("checkout", "-q", "main") + subprocess.run(["git", "-C", str(self.path), "merge", "-q", "--no-ff", + "-m", f"Merge {self.n}", "pr"], + check=True, capture_output=True, env=env) + return self.git("rev-parse", "HEAD") + + def index(self) -> dict: + return bi.build(self.path, "main", MD, REPO) + + +@pytest.fixture +def repo(tmp_path): + return Repo(tmp_path) + + +def by_id(index: dict) -> dict[str, dict]: + return {m["id"]: m for m in index["macros"]} + + +def test_lists_header_files_and_hashes(repo): + lua = macro(permissions="write-files", network="api.example.com") + head = repo.publish({"macros/a/a.lua": lua, "macros/a/README.md": readme(), + "macros/a/screenshots/x.png": b"\x89PNG\r\n\0"}) + index = repo.index() + assert index["schema"] == bi.SCHEMA + assert index["commit"] == head + assert index["repository"] == f"https://github.com/{REPO}" + m = by_id(index)["a"] + assert (m["name"], m["author"], m["version"], m["requires_api"]) == \ + ("Test macro", "Tester", "1.0", 3) + assert m["permissions"] == ["write-files"] + assert m["network"] == ["api.example.com"] + assert m["file"] == "macros/a/a.lua" + assert m["readme"] == "macros/a/README.md" + files = {f["path"]: f for f in m["files"]} + assert set(files) == {"macros/a/a.lua", "macros/a/README.md", + "macros/a/screenshots/x.png"} + png = files["macros/a/screenshots/x.png"] + assert png["size"] == 7 + assert png["sha256"] == hashlib.sha256(b"\x89PNG\r\n\0").hexdigest() + assert png["url"] == (f"https://raw.githubusercontent.com/{REPO}/{head}/" + "macros/a/screenshots/x.png") + + +def test_hash_is_of_the_bytes_in_git(repo): + lua = macro().replace("\n", "\r\n") + repo.publish({"macros/a/a.lua": lua, "macros/a/README.md": readme()}) + f = by_id(repo.index())["a"]["files"] + main = next(x for x in f if x["path"] == "macros/a/a.lua") + assert main["sha256"] == hashlib.sha256(lua.encode()).hexdigest() + assert main["size"] == len(lua.encode()) + + +def test_permissions_none(repo): + repo.publish({"macros/a/a.lua": macro(), "macros/a/README.md": readme()}) + assert by_id(repo.index())["a"]["permissions"] == ["none"] + + +def test_versions_newest_first_with_publish_dates(repo): + first = repo.publish({"macros/a/a.lua": macro("1.0"), "macros/a/README.md": readme()}) + repo.publish({"macros/a/README.md": readme("- 1.0 — first")}) # README only + second = repo.publish({"macros/a/a.lua": macro("1.1", api="4")}) + m = by_id(repo.index())["a"] + assert [v["version"] for v in m["versions"]] == ["1.1", "1.0"] + new, old = m["versions"] + assert new["commit"] == second and old["commit"] == first + assert new["requires_api"] == 4 and old["requires_api"] == 3 + assert old["sha256"] == hashlib.sha256(macro("1.0").encode()).hexdigest() + assert old["url"].endswith(f"/{first}/macros/a/a.lua") + assert old["published"] == "2026-01-01T11:00:00Z" + assert new["published"] == "2026-01-03T11:00:00Z" + assert (m["published"], m["updated"]) == (old["published"], new["published"]) + + +def test_same_version_republished_keeps_first_date_and_newest_bytes(repo): + repo.publish({"macros/a/a.lua": macro("1.0"), "macros/a/README.md": readme()}) + fix = macro("1.0.0", body="-- fixed\n") + last = repo.publish({"macros/a/a.lua": fix}) + (only,) = by_id(repo.index())["a"]["versions"] + assert only["commit"] == last + assert only["sha256"] == hashlib.sha256(fix.encode()).hexdigest() + assert only["published"] == "2026-01-01T11:00:00Z" + + +def test_changelog_from_readme(repo): + text = readme("- 1.0 — first version", + "- 1.10 – longer list", + "\t\tand a second line", + "- 1.2: fix", + "- not a version", + "* v2.0 (2026-02-01) - big one") + repo.publish({"macros/a/a.lua": macro(), "macros/a/README.md": text}) + assert by_id(repo.index())["a"]["changelog"] == [ + {"version": "2.0", "text": "big one"}, + {"version": "1.10", "text": "longer list and a second line"}, + {"version": "1.2", "text": "fix"}, + {"version": "1.0", "text": "first version"}, + ] + + +def test_no_changes_section(repo): + repo.publish({"macros/a/a.lua": macro(), "macros/a/README.md": "# A\n\n- 1.0 — x\n"}) + assert by_id(repo.index())["a"]["changelog"] == [] + + +def test_folder_without_macro_file_is_skipped(repo): + repo.publish({"macros/a/other.lua": macro(), "macros/a/README.md": readme()}) + index = repo.index() + assert index["macros"] == [] and index["withdrawn"] == [] + + +def test_withdrawn_macro(repo): + repo.publish({"macros/a/a.lua": macro("1.0", name="Old one"), + "macros/a/README.md": readme()}) + repo.publish({"macros/a/a.lua": macro("1.1", name="Old one")}) + repo.publish({"macros/b/b.lua": macro(), "macros/b/README.md": readme()}) + repo.publish({"macros/a/a.lua": None, "macros/a/README.md": None}) + index = repo.index() + assert [m["id"] for m in index["macros"]] == ["b"] + (w,) = index["withdrawn"] + assert (w["id"], w["name"], w["last_version"]) == ("a", "Old one", "1.1") + assert w["withdrawn"] == "2026-01-04T11:00:00Z" + assert [v["version"] for v in w["versions"]] == ["1.1", "1.0"] + assert w["versions"][1]["sha256"] == hashlib.sha256( + macro("1.0", name="Old one").encode()).hexdigest() + + +def test_macro_back_after_withdrawal_is_listed_not_withdrawn(repo): + repo.publish({"macros/a/a.lua": macro("1.0"), "macros/a/README.md": readme()}) + repo.publish({"macros/a/a.lua": None, "macros/a/README.md": None}) + repo.publish({"macros/a/a.lua": macro("2.0"), "macros/a/README.md": readme()}) + index = repo.index() + assert index["withdrawn"] == [] + assert [v["version"] for v in by_id(index)["a"]["versions"]] == ["2.0", "1.0"] + + +def test_unreadable_header_is_listed_with_error(repo): + repo.publish({"macros/a/a.lua": b"\xff\xfe-- broken\n", "macros/a/README.md": readme()}) + assert by_id(repo.index())["a"]["error"] == "not UTF-8 text" + + +def test_same_commit_gives_same_file(repo, tmp_path): + repo.publish({"macros/a/a.lua": macro(), "macros/a/README.md": readme()}) + out1, out2 = tmp_path / "1.json", tmp_path / "2.json" + for out in (out1, out2): + assert bi.main(["--repo", str(repo.path), "--ref", "main", "--metadata", + str(METADATA), "--repository", REPO, + "--output", str(out)]) == 0 + assert out1.read_bytes() == out2.read_bytes() + assert json.loads(out1.read_text(encoding="utf-8"))["macros"][0]["id"] == "a" + + +def test_this_repository(tmp_path): + """The real repository builds, and every listed file hashes to what git + stores.""" + if not (ROOT / ".git").exists(): + pytest.skip("not a git checkout") + index = bi.build(ROOT, "HEAD", MD) + git = bi.Git(ROOT) + for m in index["macros"]: + assert "error" not in m, m["id"] + assert m["versions"], m["id"] + for f in m["files"]: + data = git.read(index["commit"], f["path"]) + assert data is not None + assert hashlib.sha256(data).hexdigest() == f["sha256"] From c8da51e5f7b7f3bd140e04fb8b1f39ecfcac30d1 Mon Sep 17 00:00:00 2001 From: nagisml Date: Wed, 18 Nov 2026 22:14:09 +0100 Subject: [PATCH 2/3] Load OpenSAK's parser with deprecation.py; bump opensak-ref to v1.22.0 From v1.22.0, OpenSAK's src/opensak/macro/metadata.py imports opensak.macro.deprecation, so downloading metadata.py alone fails with ModuleNotFoundError. OpenSAK keeps deprecation.py free of other OpenSAK imports so the two files work together outside OpenSAK. - check.yml and index.yml download metadata.py and deprecation.py (PARSER_FILES) into an opensak/macro/ folder. - load_metadata_module() in check_macro_pr.py and build_index.py puts the folder above opensak/macro/ on sys.path. The scripts run under python -I, which ignores PYTHONPATH. A parser in any other place (e.g. beta.9 on its own) is loaded as before. - The tests expect the parser at .tools/opensak/macro/metadata.py; their docstring explains how to download both files. - .github/opensak-ref: v1.22.0-beta.9 -> v1.22.0 (2f05c47). - MAINTAINERS.md explains the two-file setup and what to do if a later bump fails with ModuleNotFoundError. - v1.22.0 blocks more Geocaching HQ domains (Groundspeak, Waymarking, Wherigo, ...): CONTRIBUTING.md and the GitHub website guide now say so. - "headless" in index.json stays null: v1.22.0 has no Headless header key yet (planned in OpenSAK #1195). The comment and MAINTAINERS.md wrongly tied it to API 4. --- .github/opensak-ref | 8 ++++---- .github/workflows/check.yml | 22 +++++++++++++++------- .github/workflows/index.yml | 12 ++++++++---- CONTRIBUTING.md | 6 ++++-- MAINTAINERS.md | 13 +++++++++---- docs/github-website-guide.md | 5 +++-- scripts/build_index.py | 17 ++++++++++++++++- scripts/check_macro_pr.py | 14 ++++++++++++++ tests/test_build_index.py | 2 +- tests/test_check_macro_pr.py | 14 +++++++++----- 10 files changed, 83 insertions(+), 30 deletions(-) diff --git a/.github/opensak-ref b/.github/opensak-ref index e330fa6..61a0b64 100644 --- a/.github/opensak-ref +++ b/.github/opensak-ref @@ -1,5 +1,5 @@ -# The OpenSAK commit whose src/opensak/macro/metadata.py the checks use to -# parse macro headers. Change it deliberately when the header rules change +# The OpenSAK commit whose src/opensak/macro/metadata.py (and the +# deprecation.py it imports) the checks use to parse macro headers. Change it deliberately when the header rules change # in OpenSAK (see MAINTAINERS.md). The first line without "#" is used. -# OpenSAK v1.22.0-beta.9 -7f2bf6daafb91aee9357c9b5363655af8424a4fc +# OpenSAK v1.22.0 +2f05c474b1dd7e688919c8f3070b8e2dec2b8be3 diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index bfc51d4..8eb8117 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -10,8 +10,9 @@ # "Test checker" runs the tests with the pull request's version of the # scripts, for pull requests that change them. # -# The header parser is OpenSAK's src/opensak/macro/metadata.py, downloaded -# from the commit in .github/opensak-ref. +# The header parser is OpenSAK's src/opensak/macro/metadata.py (with the +# deprecation.py it imports), downloaded from the commit in +# .github/opensak-ref into an opensak/macro/ folder. name: Check @@ -28,7 +29,9 @@ concurrency: env: LUPA: "lupa>=2.0,<3" - PARSER_URL: https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/%s/src/opensak/macro/metadata.py + # metadata.py imports deprecation.py (from v1.22.0); both are needed. + PARSER_URL: https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/%s/src/opensak/macro + PARSER_FILES: metadata.py deprecation.py jobs: check-macro: @@ -55,7 +58,10 @@ jobs: from_main() { git show "$BASE:$1" 2>/dev/null || cat "$1"; } from_main scripts/check_macro_pr.py > "$RUNNER_TEMP/tools/check_macro_pr.py" ref=$(from_main .github/opensak-ref | grep -v '^#' | head -n 1 | tr -d '[:space:]') - curl -fsSL -o "$RUNNER_TEMP/tools/macro_metadata.py" "$(printf "$PARSER_URL" "$ref")" + mkdir -p "$RUNNER_TEMP/opensak-src/opensak/macro" + for f in $PARSER_FILES; do + curl -fsSL -o "$RUNNER_TEMP/opensak-src/opensak/macro/$f" "$(printf "$PARSER_URL" "$ref")/$f" + done - name: Check macro env: @@ -64,7 +70,7 @@ jobs: run: | python -I "$RUNNER_TEMP/tools/check_macro_pr.py" \ --repo . --base "$BASE" --head "$HEAD" \ - --metadata "$RUNNER_TEMP/tools/macro_metadata.py" + --metadata "$RUNNER_TEMP/opensak-src/opensak/macro/metadata.py" test-checker: name: Test checker @@ -83,9 +89,11 @@ jobs: - name: Get OpenSAK header parser run: | - mkdir -p .tools ref=$(grep -v '^#' .github/opensak-ref | head -n 1 | tr -d '[:space:]') - curl -fsSL -o .tools/macro_metadata.py "$(printf "$PARSER_URL" "$ref")" + mkdir -p ".tools/opensak/macro" + for f in $PARSER_FILES; do + curl -fsSL -o ".tools/opensak/macro/$f" "$(printf "$PARSER_URL" "$ref")/$f" + done - name: Run tests run: pytest -v tests/ diff --git a/.github/workflows/index.yml b/.github/workflows/index.yml index 1facc33..90f1ac4 100644 --- a/.github/workflows/index.yml +++ b/.github/workflows/index.yml @@ -27,7 +27,9 @@ concurrency: env: INDEX_BRANCH: index - PARSER_URL: https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/%s/src/opensak/macro/metadata.py + # metadata.py imports deprecation.py (from v1.22.0); both are needed. + PARSER_URL: https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/%s/src/opensak/macro + PARSER_FILES: metadata.py deprecation.py jobs: index: @@ -48,15 +50,17 @@ jobs: - name: Get OpenSAK header parser run: | - mkdir -p "$RUNNER_TEMP/tools" ref=$(grep -v '^#' .github/opensak-ref | head -n 1 | tr -d '[:space:]') - curl -fsSL -o "$RUNNER_TEMP/tools/macro_metadata.py" "$(printf "$PARSER_URL" "$ref")" + mkdir -p "$RUNNER_TEMP/opensak-src/opensak/macro" + for f in $PARSER_FILES; do + curl -fsSL -o "$RUNNER_TEMP/opensak-src/opensak/macro/$f" "$(printf "$PARSER_URL" "$ref")/$f" + done - name: Build index.json run: | mkdir -p "$RUNNER_TEMP/out" python -I scripts/build_index.py --repo . --ref HEAD \ - --metadata "$RUNNER_TEMP/tools/macro_metadata.py" \ + --metadata "$RUNNER_TEMP/opensak-src/opensak/macro/metadata.py" \ --repository "$GITHUB_REPOSITORY" \ --output "$RUNNER_TEMP/out/index.json" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 34e2b2d..a9ccde0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,8 +64,10 @@ Declare everything the macro can do, not just what it usually does. List plain host names (or IP addresses): `api.example.com`, not `https://api.example.com/path`, no ports and no wildcards. -`geocaching.com`, `coord.info` and their subdomains are not allowed — the -Geocaching.com terms of use don't permit automated access outside their API, +Geocaching HQ's sites — `geocaching.com`, `coord.info`, `groundspeak.com`, +`waymarking.com`, `wherigo.com` and related domains — and their +subdomains are not allowed: the Geocaching.com terms of use don't permit +automated access outside their API, so OpenSAK will never let a macro contact them. ## Automatic checks diff --git a/MAINTAINERS.md b/MAINTAINERS.md index c96f437..dd75d25 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -57,7 +57,9 @@ the README explains why. ## Changing the header rules The checks parse headers with OpenSAK's `src/opensak/macro/metadata.py` from -the commit in `.github/opensak-ref`. When OpenSAK changes the header rules +the commit in `.github/opensak-ref`, together with the `deprecation.py` it +imports (from v1.22.0; OpenSAK keeps that module free of other OpenSAK +imports for this repository's sake). When OpenSAK changes the header rules (new permission words, new keys, new blocked hosts): 1. Put the new OpenSAK commit (ideally a tagged release) in @@ -69,7 +71,10 @@ the commit in `.github/opensak-ref`. When OpenSAK changes the header rules The checker also uses two internal helpers of the parser (`_header_lines`, `_KEY_LINE`) to see which keys are actually written. If "Test checker" -fails after a bump, look there first. +fails after a bump, look there first. If it fails with `ModuleNotFoundError`, +`metadata.py` has gained another top-level import: add that file to +`PARSER_FILES` in `check.yml` and `index.yml` (and to the download in +`tests/test_check_macro_pr.py`). ## The macro index @@ -87,7 +92,7 @@ after changing the script or `.github/opensak-ref`. Locally (with the header parser in `.tools/`, see `tests/`): - python scripts/build_index.py --metadata .tools/macro_metadata.py --ref main + python scripts/build_index.py --metadata .tools/opensak/macro/metadata.py --ref main Format (`schema` 1; fields are only ever added, a breaking change raises `schema`): @@ -97,7 +102,7 @@ Format (`schema` 1; fields are only ever added, a breaking change raises | `schema`, `commit`, `generated` | format version; the `main` commit indexed and its date (UTC) | | `macros[]` | one entry per folder under `macros/` that has `/.lua` | | `.id`, `.folder`, `.file`, `.readme`, `.page` | folder name, paths in the repository, link to the folder on GitHub | -| `.name`, `.author`, `.version`, `.description`, `.requires_api`, `.permissions`, `.network`, `.headless` | the header, read with OpenSAK's parser (`headless` is null with a parser from before API 4) | +| `.name`, `.author`, `.version`, `.description`, `.requires_api`, `.permissions`, `.network`, `.headless` | the header, read with OpenSAK's parser (`headless` stays null until OpenSAK's parser has a Headless key; v1.22.0 doesn't) | | `.files[]` | every file in the folder: `path`, `size`, `sha256`, and a download `url` pinned to `commit` | | `.versions[]` | every version that was on `main`, newest first: `version`, `published` (date it first reached `main`), `commit`, `requires_api`, `permissions`, `network`, and `size`, `sha256`, `url` of the macro file | | `.published`, `.updated` | when the first and the current version reached `main` | diff --git a/docs/github-website-guide.md b/docs/github-website-guide.md index 322284d..df0f503 100644 --- a/docs/github-website-guide.md +++ b/docs/github-website-guide.md @@ -48,8 +48,9 @@ the GitHub website — no git, no command line. The rules themselves are in `read-files`, `write-files`, `open`, `run-macros`). If it only reads data, write `none`. - **Network:** add this line only if the macro contacts web sites. Give - plain host names such as `api.example.com`. `geocaching.com` and - `coord.info` are not allowed. + plain host names such as `api.example.com`. Geocaching HQ's sites + (`geocaching.com`, `coord.info`, `waymarking.com`, `wherigo.com` and + related domains) are not allowed. 4. Write the `README.md`: what the macro does, how to use it, and what it needs. diff --git a/scripts/build_index.py b/scripts/build_index.py index 3117777..c8ca764 100644 --- a/scripts/build_index.py +++ b/scripts/build_index.py @@ -123,6 +123,20 @@ def history(self, rev: str, path: str) -> list[tuple[str, str]]: # --- helpers ----------------------------------------------------------------- def load_metadata_module(path: Path) -> ModuleType: + """Load OpenSAK's metadata.py from *path*. + + From v1.22.0 on it imports opensak.macro.deprecation, which has no + OpenSAK imports of its own, so the two files work outside OpenSAK as a + pair. With *path* at /opensak/macro/metadata.py and deprecation.py + next to it (as the workflows download them), is put first on + sys.path so that import finds it. A parser in any other place (an older + one, which needs nothing else) is loaded as it is. + """ + path = path.resolve() + if path.parent.name == "macro" and path.parent.parent.name == "opensak": + src = str(path.parents[2]) + if src not in sys.path: + sys.path.insert(0, src) spec = importlib.util.spec_from_file_location("macro_metadata", path) if spec is None or spec.loader is None: raise SystemExit(f"Cannot load the OpenSAK header parser from {path}") @@ -176,7 +190,8 @@ def header(md: ModuleType, data: bytes | None, path: str) -> dict[str, Any]: "permissions": list(info.permissions) or ([md.NO_PERMISSIONS] if info.permissions_declared else []), "network": list(info.network), - # null with a parser from before the Headless key (API 4) + # null until OpenSAK's parser has the Headless key (planned, OpenSAK + # #1195; v1.22.0 doesn't have it yet) "headless": getattr(info, "runs_headless", None), } diff --git a/scripts/check_macro_pr.py b/scripts/check_macro_pr.py index ba8441d..d22fa7a 100644 --- a/scripts/check_macro_pr.py +++ b/scripts/check_macro_pr.py @@ -120,6 +120,20 @@ def read(self, rev: str, path: str) -> bytes | None: # --- helpers ----------------------------------------------------------------- def load_metadata_module(path: Path) -> ModuleType: + """Load OpenSAK's metadata.py from *path*. + + From v1.22.0 on it imports opensak.macro.deprecation, which has no + OpenSAK imports of its own, so the two files work outside OpenSAK as a + pair. With *path* at /opensak/macro/metadata.py and deprecation.py + next to it (as the workflows download them), is put first on + sys.path so that import finds it. A parser in any other place (an older + one, which needs nothing else) is loaded as it is. + """ + path = path.resolve() + if path.parent.name == "macro" and path.parent.parent.name == "opensak": + src = str(path.parents[2]) + if src not in sys.path: + sys.path.insert(0, src) spec = importlib.util.spec_from_file_location("macro_metadata", path) if spec is None or spec.loader is None: raise SystemExit(f"Cannot load the OpenSAK header parser from {path}") diff --git a/tests/test_build_index.py b/tests/test_build_index.py index b7ec828..1f5fe46 100644 --- a/tests/test_build_index.py +++ b/tests/test_build_index.py @@ -21,7 +21,7 @@ import build_index as bi -METADATA = Path(os.environ.get("MACRO_METADATA", ROOT / ".tools" / "macro_metadata.py")) +METADATA = Path(os.environ.get("MACRO_METADATA", ROOT / ".tools" / "opensak" / "macro" / "metadata.py")) if not METADATA.is_file(): pytest.skip(f"OpenSAK header parser not found at {METADATA} (see " "tests/test_check_macro_pr.py)", allow_module_level=True) diff --git a/tests/test_check_macro_pr.py b/tests/test_check_macro_pr.py index 989b07c..a84d8a4 100644 --- a/tests/test_check_macro_pr.py +++ b/tests/test_check_macro_pr.py @@ -3,13 +3,17 @@ They need OpenSAK's header parser. In CI the workflow downloads it; locally: - ref=$(grep -v '^#' .github/opensak-ref | head -n 1) - mkdir -p .tools && curl -fsSL -o .tools/macro_metadata.py \ - https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/$ref/src/opensak/macro/metadata.py + ref=$(grep -v '^#' .github/opensak-ref | head -n 1 | tr -d '[:space:]') + mkdir -p .tools/opensak/macro + for f in metadata.py deprecation.py; do + curl -fsSL -o .tools/opensak/macro/$f \ + https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/$ref/src/opensak/macro/$f + done pip install "lupa>=2.0,<3" pytest pytest -v tests/ -Set MACRO_METADATA to use a parser somewhere else. +Set MACRO_METADATA to use a parser somewhere else (e.g. an OpenSAK +checkout's src/opensak/macro/metadata.py). """ from __future__ import annotations @@ -26,7 +30,7 @@ import check_macro_pr as cm -METADATA = Path(os.environ.get("MACRO_METADATA", ROOT / ".tools" / "macro_metadata.py")) +METADATA = Path(os.environ.get("MACRO_METADATA", ROOT / ".tools" / "opensak" / "macro" / "metadata.py")) if not METADATA.is_file(): pytest.skip(f"OpenSAK header parser not found at {METADATA} (see module docstring)", allow_module_level=True) From 309e986aff91836ec3716e58aad06efbbf7fd573 Mon Sep 17 00:00:00 2001 From: nagisml Date: Wed, 18 Nov 2026 22:19:35 +0100 Subject: [PATCH 3/3] Workflows: make metadata.py's imports optional; fix auto-merge "Check macro" reads .github/opensak-ref from main, so a pull request that adds deprecation.py to the download list was checked against main's beta.9 ref, where that file doesn't exist (404). metadata.py stays a required download. The modules it imports (PARSER_DEPS: deprecation.py) are downloaded if the ref has them; an older ref doesn't need them, and if a needed one is missing, loading the parser still fails with ModuleNotFoundError. auto-merge.yml downloaded metadata.py on its own. With v1.22.0 the check there would fail and, because of "|| true", quietly make every pull request ineligible for auto-merge. It now uses the same opensak/macro/ layout and downloads deprecation.py too. --- .github/workflows/auto-merge.yml | 13 +++++++++---- .github/workflows/check.yml | 24 ++++++++++++++++-------- .github/workflows/index.yml | 15 ++++++++++----- MAINTAINERS.md | 4 ++-- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index 9ce4e14..6b2090c 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -70,16 +70,21 @@ jobs: run: | pip install "lupa>=2.0,<3" ref=$(grep -v '^#' .github/opensak-ref | head -n 1 | tr -d '[:space:]') - mkdir -p "$RUNNER_TEMP/tools" - curl -fsSL -o "$RUNNER_TEMP/tools/macro_metadata.py" \ - "https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/${ref}/src/opensak/macro/metadata.py" + dir="$RUNNER_TEMP/opensak-src/opensak/macro" + url="https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/${ref}/src/opensak/macro" + mkdir -p "$dir" + curl -fsSL -o "$dir/metadata.py" "$url/metadata.py" + # metadata.py's imports, as PARSER_DEPS in check.yml + for f in deprecation.py; do + curl -fsSL -o "$dir/$f" "$url/$f" || echo "::notice::No $f at $ref (not needed there)" + done - name: Check the pull request if: steps.fetch.outputs.current == 'true' run: | python -I scripts/check_macro_pr.py \ --repo . --base "$(git rev-parse HEAD)" --head "$HEAD_SHA" \ - --metadata "$RUNNER_TEMP/tools/macro_metadata.py" \ + --metadata "$RUNNER_TEMP/opensak-src/opensak/macro/metadata.py" \ --json "$RUNNER_TEMP/check.json" || true - name: Decide diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 8eb8117..70195cb 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -29,9 +29,11 @@ concurrency: env: LUPA: "lupa>=2.0,<3" - # metadata.py imports deprecation.py (from v1.22.0); both are needed. PARSER_URL: https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/%s/src/opensak/macro - PARSER_FILES: metadata.py deprecation.py + # The modules metadata.py imports (deprecation.py from v1.22.0). An older + # ref doesn't have them and doesn't need them; if one is missing that is + # needed, loading the parser fails with ModuleNotFoundError. + PARSER_DEPS: deprecation.py jobs: check-macro: @@ -58,9 +60,12 @@ jobs: from_main() { git show "$BASE:$1" 2>/dev/null || cat "$1"; } from_main scripts/check_macro_pr.py > "$RUNNER_TEMP/tools/check_macro_pr.py" ref=$(from_main .github/opensak-ref | grep -v '^#' | head -n 1 | tr -d '[:space:]') - mkdir -p "$RUNNER_TEMP/opensak-src/opensak/macro" - for f in $PARSER_FILES; do - curl -fsSL -o "$RUNNER_TEMP/opensak-src/opensak/macro/$f" "$(printf "$PARSER_URL" "$ref")/$f" + dir="$RUNNER_TEMP/opensak-src/opensak/macro" + url="$(printf "$PARSER_URL" "$ref")" + mkdir -p "$dir" + curl -fsSL -o "$dir/metadata.py" "$url/metadata.py" + for f in $PARSER_DEPS; do + curl -fsSL -o "$dir/$f" "$url/$f" || echo "::notice::No $f at $ref (not needed there)" done - name: Check macro @@ -90,9 +95,12 @@ jobs: - name: Get OpenSAK header parser run: | ref=$(grep -v '^#' .github/opensak-ref | head -n 1 | tr -d '[:space:]') - mkdir -p ".tools/opensak/macro" - for f in $PARSER_FILES; do - curl -fsSL -o ".tools/opensak/macro/$f" "$(printf "$PARSER_URL" "$ref")/$f" + dir=".tools/opensak/macro" + url="$(printf "$PARSER_URL" "$ref")" + mkdir -p "$dir" + curl -fsSL -o "$dir/metadata.py" "$url/metadata.py" + for f in $PARSER_DEPS; do + curl -fsSL -o "$dir/$f" "$url/$f" || echo "::notice::No $f at $ref (not needed there)" done - name: Run tests diff --git a/.github/workflows/index.yml b/.github/workflows/index.yml index 90f1ac4..f90e18c 100644 --- a/.github/workflows/index.yml +++ b/.github/workflows/index.yml @@ -27,9 +27,11 @@ concurrency: env: INDEX_BRANCH: index - # metadata.py imports deprecation.py (from v1.22.0); both are needed. PARSER_URL: https://raw.githubusercontent.com/OpenSAK-Org/OpenSAK/%s/src/opensak/macro - PARSER_FILES: metadata.py deprecation.py + # The modules metadata.py imports (deprecation.py from v1.22.0). An older + # ref doesn't have them and doesn't need them; if one is missing that is + # needed, loading the parser fails with ModuleNotFoundError. + PARSER_DEPS: deprecation.py jobs: index: @@ -51,9 +53,12 @@ jobs: - name: Get OpenSAK header parser run: | ref=$(grep -v '^#' .github/opensak-ref | head -n 1 | tr -d '[:space:]') - mkdir -p "$RUNNER_TEMP/opensak-src/opensak/macro" - for f in $PARSER_FILES; do - curl -fsSL -o "$RUNNER_TEMP/opensak-src/opensak/macro/$f" "$(printf "$PARSER_URL" "$ref")/$f" + dir="$RUNNER_TEMP/opensak-src/opensak/macro" + url="$(printf "$PARSER_URL" "$ref")" + mkdir -p "$dir" + curl -fsSL -o "$dir/metadata.py" "$url/metadata.py" + for f in $PARSER_DEPS; do + curl -fsSL -o "$dir/$f" "$url/$f" || echo "::notice::No $f at $ref (not needed there)" done - name: Build index.json diff --git a/MAINTAINERS.md b/MAINTAINERS.md index dd75d25..3f950c2 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -73,8 +73,8 @@ The checker also uses two internal helpers of the parser (`_header_lines`, `_KEY_LINE`) to see which keys are actually written. If "Test checker" fails after a bump, look there first. If it fails with `ModuleNotFoundError`, `metadata.py` has gained another top-level import: add that file to -`PARSER_FILES` in `check.yml` and `index.yml` (and to the download in -`tests/test_check_macro_pr.py`). +`PARSER_DEPS` in `check.yml` and `index.yml`, to the list in +`auto-merge.yml` and to the download in `tests/test_check_macro_pr.py`. ## The macro index