diff --git a/.github/scripts/build-static.sh b/.github/scripts/build-static.sh new file mode 100755 index 0000000..35472d8 --- /dev/null +++ b/.github/scripts/build-static.sh @@ -0,0 +1,173 @@ +#!/bin/sh +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell +# +# Build a fully static x86_64-linux-musl `ddraig` binary: +# +# Ddraig.idr --(Idris 2 RefC backend)--> C --(zig cc, musl, -static)--> dist/ddraig +# +# linked against GMP and the Idris 2 C runtimes, both rebuilt here for musl +# because the copies in the image are built for glibc. Why RefC and Zig, and +# what this does not yet guarantee: RELEASING.adoc. +# +# DRAFT: written without running it. Run from the repository root inside the +# digest-pinned idris2-pack image that ci.yml and release.yml use. Needs idris2, +# git, make and gcc (in the image) plus curl, xz and m4 (installed by +# release.yml). readelf comes with gcc's binutils. + +set -eu + +# --- Pins --------------------------------------------------------------------- +# Zig: version and SHA-256 of the x86_64-linux tarball, from +# https://ziglang.org/download/index.json, recorded 2026-10-13. +ZIG_VERSION=0.16.0 +ZIG_SHA256=70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00 +# GMP: SHA-256 of gmp-6.3.0.tar.xz, computed 2026-10-13 from both gmplib.org and +# ftp.gnu.org (identical). The GNU mirror is used because gmplib.org has +# rate-limited CI runners in the past. +GMP_VERSION=6.3.0 +GMP_SHA256=a3c2b80201b89e68616f4ad30bc66aee4927c3ce50e33929ca819d5c43538898 +# The Idris 2 commit inside the pinned image (its pack state holds +# install//idris2). The C runtimes are rebuilt from this commit, so it +# must be the compiler's own commit; check_idris enforces that. +IDRIS2_COMMIT=6ca00e72e32849cb35e5c262904e9c3cffc1a9a4 + +TARGET=x86_64-linux-musl +ROOT=$(pwd) +WORK="$ROOT/build/static" +DIST="$ROOT/dist" + +# log MESSAGE...: print a build progress message to stdout. +log() { printf '==> %s\n' "$*"; } +# die MESSAGE...: print a build error to stderr and exit with status 1. +die() { printf 'build-static: %s\n' "$*" >&2; exit 1; } + +# fetch URL SHA256 DEST: download over HTTPS to DEST and verify its SHA-256. +# Overwrites DEST; leaves it in place on checksum failure. Download or checksum +# failures terminate the script with the failing command's exit status. +fetch() { + curl --fail --silent --show-error --location --proto '=https' --tlsv1.2 \ + --output "$3" "$1" + printf '%s %s\n' "$2" "$3" | sha256sum --check --strict - +} + +# The runtime sources must belong to the compiler that generates the C: +# 1. the idris2 on PATH reports the pinned commit in its version string; +# 2. pack has that commit installed; +# 3. every RefC and C-support header it installed equals the source's. +# Uses SRC as the source tree and PACK_STATE_DIR as pack's state directory. +# Missing pack state or a version, layout or header mismatch exits the script +# with status 1; idris2 --version failures propagate their exit status. +check_idris() { + short=$(printf '%s' "$IDRIS2_COMMIT" | cut -c1-9) + version=$(idris2 --version) + case "$version" in + *"$short"*) ;; + *) die "idris2 reports '$version', not commit $short; update IDRIS2_COMMIT when the image digest changes" ;; + esac + [ -n "${PACK_STATE_DIR:-}" ] \ + || die "PACK_STATE_DIR is unset; run inside the pinned idris2-pack image" + set -- "$PACK_STATE_DIR/install/$IDRIS2_COMMIT/idris2"/idris2-* + [ $# -eq 1 ] && [ -d "$1" ] \ + || die "expected one idris2- directory for commit $IDRIS2_COMMIT, found: $*" + inst=$1 + for h in "$SRC"/support/refc/*.h "$SRC"/support/c/*.h; do + sub=$(basename "$(dirname "$h")") + [ -f "$inst/support/$sub/$(basename "$h")" ] \ + || die "the image has no support/$sub/$(basename "$h")" + cmp -s "$h" "$inst/support/$sub/$(basename "$h")" \ + || die "support/$sub/$(basename "$h") differs between the image and Idris 2 $IDRIS2_COMMIT" + done +} + +mkdir -p "$WORK" "$DIST" + +# --- Zig ---------------------------------------------------------------------- +log "Zig $ZIG_VERSION" +zig_dir="$WORK/zig-x86_64-linux-$ZIG_VERSION" +if [ ! -x "$zig_dir/zig" ]; then + fetch "https://ziglang.org/download/$ZIG_VERSION/zig-x86_64-linux-$ZIG_VERSION.tar.xz" \ + "$ZIG_SHA256" "$WORK/zig.tar.xz" + tar -xJf "$WORK/zig.tar.xz" -C "$WORK" +fi +ZIG="$zig_dir/zig" +"$ZIG" version +# ccm ARG...: invoke Zig's C compiler for TARGET at -O2, forwarding ARG. +ccm() { "$ZIG" cc -target "$TARGET" -O2 "$@"; } + +# --- Idris 2 runtime sources, at the compiler's commit ------------------------ +log "Idris 2 sources at $IDRIS2_COMMIT" +SRC="$WORK/idris2-src" +if [ ! -d "$SRC/.git" ]; then + git init --quiet "$SRC" + git -C "$SRC" fetch --quiet --depth 1 https://github.com/idris-lang/Idris2 "$IDRIS2_COMMIT" + git -C "$SRC" checkout --quiet FETCH_HEAD +fi +[ "$(git -C "$SRC" rev-parse HEAD)" = "$IDRIS2_COMMIT" ] \ + || die "checked out the wrong Idris 2 commit" +check_idris + +# --- GMP, static, for musl ---------------------------------------------------- +log "GMP $GMP_VERSION for $TARGET" +GMP="$WORK/gmp-$TARGET" +if [ ! -f "$GMP/lib/libgmp.a" ]; then + fetch "https://ftp.gnu.org/gnu/gmp/gmp-$GMP_VERSION.tar.xz" "$GMP_SHA256" "$WORK/gmp.tar.xz" + rm -rf "$WORK/gmp-$GMP_VERSION" + tar -xJf "$WORK/gmp.tar.xz" -C "$WORK" + ( + cd "$WORK/gmp-$GMP_VERSION" + # Portable C only (--disable-assembly): Ddraig's Integer use is trivial, + # and generic C avoids assembler differences between gcc and zig cc. + ./configure --prefix="$GMP" --disable-shared --enable-static --disable-assembly \ + CC="$ZIG cc -target $TARGET" AR="$ZIG ar" RANLIB="$ZIG ranlib" CFLAGS=-O2 + make -j"$(nproc)" + make check + make install + ) +fi + +# --- Idris 2 C runtimes, static, for musl ------------------------------------- +# Same sources and flags as support/c/Makefile and support/refc/Makefile +# (every *.c in the directory, -O2), compiled with zig cc instead of gcc. +log "Idris 2 C runtimes for $TARGET" +RT="$WORK/runtime-$TARGET" +rm -rf "$RT" +mkdir -p "$RT" +for f in "$SRC"/support/c/*.c; do + ccm -I"$SRC/support/c" -I"$GMP/include" -c "$f" -o "$RT/c_$(basename "$f" .c).o" +done +"$ZIG" ar rcs "$RT/libidris2_support.a" "$RT"/c_*.o +for f in "$SRC"/support/refc/*.c; do + ccm -I"$SRC/support/refc" -I"$SRC/support/c" -I"$GMP/include" \ + -c "$f" -o "$RT/refc_$(basename "$f" .c).o" +done +"$ZIG" ar rcs "$RT/libidris2_refc.a" "$RT"/refc_*.o + +# --- Ddraig -> C -> static binary --------------------------------------------- +# RefC writes build/exec/ddraig-refc.c and then links a glibc binary with the +# image's gcc. We keep only the C file and link it ourselves. +log "RefC: Ddraig.idr -> C" +idris2 --cg refc Ddraig.idr -o ddraig-refc +CSRC="$ROOT/build/exec/ddraig-refc.c" +[ -f "$CSRC" ] || die "RefC did not leave $CSRC" + +log "Link static $TARGET binary" +ccm -static -s \ + -I"$SRC/support/refc" -I"$SRC/support/c" -I"$GMP/include" \ + "$CSRC" "$RT/libidris2_refc.a" "$RT/libidris2_support.a" "$GMP/lib/libgmp.a" -lm \ + -o "$DIST/ddraig" + +log "Check the binary is fully static" +# Capture readelf's output first: piping it straight into grep would let a +# readelf failure pass silently, because sh has no pipefail. +command -v readelf > /dev/null || die "readelf not found (binutils)" +headers=$(readelf --wide --program-headers "$DIST/ddraig") +dynamic=$(readelf --wide --dynamic "$DIST/ddraig") +case "$headers" in + *INTERP*) die "dist/ddraig asks for a dynamic loader (PT_INTERP)" ;; +esac +case "$dynamic" in + *NEEDED*) die "dist/ddraig needs shared libraries (DT_NEEDED)" ;; +esac +"$DIST/ddraig" --version +sha256sum "$DIST/ddraig" diff --git a/.github/scripts/check-idris-version.sh b/.github/scripts/check-idris-version.sh new file mode 100755 index 0000000..7687e08 --- /dev/null +++ b/.github/scripts/check-idris-version.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell +# +# Fail unless the idris2 on PATH is at least the minimum version this project +# supports (README.adoc: "Requires idris2 >= 0.8.0"). +# +# usage: check-idris-version.sh [MINIMUM] (default 0.8.0) +# +# DRAFT: not yet run. See RELEASING.adoc. + +set -eu + +min=${1:-0.8.0} +out=$(idris2 --version) +printf '%s\n' "$out" + +# "Idris 2, version 0.8.0-6ca00e72e" -> "0.8.0" +ver=$(printf '%s\n' "$out" | + sed -n 's/.*version \([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\).*/\1/p') +if [ -z "$ver" ]; then + printf 'check-idris-version: cannot read a version from: %s\n' "$out" >&2 + exit 1 +fi + +lowest=$(printf '%s\n%s\n' "$min" "$ver" | sort -V | head -n 1) +if [ "$lowest" != "$min" ]; then + printf 'check-idris-version: Idris 2 %s is older than the minimum %s\n' "$ver" "$min" >&2 + exit 1 +fi +printf 'OK: Idris 2 %s >= %s\n' "$ver" "$min" diff --git a/.github/scripts/test-static.sh b/.github/scripts/test-static.sh new file mode 100755 index 0000000..e3878b5 --- /dev/null +++ b/.github/scripts/test-static.sh @@ -0,0 +1,164 @@ +#!/bin/sh +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell +# +# Check the static release binary (RefC backend) against the reference binary +# (Chez backend, the one CI tests). They must behave identically: +# +# 1. every CI test subcommand passes and prints exactly what the reference prints; +# 2. validate-state accepts this repo's STATE descriptile; +# 3. building examples/ and a fixture site gives byte-identical output trees; +# 4. a large page builds without crashing (C stack depth; see RELEASING.adoc) +# and also matches the reference. +# +# usage: test-static.sh STATIC_BINARY REFERENCE_BINARY +# +# DRAFT: not yet run. See RELEASING.adoc. +# +# Run from the repository root. The fixture site written below adds what +# examples/ lacks: a nested page, a draft, and more Markdown constructs. +# +# Known false alarm: the attestation file records the build day, so if the two +# builds straddle 00:00 UTC their output trees differ. Re-run. + +set -eu + +[ $# -eq 2 ] || { echo "usage: $0 STATIC_BINARY REFERENCE_BINARY" >&2; exit 2; } +static=$1 +ref=$2 +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +# fail MESSAGE...: print a failed check to stderr and exit with status 1. +fail() { printf 'test-static: FAIL: %s\n' "$*" >&2; exit 1; } +# pass MESSAGE...: print a successful check to stdout. +pass() { printf 'test-static: ok: %s\n' "$*"; } + +# same_output NAME CMD...: run CMD with each binary; both must succeed and +# print the same bytes. +same_output() { + name=$1 + shift + "$static" "$@" > "$work/$name.static" 2>&1 || fail "$name: static binary exited non-zero" + "$ref" "$@" > "$work/$name.ref" 2>&1 || fail "$name: reference binary exited non-zero" + if ! cmp -s "$work/$name.ref" "$work/$name.static"; then + diff -u "$work/$name.ref" "$work/$name.static" | head -n 60 + fail "$name: output differs from the reference" + fi + pass "$name" +} + +# same_site NAME SRC: build SRC with each binary and compare the output trees. +same_site() { + name=$1 + src=$2 + "$static" build "$src" "$work/$name.out.static" https://example.org > "$work/$name.log.static" 2>&1 \ + || { cat "$work/$name.log.static"; fail "$name: static build failed"; } + "$ref" build "$src" "$work/$name.out.ref" https://example.org > "$work/$name.log.ref" 2>&1 \ + || { cat "$work/$name.log.ref"; fail "$name: reference build failed"; } + if ! diff -r "$work/$name.out.ref" "$work/$name.out.static" > "$work/$name.diff"; then + head -n 60 "$work/$name.diff" + fail "$name: output tree differs from the reference" + fi + pass "$name ($(find "$work/$name.out.static" -type f | wc -l) files identical)" +} + +# --- 1 and 2: CLI and test subcommands --------------------------------------- +same_output version --version +for t in test-markdown test-frontmatter test-full test-state test-attest; do + same_output "$t" "$t" +done +same_output validate-state validate-state .machine_readable/descriptiles/STATE.a2ml + +# --- 3: examples/ and a fixture site ------------------------------------------- +same_site examples examples +# Every page has exactly one h1 and no skipped heading levels, or the build's +# accessibility certification (O3) fails it. +site="$work/site" +mkdir -p "$site/blog" "$site/public" +cat > "$site/index.md" <<'EOF' +--- +title: Home +date: 2026-01-15 +description: Fixture home page +site: Fixture +tags: [idris, ssg] +--- + +# Home + +Some **bold**, *italic*, ~~struck~~ and `code` text with a [link](https://example.org/x). + +![A red dragon](dragon.png) + +## Lists + +- one +- two + - nested +1. first +2. second + +> a quote + +| a | b | +|---|---| +| 1 | 2 | + +```idris +main : IO () +``` + +--- +EOF +cat > "$site/blog/post.md" <<'EOF' +--- +title: A post +date: 2026-02-01T09:30:00Z +description: Second page +--- + +# A post + +## Part one + +### Detail + +Text & entities < > "quotes". +EOF +cat > "$site/draft.md" <<'EOF' +--- +title: Not yet +draft: true +--- + +# Not yet +EOF +printf 'User-agent: *\nAllow: /\n' > "$site/public/robots.txt" +same_site fixture "$site" +[ ! -e "$work/fixture.out.static/draft.html" ] || fail "fixture: draft page was built" + +# --- 4: large page ------------------------------------------------------------- +# The C stack does not grow the way Chez's does, so deep non-tail recursion is +# the most likely way for the static binary to fail where the reference +# succeeds. The block parser recurses once per block, so this page has many +# short blocks. BIG_BLOCKS (default 5000, a very long article) sets the size; +# the fork dry-run records where each binary actually fails (RELEASING.adoc). +# One long paragraph would be the wrong stress test: the inline parser re-packs +# the rest of the paragraph at every delimiter, which is quadratic in its length. +n=${BIG_BLOCKS:-5000} +big="$work/big" +mkdir -p "$big" +{ + printf '%s\n' '---' 'title: Large' 'date: 2026-03-01' '---' '' '# Large' '' + awk -v n="$n" 'BEGIN { + for (i = 0; i < n; i++) { + if (i % 50 == 0) printf "## Section %d\n\n", i / 50 + printf "Paragraph %d has **bold**, `code` and a [link](https://example.org/%d).\n\n", i, i + } + }' +} > "$big/index.md" +pass "large page: $n blocks, $(wc -c < "$big/index.md") bytes" +same_site large "$big" + +echo "test-static: all checks passed" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..4ef4992 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,165 @@ +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Release: a static x86_64 Linux binary, its SHA-256 checksum, and a signed SLSA +# build provenance attestation, published as a GitHub release. +# +# ⚠ DRAFT — NEVER RUN. Written 2026-10-13 without a Linux toolchain to test it. +# Do not push a release tag to the upstream repository until the fork dry-run +# recorded in RELEASING.adoc has passed. +# +# Triggers: +# push of a tag vX.Y.Z -> build, test, checksum, attest, verify, publish. +# workflow_dispatch -> build and test only; never attests or publishes. +# +# Supply chain: actions are pinned to full commit SHAs; the build container is +# the same digest-pinned image as ci.yml (bump both together); Zig, GMP and the +# Idris 2 runtime sources are pinned by checksum or commit inside +# .github/scripts/build-static.sh. Template expressions reach shell code only +# through env, never by expansion inside run: blocks. +name: Release + +on: + push: + tags: ['v[0-9]+.[0-9]+.[0-9]+'] + workflow_dispatch: + inputs: + big_blocks: + description: "Blocks in the large-page check (find the stack limit during a dry-run)" + type: number + default: 5000 + +permissions: + contents: read + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +jobs: + build: + name: Build and test the static binary + runs-on: ubuntu-latest + timeout-minutes: 60 + container: + # Same digest as ci.yml: Ubuntu 24.04, Idris 2 0.8.0 at + # idris-lang/Idris2@6ca00e72 (RELEASING.adoc, "Toolchain and pinning"). + image: ghcr.io/stefan-hoeck/idris2-pack@sha256:f0758996a931fb35d9ecb1de273c4d59dabe2a09b433afc7e357f65a08b7e1ff + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Install build tools + # curl and xz fetch and unpack the pinned Zig and GMP tarballs; GMP's + # configure needs m4. These come unpinned from Ubuntu 24.04's archive; + # none of them ends up inside the binary. + run: | + apt-get update + apt-get install --yes --no-install-recommends ca-certificates curl xz-utils m4 + + - name: Check Idris 2 version (minimum 0.8.0) + run: sh .github/scripts/check-idris-version.sh 0.8.0 + + - name: Build the reference binary (Chez backend, exactly as CI does) + run: idris2 Ddraig.idr -o ddraig + + - name: Build the static binary (RefC backend, zig cc, musl) + run: sh .github/scripts/build-static.sh + + - name: Test the static binary against the reference + env: + BIG_BLOCKS: ${{ inputs.big_blocks || 5000 }} + run: sh .github/scripts/test-static.sh dist/ddraig build/exec/ddraig + + - name: Check the tag matches the binary's version + if: github.ref_type == 'tag' + run: | + want="ddraig ${GITHUB_REF_NAME#v}" + got=$(dist/ddraig --version) + if [ "$got" != "$want" ]; then + echo "tag $GITHUB_REF_NAME does not match '$got' (ddraigVersion in Ddraig.idr)" >&2 + exit 1 + fi + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ddraig-static + path: dist/ddraig + if-no-files-found: error + retention-days: 7 + + release: + name: Checksum, attest, verify and publish + needs: build + if: github.event_name == 'push' && github.ref_type == 'tag' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: write # create the GitHub release + id-token: write # OIDC token for the short-lived Sigstore certificate + attestations: write # store the attestation + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: ddraig-static + path: dist + + - name: Name the binary and write its checksum + id: asset + run: | + set -eu + bin="ddraig-${GITHUB_REF_NAME#v}-x86_64-linux-musl" + cd dist + mv ddraig "$bin" + chmod 0755 "$bin" # artifact upload drops the executable bit + sha256sum "$bin" > "$bin.sha256" + cat "$bin.sha256" + echo "bin=$bin" >> "$GITHUB_OUTPUT" + + - name: Attest build provenance + id: attest + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-path: dist/${{ steps.asset.outputs.bin }} + + - name: Verify exactly as RELEASING.adoc tells users to + env: + GH_TOKEN: ${{ github.token }} + BIN: ${{ steps.asset.outputs.bin }} + BUNDLE: ${{ steps.attest.outputs.bundle-path }} + run: | + set -eu + cd dist + cp "$BUNDLE" "$BIN.sigstore.json" + sha256sum --check --strict "$BIN.sha256" + # Online: the attestation stored on GitHub. + gh attestation verify "$BIN" --repo "$GITHUB_REPOSITORY" \ + --signer-workflow "$GITHUB_REPOSITORY/.github/workflows/release.yml" \ + --source-ref "$GITHUB_REF" --deny-self-hosted-runners + # The bundle attached to the release. + gh attestation verify "$BIN" --bundle "$BIN.sigstore.json" --repo "$GITHUB_REPOSITORY" \ + --signer-workflow "$GITHUB_REPOSITORY/.github/workflows/release.yml" \ + --source-ref "$GITHUB_REF" --deny-self-hosted-runners + + - name: Publish the release + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + BIN: ${{ steps.asset.outputs.bin }} + run: | + set -eu + cat > notes.md < += Ddraig feeds and the feed item model +:toc: macro + +[WARNING] +==== +*Status: draft, not implemented.* This is the specification for work package 2, +milestone 5. Only the existing site-wide Atom feed exists today; RSS, JSON Feed, +section feeds and tag feeds do not. Nothing new here is tested. Sections, +authors, languages and some front matter keys depend on work package 1 and are +marked _(WP1)_. +==== + +Ddraig writes feeds in three formats from one internal *feed item model*. The +`superfeedreader` project consumes these feeds, so this document states what a +consumer may rely on (<>) as well as how each format is produced. + +toc::[] + +[#current] +== What the build does today + +This is the behaviour of `genFeed` in `Ddraig.idr` at v0.2.0. It is recorded +here because milestone 5 must keep Atom output stable, and any change to it must +show up as a golden-test diff. + +* One Atom 1.0 feed at `/feed.xml`. +* *Entries:* every built (non-draft) page, in the order the source tree was + walked. That order comes from the directory listing and is *not sorted*, so it + can differ between filesystems. +* *Feed title and author name:* the first non-empty `brand`/`site` front matter + value found, otherwise `Ddraig Site`. +* *Feed id:* the absolute URL of `/feed.xml` when a base URL is given, otherwise + `urn:ddraig:/feed.xml`. +* *Feed ``:* the greatest page `date`, compared as strings, converted + to RFC 3339. If no page has a date, today's UTC date at `T00:00:00Z`. +* `` only when a base URL is given. +* `Ddraig SSG 0.2.0`. +* *Per entry:* ``; `<id>` (absolute URL or `urn:ddraig:<path>`); + `<link href>` (absolute, or site-relative without a base URL); `<updated>` + (the page date in RFC 3339, else the feed's `<updated>`); `<summary>` (the + description, emitted even when empty). +* No `<published>`, `<content>`, `<category>`, per-entry `<author>` or `xml:lang`. +* No limit on the number of entries. + +*Test coverage today: none.* The CI test subcommands (`test-markdown`, +`test-frontmatter`, `test-full`, `test-state`, `test-attest`) never generate a +feed. The CI `build` job does write one, from `examples/`, and publishes it to +GitHub Pages without any check. So the first commit of milestone 5 must +add a golden test of the current Atom output for a fixture with a deterministic +walk order (a single directory level, or a single page), before anything else +changes. The golden file normalises the `<generator>` line, which carries the +version number. + +Known defects in the current feed, to be fixed only with a golden diff: + +. `rfc3339` appends `T00:00:00Z` to any date without a `T` and passes any date + with a `T` through unchecked. So `2024-01-15T10:00` (no zone) and + `15 Jan 2024` both produce invalid timestamps. +. Front matter values keep their quotes, so `date: "2024-01-15"` becomes + `"2024-01-15"T00:00:00Z`. +. `<updated>` values are not XML-escaped, so a `&` or `<` in a date breaks the + feed. +. The "latest date" is a string comparison, which is wrong once dates carry + different UTC offsets. +. Entry order is unspecified (see above). +. An undated entry takes the feed's `<updated>`. It therefore looks like the + newest entry, and it changes whenever another page's date changes. +. `<summary>` is emitted even when empty, which says "this entry has an empty + summary" rather than "no summary". +. Characters that XML 1.0 forbids (U+0000–U+0008, U+000B, U+000C, + U+000E–U+001F) are not removed, so a stray control character in a title makes + the whole feed invalid. +. With no dated pages, the output depends on the day the build runs. + +[#model] +== The feed item model + +Every feed format is a rendering of the same two records. Field names below are +the model's; <<mapping>> shows where each one goes in each format. + +=== FeedItem + +[cols="1,1,3",options="header"] +|=== +| Field | Type | Source and rules +| `id` | IRI | The absolute permalink when a base URL is set, otherwise + `urn:ddraig:<path>`. A pure function of base URL and output path (the same rule + as today's Atom `<id>`). +| `url` | URL | Absolute permalink. Site-relative only in Atom without a base URL. +| `title` | text | Front matter `title`; falls back to the source path, as today. +| `published` | timestamp? | Front matter `date`. +| `updated` | timestamp? | Front matter `updated` _(WP1: new key)_, else `published`. +| `summary` | text? | Front matter `description`. +| `content` | HTML? | The rendered page body. Only when `feeds.full_content` is on. +| `authors` | list of author | Front matter `author`, else the site author _(WP1)_. +| `tags` | list of tag | Front matter `tags`. A tag has a `label` (as written) and a + `term` (its `slugify` form). +| `section` | name? | The page's section _(WP1)_. +| `lang` | language tag? | The page's language _(WP1)_. +|=== + +=== Feed + +[cols="1,3",options="header"] +|=== +| Field | Rules +| `id` | Absolute URL of the feed file, else `urn:ddraig:<feed path>` (Atom only). +| `title` | Site title _(WP1 config)_, else the current `brand` rule. Section and tag + feeds append ` — <section>` or ` — #<tag label>`. +| `home` | Absolute URL of the site root (or the section or tag page). +| `self` | Absolute URL of this feed file. +| `description` | Site description _(WP1)_, else `<title> feed`. +| `updated` | The greatest `updated` over the feed's dated items, compared as + instants. If none of this feed's items has a date, the greatest `updated` + over the whole site. Only if no page in the site has a date: + `SOURCE_DATE_EPOCH` when set, otherwise today, as now. +| `lang` | Site language _(WP1)_. +| `authors` | Site author _(WP1)_, else `title` (today's rule). +| `items` | See <<selection>>. +|=== + +=== Timestamps + +Accepted front matter forms _(WP1 may widen this)_: + +* `YYYY-MM-DD` means midnight UTC on that day; +* `YYYY-MM-DDTHH:MM:SSZ`; +* `YYYY-MM-DDTHH:MM:SS+HH:MM` or `-HH:MM`. + +Anything else is a build error with file, line and column (milestone 8), not a +pass-through. Atom and JSON Feed use RFC 3339. RSS uses the RFC 822 date form +with a four-digit year and a numeric zone, e.g. `Mon, 15 Jan 2024 00:00:00 +0000`. +The weekday is computed from days since 1970-01-01, which was a Thursday. + +[#selection] +== Which pages, in which order + +. *Candidates:* every built content page. Drafts are never built. Alias pages, + the 404 page, listing pages and machine views are not candidates. +. *Undated pages:* included in site-wide feeds when `feeds.include_undated` is + true. The default is `true`, because that is what today's Atom feed does. +. *Order:* `updated` descending, then `id` ascending. Undated items sort last. +. *Limit:* `feeds.limit` (default: unlimited, as today) applies after ordering. + +Membership: + +* *Site feeds:* all selected pages. +* *Section feeds:* pages in that section _(WP1)_. +* *Tag feeds:* pages carrying the tag. Tags whose `term` collides (e.g. `Rust` + and `rust`) are merged into one feed with a build warning; the first label in + sorted order is used. + +== Files + +[cols="2,1,1,1",options="header"] +|=== +| Scope | Atom | RSS 2.0 | JSON Feed 1.1 +| Site | `/feed.xml` (exists) | `/rss.xml` | `/feed.json` +| Section | `/<section>/feed.xml` | `/<section>/rss.xml` | `/<section>/feed.json` +| Tag | `/tags/<term>/feed.xml` | `/tags/<term>/rss.xml` | `/tags/<term>/feed.json` +|=== + +* *RSS and JSON Feed are written only with a base URL.* This is Ddraig's policy, + not a requirement of both formats. RSS 2.0's `link` elements must be URLs, + and a relative URL is useless to a reader that fetched the feed elsewhere. + JSON Feed 1.1 makes its URLs optional, but a feed without them gives readers + nothing to link to. Without a base URL the build skips both, with a warning. + Atom keeps its `urn:` fallback. +* JSON Feed is optional: `feeds.json = true` turns it on (default off). +* Section and tag feeds: `feeds.sections` and `feeds.tags` (default off). +* If a feed path collides with a content page's output path, the build fails + (milestone 8 diagnostics). + +Pages advertise their feeds with `<link rel="alternate">` in `<head>`. This +changes every page's HTML, so it lands behind a golden test _(WP1 templates)_. + +[#mapping] +== Format mapping + +=== Feed level + +[cols="1,2,2,2",options="header"] +|=== +| Model | Atom | RSS 2.0 (`channel`) | JSON Feed 1.1 +| `id` | `id` | — | — +| `title` | `title` | `title` | `title` +| `home` | `link rel="alternate"` _(new)_ | `link` | `home_page_url` +| `self` | `link rel="self"` | `atom:link rel="self"` | `feed_url` +| `description` | `subtitle` _(new)_ | `description` (required) | `description` +| `updated` | `updated` | — (see notes) | — +| `lang` | `xml:lang` _(new)_ | `language` | `language` +| `authors` | `author/name` | — | `authors[].name` +| generator | `generator` | `generator` | — +| version | — | `rss version="2.0"` | `version` = `https://jsonfeed.org/version/1.1` +|=== + +=== Item level + +[cols="1,2,2,2",options="header"] +|=== +| Model | Atom `entry` | RSS 2.0 `item` | JSON Feed `items[]` +| `id` | `id` | `guid`; `isPermaLink="true"` (always absolute here) | `id` +| `url` | `link href` | `link` | `url` +| `title` | `title` | `title` | `title` +| `published` | `published` _(new)_ | `pubDate` | `date_published` +| `updated` | `updated` (required; undated items get `1970-01-01T00:00:00Z` _(changed)_) | — | `date_modified` +| `summary` | `summary`, omitted when there is none _(changed)_ | `description` (plain text, escaped) | `summary` +| `content` | `content type="html"` | `content:encoded` | `content_html` +| `authors` | `author/name` | `dc:creator` | `authors[].name` +| `tags` | `category term="<term>" label="<label>"` _(new)_ | `category` (label) | `tags` (labels) +| `lang` | `xml:lang` | — | `language` +|=== + +Notes: + +* RSS has no element for the feed's `updated`. Its `lastBuildDate` means "the + last time the content of the channel changed". The newest item date is not + that: adding a back-dated page, or editing a page without changing its + dates, changes the channel without moving it. The build cannot know the true + time deterministically, so `lastBuildDate` (and the channel `pubDate`) are + omitted. Both are optional in RSS 2.0. +* RSS 2.0's own `author` element must hold an e-mail address, so names go in + Dublin Core `dc:creator` (namespace `http://purl.org/dc/elements/1.1/`). +* `content:encoded` uses namespace `http://purl.org/rss/1.0/modules/content/` and + holds entity-escaped HTML, not CDATA, so no `]]>` splitting is needed. +* JSON Feed 1.1 requires `content_html` or `content_text` on every item. With + `full_content` off, `content_text` is the summary, or the title when there is + no summary. +* Elements marked _(new)_ or _(changed)_ change Atom output. Each lands behind a + golden-test diff. New elements go *after* the existing ones to keep that diff + small. +* *Why undated Atom entries get the epoch:* Atom requires `<updated>`. Today's + fallback (the feed's `<updated>`) makes an undated page look like the newest + entry, and moves it whenever another page's date changes. A fixed epoch is + stable, sorts undated entries last for consumers that sort by date themselves, + and is plainly a placeholder. RSS and JSON Feed simply omit the date. + +=== Examples + +Site at `https://example.org` with no site description or language configured, +and one dated, tagged page. Atom as milestone 5 would emit it, with the new +elements placed after the existing ones: + +[source,xml] +---- +<?xml version="1.0" encoding="UTF-8"?> +<feed xmlns="http://www.w3.org/2005/Atom"> + <title>Example + https://example.org/feed.xml + 2024-01-15T00:00:00Z + Example + + Ddraig SSG 0.3.0 + + Example feed + + Hello + https://example.org/blog/hello.html + + 2024-01-15T00:00:00Z + First post + 2024-01-15T00:00:00Z + + + +---- + +RSS 2.0: + +[source,xml] +---- + + + + Example + https://example.org/ + Example feed + Ddraig SSG 0.3.0 + + + Hello + https://example.org/blog/hello.html + https://example.org/blog/hello.html + Mon, 15 Jan 2024 00:00:00 +0000 + First post + Idris + + + +---- + +JSON Feed 1.1 (indented here; the file has no whitespace): + +[source,json] +---- +{ + "version": "https://jsonfeed.org/version/1.1", + "title": "Example", + "home_page_url": "https://example.org/", + "feed_url": "https://example.org/feed.json", + "description": "Example feed", + "authors": [{"name": "Example"}], + "items": [ + { + "id": "https://example.org/blog/hello.html", + "url": "https://example.org/blog/hello.html", + "title": "Hello", + "content_text": "First post", + "summary": "First post", + "date_published": "2024-01-15T00:00:00Z", + "date_modified": "2024-01-15T00:00:00Z", + "tags": ["Idris"] + } + ] +} +---- + +[#contract] +== Consumer contract (for superfeedreader) + +A consumer may rely on the following. + +. *Stable ids.* An item's id depends only on the base URL and the page's output + path. Editing the title, date, tags or body never changes it. Moving a page + changes it, and the moved page is a new item. Aliases _(WP1)_ never create items. +. *One id everywhere.* A page has the same id in every feed that contains it and + in every format: Atom `id` = RSS `guid` = JSON Feed `id`. Deduplicate across + site, section and tag feeds by id. +. *Honest `updated`.* An item's `updated` changes only when its own front + matter dates change: not when the site is rebuilt, and not when another page + changes. An undated item has no `updated` in RSS or JSON Feed. In Atom, which + requires one, it is the fixed placeholder `1970-01-01T00:00:00Z`; treat that + as "unknown". A *feed's* `updated` is its newest item date, or the site's + newest date if none of its items is dated. Only in a site with no dated pages + at all does it fall back to the build day (or `SOURCE_DATE_EPOCH`). +. *Same selection in every format.* For a given scope, the Atom, RSS and JSON + feeds list the same items in the same order. +. *Order is a courtesy.* Items are sorted (<>), but consumers should + sort by date themselves. +. *Limits apply after sorting*, so a limited feed holds the newest items. +. *Missing optional fields mean "unknown".* An absent summary is not an empty + summary. (Today's Atom feed emits an empty `` for pages with no + description. Milestone 5 omits it, as an intended golden-test diff.) +. *Changes are versioned here.* Any change to ids, selection or mapping is + listed in the changelog at the end of this document before it ships. + +== Escaping + +* *XML:* `&`, `<`, `>` and `"` are escaped as today (`strEscape`). Characters + that XML 1.0 forbids are replaced with U+FFFD. +* *JSON:* the escaping rules in `SEARCH-INDEX.adoc` ("JSON encoding"). + +== Validation (milestone 5) + +The brief allows recorded reference fixtures or an external validator in CI. +Decision: *recorded fixtures plus structural checks with C tools*, because the +W3C Feed Validator is written in Python, and Python is banned in this estate's +builds. + +* *Golden fixtures:* the exact expected feeds for a fixture site, committed and + compared byte for byte. +* *Atom:* `xmllint --noout --relaxng atom.rng` against the RELAX NG schema from + RFC 4287, appendix B. The RFC gives the compact syntax; the XML-syntax copy is + committed once and its provenance recorded beside it. +* *RSS 2.0:* there is no normative schema. `xmllint` checks that it is well + formed, and `xmllint --xpath` checks that the required channel and item + elements are present. +* *JSON Feed:* `jq` assertions on the required keys and on `content_html` or + `content_text` being present on every item. +* *Optional, one-off:* submit the committed fixtures to the W3C validator by + hand and record the result here. This is not run in CI. + +Neither tool adds Node, npm or Python. CI's test job runs inside the +idris2-pack container, which has neither, so the job installs them first +(`apt-get install --yes libxml2-utils jq`). + +== Planned tests + +* Golden test of today's Atom output, added *first*. +* Atom stays byte-identical for that fixture (with the `` version + normalised), except for diffs this document marks _(new)_ or _(changed)_. +* An undated page's Atom `` does not move when another page's date + changes. +* RSS and JSON golden files; schema and structure checks as above. +* The same ids appear in every format and scope. +* Drafts never appear. Undated pages follow `feeds.include_undated`. +* RSS and JSON are skipped, with a warning, when there is no base URL. +* Tag and section membership for a fixture with overlapping tags. +* `SOURCE_DATE_EPOCH` makes an all-undated site's feeds reproducible. + +== Open questions for WP1 + +* What a section is (top-level directory, or declared in config). +* The author model (site author and per-page authors). +* Per-language feeds: probably `//feed.xml` and so on, one per language. +* Whether `updated` becomes a front matter key. + +== Changelog + +* *Draft 1*: initial model. Atom output unchanged until the golden test lands. diff --git a/RELEASING.adoc b/RELEASING.adoc new file mode 100644 index 0000000..5521559 --- /dev/null +++ b/RELEASING.adoc @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell += Releasing and verifying Ddraig +:toc: macro + +[WARNING] +==== +*Status: draft, never run.* The workflow (`.github/workflows/release.yml`) and +its scripts (`.github/scripts/`) were written on 2026-10-13 on a machine with no +Linux Idris 2 toolchain, so *none of it has executed*. Facts about the pinned +toolchain below were checked by reading the container image's manifest and +layers, not by running the image. Do not push a release tag upstream until the +<> has passed and been recorded. +==== + +toc::[] + +== What a release contains + +A push of a tag `vX.Y.Z` produces a GitHub release with three files: + +[cols="2,3",options="header"] +|=== +| File | What it is +| `ddraig-X.Y.Z-x86_64-linux-musl` | Fully static Linux binary for x86_64. No + shared libraries, no Chez Scheme, no Idris 2 needed to run it. +| `ddraig-X.Y.Z-x86_64-linux-musl.sha256` | Its SHA-256 checksum, in the format + `sha256sum --check` reads. +| `ddraig-X.Y.Z-x86_64-linux-musl.sigstore.json` | The signed build provenance + attestation, as a Sigstore bundle. +|=== + +The attestation is also stored in GitHub's attestation store for the repository. + +[#verify] +== Verifying a release + +You need `sha256sum` (coreutils) and a current GitHub CLI (`gh attestation` +first appeared in 2.49; use a recent release so every flag below exists). + +[source,bash] +---- +V=0.3.0 +B=ddraig-$V-x86_64-linux-musl +gh release download v$V --repo hyperpolymath/ddraig-ssg --pattern "$B*" + +# 1. Integrity: the file is the one the release published. +sha256sum --check --strict $B.sha256 + +# 2. Provenance: built by this repository's release workflow, from tag v$V, +# on a GitHub-hosted runner. +gh attestation verify $B --repo hyperpolymath/ddraig-ssg \ + --signer-workflow hyperpolymath/ddraig-ssg/.github/workflows/release.yml \ + --source-ref refs/tags/v$V --deny-self-hosted-runners + +# 2b. The same check against the bundle attached to the release, instead of +# the copy in GitHub's attestation store. +gh attestation verify $B --bundle $B.sigstore.json --repo hyperpolymath/ddraig-ssg \ + --signer-workflow hyperpolymath/ddraig-ssg/.github/workflows/release.yml \ + --source-ref refs/tags/v$V --deny-self-hosted-runners + +chmod +x $B && ./$B --version +---- + +The release workflow runs steps 1, 2 and 2b on its own output before it +publishes, so the commands above are exercised on every release. + +What each check does and does not tell you: + +* *The checksum* detects a corrupted or swapped download. It is published + alongside the binary, so on its own it does not prove who built it. +* *The attestation* is signed with a short-lived Sigstore certificate issued to + this repository's `release.yml` running for that tag. It binds the binary's + SHA-256 digest to the repository, commit, workflow and runner. It shows *where + and how* the binary was built. It does not show that the source is free of + bugs, and it does not make the build reproducible. + +== Toolchain and pinning + +=== Minimum Idris 2 version + +*Idris 2 0.8.0* (stated in `README.adoc` and `manifest.json`). +`.github/scripts/check-idris-version.sh` enforces it in the release workflow. +`ci.yml` does not run that check yet; adding it is a one-line step. + +=== How the toolchain is pinned + +[cols="1,2,2",options="header"] +|=== +| Component | Pin | Where +| Container image | `ghcr.io/stefan-hoeck/idris2-pack@sha256:f0758996a931…7e1ff` | `ci.yml` and `release.yml` (must match) +| Idris 2 | 0.8.0 at `idris-lang/Idris2@6ca00e72e32849cb35e5c262904e9c3cffc1a9a4` (2026-06-12), installed by pack inside the image | Implied by the image digest. Also `IDRIS2_COMMIT` in `build-static.sh`. +| Base OS | Ubuntu 24.04 with `gcc`, `make`, `chezscheme`, `libgmp3-dev` and `git` from apt | Implied by the image digest +| Zig | 0.16.0, tarball SHA-256 `70e49664…3d00` | `ZIG_VERSION`, `ZIG_SHA256` in `build-static.sh` +| GMP | 6.3.0, tarball SHA-256 `a3c2b802…8898` | `GMP_VERSION`, `GMP_SHA256` in `build-static.sh` +| Actions | Full commit SHAs, with the tag in a comment | `release.yml` +| Extra apt tools | `curl`, `xz-utils`, `m4`: *not pinned* | `release.yml`. Build-time only; none is linked into the binary. +|=== + +How the image's contents were found (2026-10-13): the image config lists the +apt packages above and the Ubuntu 24.04 label, and the pack state layer contains +`install/6ca00e72…/idris2/idris2-0.8.0/`. The image was not run. + +`build-static.sh` checks that the image's Idris 2 is the pinned commit, and stops +the build if any check fails: + +. `idris2 --version` contains the first nine characters of `IDRIS2_COMMIT`; +. pack has exactly one `idris2-` directory under + `$PACK_STATE_DIR/install//idris2`; +. every RefC and C-support header installed there is identical to the same file + in the source at that commit. + +This was exercised against fake layouts on Windows, not against the real image. +If pack's build of Idris 2 does not put the commit in the version string, check +1 fails closed, and the dry-run will show it. + +*To change the toolchain:* update the image digest in `ci.yml` and `release.yml` +together, set `IDRIS2_COMMIT` to the commit the new image contains, run CI, then +repeat the <>. + +== How the static binary is built + +=== Decision: RefC backend + `zig cc` + musl + +Idris 2's default Chez backend produces a launcher script plus a compiled Chez +program that needs the Chez Scheme runtime at run time, so it cannot be a single +static binary. The release therefore uses Idris 2's *RefC* backend, which emits +C, and compiles that C with *`zig cc`* for `x86_64-linux-musl` with `-static`. + +Why this route: + +* RefC output is ordinary C, so static linking is ordinary static linking. +* `zig cc` brings musl, a linker and cross-compilation in one pinned, + checksummed download. No second C toolchain or distribution image is needed. +* It matches the estate's preference for Zig for native code, and later Zig FFI + libraries (image decoding for milestone 2, file watching for milestone 7) can + be linked into the same static binary. + +Alternatives considered: + +* *Chez Scheme with embedded boot files.* Keeps the tested backend, but needs + Chez itself rebuilt for musl and a custom C entry point. Much more build + machinery. +* *Alpine and `musl-gcc`.* Needs Idris 2 rebuilt on Alpine, from apk packages + that are harder to pin. + +=== Steps (`.github/scripts/build-static.sh`) + +. Download Zig and check its SHA-256. +. Fetch the Idris 2 source at `IDRIS2_COMMIT` (the git commit is the content + pin) and check that it matches the compiler in the image. +. Download GMP, check its SHA-256, build it static for musl with + `--disable-assembly`, and run its own test suite (`make check`). RefC links + GMP for `Integer`. +. Rebuild the Idris 2 C runtimes (`support/c` and `support/refc`) for musl with + the same sources and `-O2` as their Makefiles. The copies in the image are + built for glibc. +. Run `idris2 --cg refc Ddraig.idr -o ddraig-refc`. RefC also links a glibc + binary with the image's gcc; only its `build/exec/ddraig-refc.c` is used. +. Compile and link that C file with `zig cc -target x86_64-linux-musl -static -s`. +. Fail unless `readelf` shows no program interpreter and no needed libraries. + +=== Checks before anything is published (`.github/scripts/test-static.sh`) + +CI tests the *Chez* build, but the release ships the *RefC* build, so the release +job compares the two directly: + +. Every CI test subcommand, plus `--version` and `validate-state`, must succeed + and print *byte-identical* output from both binaries. +. The `examples/` site, and a fixture site the script writes (adding a nested + page, a draft and more Markdown features), must each build to byte-identical + output trees. +. A large page (5,000 short blocks by default) must build without crashing and + match. +. On a tag, `ddraig --version` must equal the tag. + +=== Known risks (not verified) + +. *Backend divergence.* RefC and Chez differ in their runtimes. The comparison + checks above catch differences on the inputs they cover, not on all inputs. +. *Stack depth.* Chez grows its stack on demand; a C program has a fixed stack + (usually 8 MiB on Linux). The Markdown parser recurses once per block and once + per inline element, so a large enough page could crash the static binary while + the Chez build succeeds. The large-page check guards a realistic size. If it + fails, the fix is either to make those loops tail-recursive, or to run the + program on a thread with a large stack through a small Zig entry shim. +. *The C runtimes under musl.* Building `support/c` and `support/refc` with + `zig cc` for musl has not been tried. A compile error there stops the build + rather than producing a bad binary. One specific risk: `support/c/getline.c` + defines `getdelim` and `getline` unconditionally, as musl's libc also does. + Static archive resolution probably avoids a clash, but a duplicate-symbol link + error is possible. +. *Zig download source.* `build-static.sh` downloads Zig from `ziglang.org`. The + Zig project may prefer automated downloads to come from its community + mirrors; this was not checked. The SHA-256 pin makes a mirror equally safe. +. *Midnight.* The attestation file records the build day. If the two builds in + `test-static.sh` straddle 00:00 UTC, the comparison fails; re-run. +. *Version string.* The tag check assumes `ddraig --version` prints exactly + `ddraig X.Y.Z`, which is what `Ddraig.idr` does today. +. *Estate policy.* This repository rejects, at job set-up, any action not + pinned to a full commit SHA, including the inner actions of a composite action + (see the note on `upload-pages-artifact` in `ci.yml`). All four actions in + `release.yml` are JavaScript actions with no inner actions, so they comply. + This is also why the workflow calls `actions/attest` directly: + `actions/attest-build-provenance` v4 is a composite wrapper around it. + Separately, workflow comments mention an estate-wide + `.github/workflows/actions.lock` keyed by workflow path. This repo has no such + file and `ci.yml` uses actions, so it does not appear to be enforced here. If + it is, `release.yml` needs an entry before it can start. A fork cannot reveal + either policy, because organisation and repository settings do not carry over + to it. + +== Cutting a release + +. Set the version in `Ddraig.idr` (`ddraigVersion`), `manifest.json` and the + STATE descriptile. Merge through the normal PR process. +. Tag the merge commit `vX.Y.Z` and push the tag. +. The workflow builds, tests, attests and verifies. Only then does it publish. +. Check that the release page has all three files, then run the <> + commands once yourself. + +[#dry-run] +== Fork dry-run + +The brief requires the workflow to be tested on a tag in a fork before anyone +relies on it. + +. Fork `hyperpolymath/ddraig-ssg`. In the fork, enable Actions (forks start with + workflows disabled). +. Push this branch to the fork and merge it into the fork's `main`. +. Optional first pass: run *Release* by hand (`workflow_dispatch`). This builds + and tests only. +. Push the tag `v0.2.0` to the fork. It must match `ddraigVersion`. +. When the run finishes, run the <> commands with + `--repo /ddraig-ssg` and the fork's workflow path. +. Run *Release* by hand again with `big_blocks` set to 20000, then 50000, then + 100000. Record the size at which the static binary first fails. (The + reference build runs only after the static one succeeds, so its limit is not + measured.) +. Fill in the record below. Then delete the fork's dry-run release and tag: + they are public. The dry-run's attestation stays in Sigstore's public + transparency log, which cannot be deleted. It names the fork, the workflow + and the binary's digest, and nothing secret. + +=== Dry-run record + +*Not yet run.* + +[cols="1,3",options="header"] +|=== +| Item | Result +| Date | +| Fork and run URL | +| `workflow_dispatch` build and test | +| Tag run: build job | +| Tag run: release job | +| Static checks (`readelf`) | +| `test-static.sh` results | +| Large-page limit (static binary) | +| User verification (steps 1, 2, 2b) | +| Binary size and SHA-256 | +| Problems found and fixes | +|=== diff --git a/SEARCH-INDEX.adoc b/SEARCH-INDEX.adoc new file mode 100644 index 0000000..1b96f48 --- /dev/null +++ b/SEARCH-INDEX.adoc @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell += Ddraig search index format (version 1) +:toc: macro + +[WARNING] +==== +*Status: draft, not implemented.* This is the specification for work package 2, +milestone 4. No code emits this file yet and nothing here is tested. Config key +names, page fields and per-language paths are *provisional* until work package 1 +(site config, content model, i18n) is merged; they are marked _(WP1)_ below. +==== + +Ddraig can write a static JSON search index that a page can load and query +without a JavaScript search framework. This document fixes the file's format so +that clients written against it keep working across releases. + +toc::[] + +== Goals and non-goals + +* *Static.* One JSON file per language, generated at build time. No server. +* *No framework.* The index is shaped so that a short hand-written script can + answer queries: terms are pre-tokenised, pre-weighted and sorted. +* *Deterministic.* The same sources and config produce a byte-identical file. + There is no timestamp in it. Milestone 6 (incremental builds) relies on this. +* *Bounded.* A configurable size limit is enforced by a documented, deterministic + truncation rule. Pages are never dropped silently. +* *Not a goal:* stemming, fuzzy matching, phrase search or ranking beyond summed + field weights. Clients may add these on top. + +== Enabling it + +The index is *off by default*, so existing sites gain no new file unless they ask +for one. + +[source,toml] +---- +[search] # (WP1) final config syntax follows the WP1 config module +enabled = true # default false +max_bytes = 1048576 # default 1 MiB; hard limit on the serialised file +stop_words = "builtin" # "builtin" (English), "none", or a list of words +---- + +A page opts out with front matter `search: false` _(WP1: new key)_. + +== Location + +* Default language: `/search-index.json` at the site root. +* Other languages: `//search-index.json` _(WP1: follows the i18n URL + scheme)_. Each language has its own stop-word list and its own index. + +Served as `application/json`. Encoded as UTF-8 with no byte-order mark. + +== What is indexed + +A *searchable page* is any built content page except: + +* drafts (they are never built); +* pages with `search: false`; +* generated pages that are not content: alias redirects, the 404 page, listing + and taxonomy pages _(WP1)_, machine views (milestone 9). + +From each searchable page the indexer takes four *fields*: + +[cols="1,3",options="header"] +|=== +| Field | Source +| `title` | Front matter `title` (falls back to the source path, as the build does today). +| `description` | Front matter `description`. +| `headings` | The text of every heading, `h1` to `h6`, in document order. +| `body` | The text of every other block, taken from the typed `Html` tree (not + by stripping tags from rendered HTML). Image alt text is included. + Fenced code blocks and raw-HTML passthrough blocks are excluded. +|=== + +== Tokenisation (normative) + +Clients must tokenise queries with exactly these rules, or exact-term lookups +will miss. + +. *Case.* ASCII `A`–`Z` are lowered to `a`–`z`. Nothing else is changed. No + Unicode case folding or normalisation is applied, because the result must not + depend on the Idris 2 backend's character tables. +. *Separators.* A token is a maximal run of characters that are not separators. + Separators are: + ** every code point below U+0080 except `0`–`9`, `A`–`Z` and `a`–`z`; + ** U+0080–U+00BF, U+00D7 and U+00F7 (Latin-1 controls, punctuation and signs); + ** U+2000–U+206F (General Punctuation, including curly quotes and dashes); + ** U+2E00–U+2E7F (Supplemental Punctuation); + ** U+3000–U+303F (CJK Symbols and Punctuation). +. *Length.* Tokens of a single ASCII character are dropped. Tokens longer than + 64 code points are dropped. +. *Stop words.* Tokens in the stop-word list are dropped, after step 1. + +Consequences worth knowing: + +* `don't` yields `don`; the lone `t` is dropped. +* Digits are kept, so `2024` is searchable. `0.8.0` is not: it splits into + single digits, which are dropped. +* Scripts written without spaces (Chinese, Japanese) become one long token per + run, which makes them effectively unsearchable in version 1. + +=== Built-in English stop words + +The built-in list is the classic 33-word English set used by Lucene's +`StopAnalyzer`, chosen because it is small, well known and removes no +content words: + +`a an and are as at be but by for if in into is it no not of on or such that the +their then there these they this to was will with` + +There is no built-in list for other languages; set `stop_words` per language +_(WP1)_. The list in force is written into the index so clients can apply it to +queries. + +== Weights + +Each (term, page) pair gets one integer weight: + +---- +weight = 8 × (occurrences in title) + + 4 × (occurrences in headings) + + 2 × (occurrences in description) + + 1 × (occurrences in body) +---- + +The field weights are fixed for format version 1 and recorded in the file. + +== File structure + +The top-level value is an object with these keys, *in this order*: + +[cols="1,1,3",options="header"] +|=== +| Key | Type | Meaning +| `format` | string | Always `"ddraig-search"`. +| `version` | integer | Format version. This document describes `1`. +| `generator` | string | `"Ddraig SSG "`. +| `lang` | string | Language of this index, e.g. `"en"` _(WP1)_. +| `weights` | object | `{"title":8,"headings":4,"description":2,"body":1}`. +| `stop_words` | array of string | The stop-word list applied, sorted. +| `limits` | object | See <>. +| `docs` | array of object | One entry per searchable page, sorted by `url` + (code-point order). A page's *doc id* is its index in this array. +| `terms` | array of `[string, postings]` | Every indexed term with its + postings, sorted by term (code-point order). Sorted pairs rather than an object + so that a client can binary-search for a prefix. +|=== + +Each `docs` entry is an object with these keys, in this order: + +[cols="1,1,3",options="header"] +|=== +| Key | Type | Meaning +| `url` | string | The page's site-relative URL, the build's `Page.url` (e.g. + `/blog/post.html`). Never absolute, so the index works whatever the base URL. +| `title` | string | Page title. +| `description` | string | Omitted when empty. +| `date` | string | Front matter date as normalised by the build. Omitted when absent. +| `headings` | array | `{"level":n,"text":s,"anchor":a}` per heading, in document + order. `url + "#" + anchor` deep-links to the heading. Always present, may be empty. +| `excerpt` | string | A short plain-text opening of the body; see <>. + Omitted when empty. +|=== + +[#excerpt] +=== Excerpt (normative) + +Start from the page's body text: the text of each body block, in document +order, joined with one U+0020 between blocks. Then, in this order: + +. Replace every run of *whitespace* with one U+0020. Whitespace means exactly + U+0009, U+000A, U+000C, U+000D, U+0020 and U+00A0; no other code point counts. +. Remove a leading and a trailing U+0020, if present. +. If the result is longer than 160 code points, keep the first 160, then remove + a trailing U+0020 if one is left, then append U+2026 (`…`). Otherwise keep the + result unchanged, with no ellipsis. + +An empty result means the key is omitted. + +`postings` is an array of `[docId, weight]` pairs sorted by `docId`. + +=== JSON encoding + +* No insignificant whitespace. +* Strings escape `"` as `\"`, `\` as `\\`, newline as `\n`, carriage return as + `\r`, tab as `\t`, and every other code point below U+0020 as `\u00XX` with + lower-case hex. Nothing else is escaped: non-ASCII is written as UTF-8 and `/` + is not escaped. + +[NOTE] +==== +The existing `jsonEsc` in `Ddraig.idr` escapes only `"`, `\` and newline, so a +tab or carriage return in a title would produce invalid JSON. The search index +needs a complete escaper. If the attestation file is switched to it too, its +output changes for inputs containing those characters, and that change needs a +golden test. +==== + +[#limits] +== Size limit (normative) + +The `limits` object is `{"max_bytes":n,"body_tokens":k,"excerpts":b}`. + +. Build the full index. If its serialised size is at most `max_bytes`, write it + with `body_tokens` = `null` and `excerpts` = `true`. +. Otherwise index only the first `k` body tokens of each page (in document order, + counted after stop-word removal), choosing the largest `k` for which the file + fits. `k` is found by binary search, so the result is deterministic. Write + `body_tokens` = `k`. +. If the file does not fit even with `k` = 0, drop every `excerpt`, set + `excerpts` = `false`, and repeat the search for `k` from scratch. (Dropping + the excerpts frees space, so some body tokens may fit again.) +. If it still does not fit, the build fails. The error states the size of each + field across all pages and suggests raising `max_bytes` or excluding pages with + `search: false`. + +Titles, descriptions and headings are never truncated. + +== Example + +A two-page site. Real output has no whitespace; it is shown indented here. + +[source,json] +---- +{ + "format": "ddraig-search", + "version": 1, + "generator": "Ddraig SSG 0.3.0", + "lang": "en", + "weights": {"title": 8, "headings": 4, "description": 2, "body": 1}, + "stop_words": ["a", "an", "and", "are", "as", "at", "be", "but", "by", "for", + "if", "in", "into", "is", "it", "no", "not", "of", "on", "or", "such", + "that", "the", "their", "then", "there", "these", "they", "this", "to", + "was", "will", "with"], + "limits": {"max_bytes": 1048576, "body_tokens": null, "excerpts": true}, + "docs": [ + {"url": "/blog/feeds.html", "title": "Feeds", "date": "2026-01-15", + "headings": [{"level": 1, "text": "Feeds", "anchor": "feeds"}, + {"level": 2, "text": "Atom and RSS", "anchor": "atom-and-rss"}], + "excerpt": "Ddraig writes Atom."}, + {"url": "/index.html", "title": "Ddraig", + "description": "A dependently typed generator", + "headings": [{"level": 1, "text": "Ddraig", "anchor": "ddraig"}], + "excerpt": "Types that breathe fire."} + ], + "terms": [ + ["atom", [[0, 5]]], + ["breathe", [[1, 1]]], + ["ddraig", [[0, 1], [1, 12]]], + ["dependently", [[1, 2]]], + ["feeds", [[0, 12]]], + ["fire", [[1, 1]]], + ["generator", [[1, 2]]], + ["rss", [[0, 4]]], + ["typed", [[1, 2]]], + ["types", [[1, 1]]], + ["writes", [[0, 1]]] + ] +} +---- + +`atom` scores 5 on the feeds page: 4 for the heading "Atom and RSS" plus 1 for +the body. `ddraig` scores 12 on the home page: 8 for the title plus 4 for its +`h1`. + +== Querying (non-normative) + +A client can answer a query in a few lines: + +. Tokenise the query with the rules above and drop stop words. +. For each query token, find the term exactly, or every term with that prefix + (binary search on the sorted `terms`). +. Sum weights per doc id. For an AND query, keep only docs that matched every + token. +. Sort by total weight descending, then doc id ascending. Show `title`, `url` + and `excerpt` from `docs`; use `headings` to link straight to a section. + +== Stability policy + +* `version` increases whenever tokenisation, weights, structure, or the meaning + of an existing key changes. +* Within a version, new *optional* keys may be added. Clients must ignore keys + they do not know. +* Ddraig emits only the current version. Each change is listed below. + +== Planned tests (milestone 4) + +* Golden index for a fixture site, compared byte for byte. +* Drafts and `search: false` pages are absent. +* Stop words are absent from `terms`; the list in the file matches config. +* Size limit: a fixture over the limit produces the documented `body_tokens` + value, twice in a row, byte-identically; a fixture that cannot fit fails with + the documented message. +* JSON escaping: a title containing a tab and a quote round-trips through a + strict parser. + +== Open questions + +* _(WP1)_ Exact page fields, the i18n URL scheme and config syntax. +* Whether listing pages should be searchable. Version 1 says no. +* Whether to index fenced code. Version 1 says no; a `search.include_code` + option could be added later without a format change, because it only changes + which text is fed in. + +== Changelog + +* *Version 1* (draft): initial format.