From fba9e65c34710be5844477ca9eb67cc884a6b325 Mon Sep 17 00:00:00 2001 From: Rebecca Hum Date: Thu, 10 Sep 2026 10:21:21 -0600 Subject: [PATCH 1/2] fix: publish validated npm tarballs from fresh staging copies --- .github/workflows/npm-publish.yml | 29 ++++++++++++++++++++--------- docs/NPM-RELEASE-RECOVERY.md | 18 ++++++++++++++++++ 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 14c471a70..f8bd70757 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -33,15 +33,12 @@ jobs: id-token: write pull-requests: write steps: - - uses: Automattic/vip-actions/npm-publish@438cd00aac2463a2dbfb4a747ad7b1ac67a2575a # v0.7.5 - env: - # The action captures `npm version` stdout to build the next-dev branch name. - # Keep lifecycle script banners from polluting that value while still allowing - # the `version` script to sync npm-shrinkwrap.json. - NPM_CONFIG_LOGLEVEL: silent + - uses: Automattic/vip-actions/npm-publish@a21de3c167c01a17bde269771490fda0b8fdbb9c # staged npm artifacts with: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} npm-version: '11' + STAGE_PACKAGE: 'true' + SMOKE_SCRIPT: 'smoke:release' USE_TRUSTED_PUBLISHING: 'true' PROVENANCE: 'true' CONVENTIONAL_COMMITS: 'true' @@ -118,10 +115,22 @@ jobs: npm rebuild npm run prepare --if-present npm test - npm publish --access public --tag latest --dry-run + - name: Stage, pack and validate release + id: package + uses: Automattic/vip-actions/npm-pack-staged@a21de3c167c01a17bde269771490fda0b8fdbb9c # staged npm artifacts + with: + tag: latest + smoke-script: smoke:release + + - name: Verify publishing the staged tarball + env: + PACKAGE_TARBALL: ${{ steps.package.outputs.tarball }} + run: npm publish "$PACKAGE_TARBALL" --access public --tag latest --dry-run --ignore-scripts --loglevel error - name: Publish existing release to npm - run: npm publish --access public --tag latest --provenance --loglevel error + env: + PACKAGE_TARBALL: ${{ steps.package.outputs.tarball }} + run: npm publish "$PACKAGE_TARBALL" --access public --tag latest --provenance --ignore-scripts --loglevel error changelog: name: Publish docs changelog @@ -154,10 +163,12 @@ jobs: id-token: write pull-requests: write steps: - - uses: Automattic/vip-actions/npm-publish-prerelease@438cd00aac2463a2dbfb4a747ad7b1ac67a2575a # v0.7.5 + - uses: Automattic/vip-actions/npm-publish-prerelease@a21de3c167c01a17bde269771490fda0b8fdbb9c # staged npm artifacts with: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} npm-version: '11' + STAGE_PACKAGE: 'true' + SMOKE_SCRIPT: 'smoke:release' USE_TRUSTED_PUBLISHING: 'true' PROVENANCE: 'true' NPM_TAG: ${{ inputs.npm_tag }} diff --git a/docs/NPM-RELEASE-RECOVERY.md b/docs/NPM-RELEASE-RECOVERY.md index 18179b15e..5f98ba451 100644 --- a/docs/NPM-RELEASE-RECOVERY.md +++ b/docs/NPM-RELEASE-RECOVERY.md @@ -24,3 +24,21 @@ Recovery does not create the next development-version PR that the regular publishing action normally opens after publication. Handle that version bump separately once recovery succeeds. If npm publication succeeds but a downstream documentation job fails, rerun only the failed jobs. + +## How the release artifact is prepared + +Stable releases, prereleases and recovery use the shared `npm-pack-staged` +helper in `Automattic/vip-actions`. After building and testing, it runs the +prepublish/pack preparation hooks in the source checkout and uses `rsync -a` +without `-H` to copy into a fresh staging directory. This turns native dependency +hard links into independent files without changing the source checkout's links. + +The helper packs with lifecycle scripts disabled, validates the archive and runs +`smoke:release` inside an extraction of that exact tarball. Validation rejects +links and unsafe archive entries and verifies the package name/version. Both +the dry run and real publication use the same validated `.tgz` with +`--ignore-scripts`, so another build cannot recreate hard links after validation. + +For `4.1.2`, the Linux builds of bundled `cpu-features` and `ssh2` created three +hard links that npm rejected with `E415: Hard link is not allowed`. Staging fixes +the archive while preserving dependency bundling and the existing release tag. From e2c06ce5c00e52d1ccc36292bae9b39318705b81 Mon Sep 17 00:00:00 2001 From: Rebecca Hum Date: Thu, 10 Sep 2026 10:31:18 -0600 Subject: [PATCH 2/2] fix: keep staged npm release tooling in vip-cli --- .github/actions/npm-release/action.yml | 45 +++ .github/scripts/npm-release/README.md | 53 ++++ .github/scripts/npm-release/pack.sh | 55 ++++ .../scripts/npm-release/publish-prerelease.sh | 90 ++++++ .github/scripts/npm-release/publish.sh | 161 +++++++++++ .../scripts/npm-release/tests/test_pack.py | 258 ++++++++++++++++++ .github/scripts/npm-release/validate.py | 33 +++ .github/workflows/npm-publish.yml | 43 +-- .github/workflows/npm-release-tests.yml | 28 ++ docs/NPM-RELEASE-RECOVERY.md | 9 +- 10 files changed, 752 insertions(+), 23 deletions(-) create mode 100644 .github/actions/npm-release/action.yml create mode 100644 .github/scripts/npm-release/README.md create mode 100755 .github/scripts/npm-release/pack.sh create mode 100755 .github/scripts/npm-release/publish-prerelease.sh create mode 100755 .github/scripts/npm-release/publish.sh create mode 100644 .github/scripts/npm-release/tests/test_pack.py create mode 100644 .github/scripts/npm-release/validate.py create mode 100644 .github/workflows/npm-release-tests.yml diff --git a/.github/actions/npm-release/action.yml b/.github/actions/npm-release/action.yml new file mode 100644 index 000000000..6da1ad6ab --- /dev/null +++ b/.github/actions/npm-release/action.yml @@ -0,0 +1,45 @@ +name: Publish VIP CLI npm release +description: Build, test, stage and publish the exact validated package tarball. +inputs: + mode: + description: stable or prerelease + required: true + npm-tag: + description: npm tag for prereleases + default: next +runs: + using: composite + steps: + - uses: actions/setup-node@v7 + with: + node-version: 'lts/*' + registry-url: https://registry.npmjs.org/ + - name: Prepare release tools + shell: bash + run: | + npm install --global npm@11 + git config --global user.name 'WordPress VIP Bot' + git config --global user.email '<22917138+wpcomvip-bot@users.noreply.github.com>' + # Keep tools from this workflow revision when the stable script changes branches. + tools_dir=$(mktemp -d "$RUNNER_TEMP/vip-release-tools.XXXXXXXX") + cp -R "$GITHUB_WORKSPACE/.github/scripts/npm-release/." "$tools_dir/" + printf 'VIP_RELEASE_TOOLS=%s\n' "$tools_dir" >> "$GITHUB_ENV" + - name: Validate and publish release + shell: bash + env: + RELEASE_MODE: ${{ inputs.mode }} + NPM_TAG: ${{ inputs.npm-tag }} + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_HEAD_REF: ${{ github.head_ref }} + PR_ASSIGNEE: ${{ github.actor }} + USE_TRUSTED_PUBLISHING: 'true' + PROVENANCE: 'true' + CONVENTIONAL_COMMITS: 'true' + SMOKE_SCRIPT: smoke:release + run: | + case "$RELEASE_MODE" in + stable) bash "$VIP_RELEASE_TOOLS/publish.sh" ;; + prerelease) bash "$VIP_RELEASE_TOOLS/publish-prerelease.sh" ;; + *) echo 'Unsupported release mode' >&2; exit 1 ;; + esac diff --git a/.github/scripts/npm-release/README.md b/.github/scripts/npm-release/README.md new file mode 100644 index 000000000..a838244fa --- /dev/null +++ b/.github/scripts/npm-release/README.md @@ -0,0 +1,53 @@ +# VIP CLI npm release tooling + +The local `.github/actions/npm-release` composite action runs `publish.sh` for +stable releases and `publish-prerelease.sh` for prereleases. Recovery calls +`pack.sh` directly. These scripts replace the npm publishing actions previously +used from `Automattic/vip-actions`; no change to that repository is required. + +Stable publishing retains release-PR file/type validation, release branch and +clean-checkout checks, build/tests, GitHub release creation, npm publication and +the next development-version PR. Prereleases retain their selected npm tag, +clean-checkout check, build/tests and prerelease creation. The workflow supplies +npm trusted publishing credentials and keeps the existing permissions and +`npm-publish` environment. GitHub errors during release-PR inspection stop the +release. Only `npm version` uses silent logging to keep its captured version +string clean. + +## Artifact preparation + +`pack.sh SOURCE OUTPUT TAG [SMOKE_SCRIPT]` expects a built and tested checkout, +plus an existing output directory outside that checkout. Requires Node/npm, +`rsync`, Python 3 and `tar`, available on GitHub-hosted Ubuntu runners. It never +publishes. + +The helper runs `prepublishOnly`, `prepack` and `prepare` in the source checkout +with the npm token cleared. It copies to a fresh temporary directory using +`rsync -a` **without `-H`**, so native build hard links become independent files. +It then packs with lifecycle scripts disabled, validates package identity and +rejects links, duplicate paths and unsafe entries. `postpack` runs in the source +checkout after packing. `smoke:release` runs inside an extraction of that exact +archive using bundled dependencies. + +The only stdout is the validated tarball's absolute path. Build output goes to +stderr. The staging directory is always cleaned; the caller owns output cleanup. +Dry-run and publication use that exact tarball with `--ignore-scripts`. Neither +rebuild nor repack afterward. Directory `publish`/`postpublish` hooks are not run; +VIP CLI does not define them. + +The workflow saves these tools under `RUNNER_TEMP` before changing the source +checkout. This lets recovery build an old release tag, such as `4.1.2`, using +tools from the workflow revision without adding or changing files in that tag. + +## Tests + +```sh +python3 -m unittest discover -s .github/scripts/npm-release/tests -v +shellcheck .github/scripts/npm-release/*.sh +actionlint .github/workflows/npm-publish.yml .github/workflows/npm-release-tests.yml +``` + +Tests build real fixture archives without registry access. Publisher integration +tests mock GitHub, git and npm publication, so they cannot publish or create +remote releases. The Ubuntu packaging workflow runs them separately from CLI +runtime tests. diff --git a/.github/scripts/npm-release/pack.sh b/.github/scripts/npm-release/pack.sh new file mode 100755 index 000000000..a2266f843 --- /dev/null +++ b/.github/scripts/npm-release/pack.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# stdout is exclusively the validated tarball path. All build output goes to stderr. +set -euo pipefail + +source_dir=$(cd "${1:-.}" && pwd -P) +output_dir=$(cd "${2:?Provide an existing output directory outside the source tree}" && pwd -P) +tag=${3:-latest} +smoke_script=${4:-} +case "$output_dir/" in + "$source_dir/"*) echo 'Output directory must be outside the source tree.' >&2; exit 1 ;; +esac +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) +work_dir=$(mktemp -d "${TMPDIR:-/tmp}/npm-pack-staged.XXXXXXXX") +trap 'rm -rf "$work_dir"' EXIT +case "$work_dir/" in + "$source_dir/"*) echo 'TMPDIR must be outside the source tree.' >&2; exit 1 ;; +esac + +# Preserve validation/pack hooks, but run all artifact-producing hooks before copying. +# Never expose the registry token to lifecycle scripts or smoke tests. +export NODE_AUTH_TOKEN='' +export NPM_CONFIG_LOGLEVEL=error +export npm_config_tag="$tag" +cd "$source_dir" +npm run prepublishOnly --if-present >&2 +npm run prepack --if-present >&2 +npm run prepare --if-present >&2 + +mkdir "$work_dir/package" +# -a intentionally omits -H: every hard-linked name becomes an independent file. +# A fresh destination is essential; updating an old staging tree is not sufficient. +rsync -a --exclude='/.git' --exclude='/.npmrc' "$source_dir/" "$work_dir/package/" +( + cd "$work_dir/package" + npm pack --ignore-scripts --json --pack-destination "$work_dir" > "$work_dir/pack.json" +) +filename=$(node -e 'const p = require(process.argv[1]); if (p.length !== 1 || require("node:path").basename(p[0].filename) !== p[0].filename) process.exit(1); process.stdout.write(p[0].filename);' "$work_dir/pack.json") +archive="$work_dir/$filename" +python3 "$script_dir/validate.py" "$archive" "$source_dir/package.json" >&2 +npm run postpack --if-present >&2 + +if [[ -n "$smoke_script" ]]; then + mkdir "$work_dir/smoke" + tar -xzf "$archive" -C "$work_dir/smoke" + # Run against the extracted artifact, not the original checkout or staging tree. + (cd "$work_dir/smoke/package" && npm run "$smoke_script") >&2 +fi + +# Only expose a completed, validated artifact. The caller owns output_dir cleanup. +if [[ -e "$output_dir/$filename" ]]; then + echo "Refusing to overwrite an existing artifact: $output_dir/$filename" >&2 + exit 1 +fi +cp "$archive" "$output_dir/$filename" +printf '%s\n' "$output_dir/$filename" diff --git a/.github/scripts/npm-release/publish-prerelease.sh b/.github/scripts/npm-release/publish-prerelease.sh new file mode 100755 index 000000000..af1bdd5c5 --- /dev/null +++ b/.github/scripts/npm-release/publish-prerelease.sh @@ -0,0 +1,90 @@ +#!/bin/sh + +set -eu + +# Set inputs +: "${NPM_TAG:=next}" +: "${PROVENANCE:=false}" + +echo_title() { + echo "" + echo "== $1 ==" +} + +# Fetch some basic package information +echo_title "Fetching local package info" +LOCAL_NAME=$(node -p "require('./package.json').name") +LOCAL_VERSION=$(node -p "require('./package.json').version") +LOCAL_BRANCH=$(git branch --show-current) +echo "✅ Found ${LOCAL_NAME} ${LOCAL_VERSION} on branch ${LOCAL_BRANCH}" + +# If not using Trusted Publishing, validate npm is logged in and ready +if [ "${USE_TRUSTED_PUBLISHING:-}" != "true" ]; then + echo_title "Checking npm auth" + if ! NPM_USER=$(npm whoami); then + echo "❌ npm cli is not authenticated. Please make sure you're logged in or NPM_TOKEN is set." + exit 202 + fi + echo "✅ Logged in as ${NPM_USER} and ready to publish" +fi + +# Validate no uncommitted changes. +# Shouldn't happen in CI but protects against local runs. +echo_title "Checking for local changes" +if ! git diff-index --quiet HEAD --; then + echo "❌ Working directory has uncommitted changes; please clean up before proceeding." + exit 204 +fi +echo "✅ No local changes found" + +# Install +echo_title "npm ci + test" + +# Install dependencies but skip pre/post scripts since our auth token is in place +npm ci --ignore-scripts + +# Run scripts + tests without auth token to prevent malicious access +NODE_AUTH_TOKEN='' npm rebuild +NODE_AUTH_TOKEN='' npm run prepare --if-present +NODE_AUTH_TOKEN='' npm test +echo "✅ npm install + npm test look good" + +# Confirm y/n (if running locally) +if [ -t 0 ]; then + echo_title "Confirm release" + printf "Are you sure you want to publish a new release? (y/n)" + read -r yn + case $yn in + [Yy]*) + ;; + + *) + echo "❌ Aborting release" + exit 205 + ;; + esac +fi + +# Pack once from a fresh copy; never rebuild or repack after validation. +echo_title "Pack and validate staged npm artifact" +artifact_dir=$(mktemp -d "${TMPDIR:-/tmp}/npm-publish-artifact.XXXXXXXX") +trap 'rm -rf "$artifact_dir"' EXIT +script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +PACKAGE_TARBALL=$(bash "$script_dir/pack.sh" "$PWD" "$artifact_dir" "${NPM_TAG}" "${SMOKE_SCRIPT:-smoke:release}") +npm publish "$PACKAGE_TARBALL" --access public --tag "${NPM_TAG}" --dry-run --ignore-scripts --loglevel error + +# Publish on GitHub and tag +echo_title "Publishing a new release on GitHub and tagging" +gh release create "${LOCAL_VERSION}" --generate-notes --prerelease --target "${LOCAL_BRANCH}" +echo "✅ Released version ${LOCAL_VERSION} on GitHub and tagged" + +# Publish to NPM +echo_title "npm publish" +OPTIONS="--access public" +if [ "${PROVENANCE}" = "true" ] && [ "${CI:-}" = "true" ] && [ "${GITHUB_ACTIONS:-}" = "true" ]; then + OPTIONS="${OPTIONS} --provenance" +fi + +# shellcheck disable=SC2086 # OPTIONS contains the existing publish flags. +npm publish "$PACKAGE_TARBALL" ${OPTIONS} --tag "${NPM_TAG}" --ignore-scripts --loglevel error +echo "✅ Successfully published new release for ${LOCAL_NAME} as ${LOCAL_VERSION}" diff --git a/.github/scripts/npm-release/publish.sh b/.github/scripts/npm-release/publish.sh new file mode 100755 index 000000000..08d786d81 --- /dev/null +++ b/.github/scripts/npm-release/publish.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash + +set -o errexit # exit on error +set -o errtrace # exit on error within function/sub-shell +set -o nounset # error on undefined vars +set -o pipefail # error if piped command fails + +# Default variables +RELEASE_BRANCH=$(echo "$PR_HEAD_REF" | awk -F '--' '{print $2}' | awk -F '--' '{print $1}') + +echo_title() { + echo "" + echo "== $1 ==" +} + +# Determine which files were changed in PR +echo_title "Determining which files were changed in PR #$PR_NUMBER" +# Fail closed if GitHub cannot provide the changed files. +PR_FILES_CHANGED=$(gh pr diff "$PR_NUMBER" --name-only) +while IFS= read -r changed_file; do + if [[ -n "$changed_file" && "$changed_file" != *.json ]]; then + echo "❌ Unexpected file changed in release PR: $changed_file" + exit 200 + fi +done <<< "$PR_FILES_CHANGED" +echo "✅ Determined only .json files are changed in PR" + +# Determine and validate release type +echo_title "Determining and validating NPM release type" +NPM_VERSION_TYPE=$(echo "$PR_HEAD_REF" | awk -F '/' '{print $2}' | awk -F '-' '{print $1}') + +# Validate release type value +if [ "$NPM_VERSION_TYPE" != "major" ] && [ "$NPM_VERSION_TYPE" != "minor" ] && [ "$NPM_VERSION_TYPE" != "patch" ]; then + echo "❌ Invalid release type found." + exit 201 +else + echo "✅ NPM release type: $NPM_VERSION_TYPE" +fi + +git fetch origin "$RELEASE_BRANCH" +echo "✅ Fetched $RELEASE_BRANCH from GitHub" + +git checkout "$RELEASE_BRANCH" +echo "✅ Checked out branch $RELEASE_BRANCH" + +# Fetch some basic package information +echo_title "Fetching local package info" +LOCAL_NAME=$(node -p "require('./package.json').name") +LOCAL_VERSION=$(node -p "require('./package.json').version") +LOCAL_BRANCH=$(git branch --show-current) +REMOTE_VERSION=$(npm view "$LOCAL_NAME" version) +echo "✅ Found $LOCAL_NAME $LOCAL_VERSION on branch $LOCAL_BRANCH" +echo "✅ Published version is $REMOTE_VERSION" + +# If not using Trusted Publishing, validate npm is logged in and ready +if [ "${USE_TRUSTED_PUBLISHING:-}" != "true" ]; then + echo_title "Checking npm auth" + if ! NPM_USER=$( npm whoami ); then + echo "❌ npm cli is not authenticated. Please make sure you're logged in or NPM_TOKEN is set." + exit 202 + fi + echo "✅ Logged in as $NPM_USER and ready to publish" +fi + +# Validate current branch +echo_title "Checking branch" +if [ "$LOCAL_BRANCH" != "$RELEASE_BRANCH" ]; then + echo "❌ You can only publish from the '$RELEASE_BRANCH' branch. Please switch branches and try again." + exit 203 +fi +echo "✅ On a valid release branch ($LOCAL_BRANCH)" + +# Validate no uncommitted changes. +# Shouldn't happen in CI but protects against local runs. +echo_title "Checking for local changes" +if ! git diff-index --quiet HEAD --; then + echo "❌ Working directory has uncommitted changes; please clean up before proceeding." + exit 204 +fi +echo "✅ No local changes found" + +# Install +echo_title "npm ci + test" + +# Install dependencies but skip pre/post scripts since auth token may be in place +npm ci --ignore-scripts + +# Run scripts + tests without auth token to prevent malicious access +NODE_AUTH_TOKEN='' npm rebuild +NODE_AUTH_TOKEN='' npm run prepare --if-present +NODE_AUTH_TOKEN='' npm test +echo "✅ npm install + npm test look good" + +# Pack once from a fresh copy; never rebuild or repack after validation. +echo_title "Pack and validate staged npm artifact" +artifact_dir=$(mktemp -d "${TMPDIR:-/tmp}/npm-publish-artifact.XXXXXXXX") +trap 'rm -rf "$artifact_dir"' EXIT +script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +PACKAGE_TARBALL=$(bash "$script_dir/pack.sh" "$PWD" "$artifact_dir" "latest" "${SMOKE_SCRIPT:-smoke:release}") +npm publish "$PACKAGE_TARBALL" --access public --tag "latest" --dry-run --ignore-scripts --loglevel error + +# Publish on GitHub and tag +echo_title "Publishing a new release on GitHub and tagging" +gh release create "${TAG_PREFIX:-}${LOCAL_VERSION}" --generate-notes --target "${RELEASE_BRANCH}" +echo "✅ Released version $LOCAL_VERSION on GitHub and tagged" + +# Publish to NPM +echo_title "npm publish" +OPTIONS="--access public" +if [ "${PROVENANCE}" = "true" ] && [ "${CI:-}" = "true" ] && [ "${GITHUB_ACTIONS:-}" = "true" ]; then + OPTIONS="${OPTIONS} --provenance" +fi + +# shellcheck disable=SC2086 # OPTIONS contains the existing publish flags. +npm publish "$PACKAGE_TARBALL" ${OPTIONS} --tag "latest" --ignore-scripts --loglevel error +echo "✅ Successfully published new '$NPM_VERSION_TYPE' release for $LOCAL_NAME as $LOCAL_VERSION" + +# Version bump to dev - create a branch and a PR, then merge +if [ "$LOCAL_BRANCH" == "$RELEASE_BRANCH" ] && [ "${SKIP_BUMP_TO_DEV:-}" != 'true' ]; then + echo_title "npm version (to next dev)" + + NEXT_LOCAL_DEV_VERSION_TYPE="prepatch" + NEXT_LOCAL_DEV_VERSION=$( npm --silent version --no-git-tag-version --preid "dev" "$NEXT_LOCAL_DEV_VERSION_TYPE" ) + echo "✅ Determined next local dev version: $NEXT_LOCAL_DEV_VERSION" + + # Configure git + echo_title "Configure git" + git config push.autoSetupRemote true + echo "✅ Configured git to auto-setup remote origins" + + # Checkout branch for release + echo_title "Create new git branch, commit to git and create and merge pull request" + NEW_BRANCH="dev-release/$NEXT_LOCAL_DEV_VERSION" + git checkout -b "$NEW_BRANCH" + echo "✅ Check out git branch ($NEW_BRANCH)" + + git add -u + if [ "${CONVENTIONAL_COMMITS:-}" = 'true' ]; then + git commit -m "chore: bump to next $NEXT_LOCAL_DEV_VERSION_TYPE: ($NEXT_LOCAL_DEV_VERSION)" + else + git commit -m "Bump to next $NEXT_LOCAL_DEV_VERSION_TYPE: ($NEXT_LOCAL_DEV_VERSION)" + fi + echo "✅ Commit to GitHub repository ($NEW_BRANCH)" + git push --follow-tags + echo "✅ Pushed commit to GitHub repository" + + NEXT_LOCAL_DEV_VERSION=$(node -p "require('./package.json').version") + echo "✅ Bumped local version to next $NEXT_LOCAL_DEV_VERSION_TYPE: $NEXT_LOCAL_DEV_VERSION" + + # Create pull request in GitHub + echo_title "Create pull request in GitHub" + LABEL='[ Type ] NPM version update' + cat > "$artifact_dir/development-pr.md" <<'BODY' +## Description + +This pull request updates the npm package version to the next development version. +Merge when convenient; this will not trigger publishing to npm. +BODY + PR_URL=$(gh pr create --base "$RELEASE_BRANCH" --head "$NEW_BRANCH" --title "New develop release: $NEXT_LOCAL_DEV_VERSION" --body-file "$artifact_dir/development-pr.md" --label "$LABEL" --assignee "$PR_ASSIGNEE") + echo "✅ Created pull request: $PR_URL" +fi diff --git a/.github/scripts/npm-release/tests/test_pack.py b/.github/scripts/npm-release/tests/test_pack.py new file mode 100644 index 000000000..a30160b2b --- /dev/null +++ b/.github/scripts/npm-release/tests/test_pack.py @@ -0,0 +1,258 @@ +import importlib.util +import io +import json +import os +import shutil +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('validator', ROOT / 'validate.py') +validator = importlib.util.module_from_spec(spec) +spec.loader.exec_module(validator) + + +class StagedPackageTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix='npm staging test ') + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.source = self.root / 'source' + self.output = self.root / 'output' + self.source.mkdir() + self.output.mkdir() + self.manifest = { + 'name': 'staging-fixture', 'version': '1.0.0', + 'dependencies': {'native-fixture': '1.0.0'}, + 'bundleDependencies': True, + 'scripts': { + 'prepublishOnly': 'node -e "if(process.env.npm_config_tag!==\'next\')process.exit(1)"', + 'prepare': 'node prepare.js', + 'postpack': 'node -e "require(\'fs\').writeFileSync(\'built.txt\',\'changed-after-pack\')"', + 'smoke': 'node smoke.js', + 'test': 'node smoke.js', + }, + } + (self.source / 'package.json').write_text(json.dumps(self.manifest)) + dep = self.source / 'node_modules/native-fixture' + dep.mkdir(parents=True) + (dep / 'package.json').write_text(json.dumps({'name': 'native-fixture', 'version': '1.0.0'})) + (self.source / 'prepare.js').write_text(''' +const fs = require('fs'); +const dir = 'node_modules/native-fixture/'; +fs.writeFileSync('built.txt', 'built-before-copy'); +fs.writeFileSync(dir + 'original.node', 'native-bytes'); +fs.chmodSync(dir + 'original.node', 0o755); +if (fs.existsSync(dir + 'copy.node')) fs.unlinkSync(dir + 'copy.node'); +fs.linkSync(dir + 'original.node', dir + 'copy.node'); +''') + (self.source / 'smoke.js').write_text(''' +const fs = require('fs'); +if (fs.readFileSync('built.txt', 'utf8') !== 'built-before-copy') process.exit(1); +if (fs.readFileSync('node_modules/native-fixture/copy.node', 'utf8') !== 'native-bytes') process.exit(1); +if (process.env.NODE_AUTH_TOKEN) process.exit(1); +''') + self.env = {**os.environ, 'HOME': str(self.root), 'NODE_AUTH_TOKEN': 'must-not-reach-hooks', + 'NPM_CONFIG_USERCONFIG': os.devnull, 'NPM_CONFIG_CACHE': str(self.root / 'cache'), + 'NPM_CONFIG_REGISTRY': 'http://127.0.0.1:9', 'TMPDIR': str(self.root)} + + def pack(self, smoke='smoke'): + return subprocess.run(['bash', str(ROOT / 'pack.sh'), str(self.source), str(self.output), + 'next', smoke], env=self.env, text=True, capture_output=True) + + def test_copies_links_and_smokes_exact_archive_without_rebuilding(self): + result = self.pack() + self.assertEqual(result.returncode, 0, result.stderr) + archive = Path(result.stdout.strip()) + self.assertEqual(archive.parent, self.output.resolve()) + with tarfile.open(archive) as package: + self.assertFalse(any(m.islnk() or m.issym() for m in package)) + for name in ['original.node', 'copy.node']: + member = package.getmember('package/node_modules/native-fixture/' + name) + self.assertTrue(member.isfile()) + self.assertEqual(member.mode & 0o777, 0o755) + self.assertEqual(package.extractfile(member).read(), b'native-bytes') + self.assertEqual(package.extractfile('package/built.txt').read(), b'built-before-copy') + dep = self.source / 'node_modules/native-fixture' + self.assertEqual((dep / 'original.node').stat().st_ino, (dep / 'copy.node').stat().st_ino) + self.assertEqual((self.source / 'built.txt').read_text(), 'changed-after-pack') + self.assertEqual(list(self.root.glob('npm-pack-staged.*')), []) + + def test_hook_failure_does_not_produce_an_artifact(self): + self.manifest['scripts']['prepublishOnly'] = 'node -e "process.exit(7)"' + (self.source / 'package.json').write_text(json.dumps(self.manifest)) + self.assertNotEqual(self.pack().returncode, 0) + self.assertEqual(list(self.output.iterdir()), []) + + def test_smoke_failure_does_not_produce_an_artifact(self): + (self.source / 'smoke.js').write_text('process.exit(8)') + self.assertNotEqual(self.pack().returncode, 0) + self.assertEqual(list(self.output.iterdir()), []) + self.assertEqual(list(self.root.glob('npm-pack-staged.*')), []) + + def test_output_inside_source_is_rejected(self): + self.output = self.source + result = self.pack() + self.assertNotEqual(result.returncode, 0) + self.assertIn('outside the source', result.stderr) + + def test_existing_artifact_is_not_overwritten(self): + artifact = self.output / 'staging-fixture-1.0.0.tgz' + artifact.write_bytes(b'keep-me') + self.assertNotEqual(self.pack().returncode, 0) + self.assertEqual(artifact.read_bytes(), b'keep-me') + + def test_stable_and_prerelease_publish_the_exact_dry_run_artifact(self): + commands = self.root / 'commands' + commands.mkdir() + real_npm = shutil.which('npm') + for name, body in { + 'git': r'''#!/usr/bin/env python3 +import json, os, sys +args = sys.argv[1:] +with open(os.environ['GIT_LOG'], 'a') as f: + f.write(json.dumps(args) + '\n') +if args[0] == 'branch': + print('trunk') +elif args[0] not in ('fetch', 'checkout', 'diff-index', 'config', 'add', 'commit', 'push'): + sys.exit(9) +''', + 'gh': r'''#!/usr/bin/env python3 +import json, os, sys +from pathlib import Path +args = sys.argv[1:] +with open(os.environ['GH_LOG'], 'a') as f: + f.write(json.dumps(args) + '\n') +if args[:2] == ['pr', 'diff']: + print('package.json') +elif args[:2] == ['pr', 'create']: + assert 'development version' in Path(args[args.index('--body-file') + 1]).read_text() + print('https://example.invalid/pull/1') +''', + 'npm': r'''#!/usr/bin/env python3 +import hashlib, json, os, subprocess, sys, tarfile +args = sys.argv[1:] +if args[0] == 'view': + print('0.9.0') +elif args[0] in ('ci', 'rebuild'): + pass +elif args[0] == 'publish': + artifact = args[1] + assert artifact.endswith('.tgz'), args + assert '--ignore-scripts' in args, args + with tarfile.open(artifact) as t: + assert not any(m.islnk() or m.issym() for m in t) + with open(artifact, 'rb') as f: + digest = hashlib.sha256(f.read()).hexdigest() + with open(os.environ['PUBLISH_LOG'], 'a') as f: + f.write(json.dumps({'args': args, 'sha256': digest}) + '\n') +else: + sys.exit(subprocess.call([os.environ['REAL_NPM'], *args])) +''', + }.items(): + script = commands / name + script.write_text(body) + script.chmod(0o755) + # The stable action supplies latest; test the prerelease tag independently. + self.manifest['scripts']['prepublishOnly'] = 'node -e "if(process.env.NODE_AUTH_TOKEN)process.exit(1)"' + (self.source / 'package.json').write_text(json.dumps(self.manifest)) + for action in ['npm-publish', 'npm-publish-prerelease']: + with self.subTest(action=action): + log = self.root / (action + '.jsonl') + env = {**self.env, 'PATH': str(commands) + os.pathsep + os.environ['PATH'], + 'REAL_NPM': real_npm, 'PUBLISH_LOG': str(log), 'CI': 'true', + 'GIT_LOG': str(log) + '.git', 'GH_LOG': str(log) + '.gh', + 'GITHUB_ACTIONS': 'true', 'PROVENANCE': 'true', + 'USE_TRUSTED_PUBLISHING': 'true', + 'SMOKE_SCRIPT': 'smoke', 'SKIP_BUMP_TO_DEV': 'false', + 'CONVENTIONAL_COMMITS': 'true', 'PR_ASSIGNEE': 'fixture-actor', + 'PR_HEAD_REF': 'release/patch--trunk', 'PR_NUMBER': '1', 'NPM_TAG': 'next'} + result = subprocess.run(['bash', str(ROOT / ('publish.sh' if action == 'npm-publish' else 'publish-prerelease.sh'))], + cwd=self.source, env=env, text=True, capture_output=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + calls = [json.loads(line) for line in log.read_text().splitlines()] + self.assertEqual(len(calls), 2) + self.assertEqual(calls[0]['args'][1], calls[1]['args'][1]) + self.assertEqual(calls[0]['sha256'], calls[1]['sha256']) + self.assertIn('--dry-run', calls[0]['args']) + self.assertNotIn('--dry-run', calls[1]['args']) + self.assertIn('--provenance', calls[1]['args']) + self.assertFalse(Path(calls[1]['args'][1]).exists(), 'temporary artifact must be cleaned') + expected_tag = 'latest' if action == 'npm-publish' else 'next' + for call in calls: + self.assertEqual(call['args'][call['args'].index('--tag') + 1], expected_tag) + github_calls = [json.loads(line) for line in Path(str(log) + '.gh').read_text().splitlines()] + release = next(call for call in github_calls if call[:2] == ['release', 'create']) + if action == 'npm-publish': + self.assertNotIn('--prerelease', release) + git_calls = [json.loads(line) for line in Path(str(log) + '.git').read_text().splitlines()] + self.assertIn(['checkout', '-b', 'dev-release/v1.0.1-dev.0'], git_calls) + pr = next(call for call in github_calls if call[:2] == ['pr', 'create']) + self.assertEqual(pr[pr.index('--head') + 1], 'dev-release/v1.0.1-dev.0') + self.assertEqual(json.loads((self.source / 'package.json').read_text())['version'], '1.0.1-dev.0') + else: + self.assertIn('--prerelease', release) + self.assertFalse(any(call[:2] == ['pr', 'create'] for call in github_calls)) + + + def test_release_validation_stops_before_install_or_publish(self): + commands = self.root / 'commands' + commands.mkdir() + for name, body in { + 'git': '#!/bin/sh\ncase "$1" in branch) echo "${MOCK_BRANCH:-trunk}" ;; diff-index) exit "${MOCK_DIRTY:-0}" ;; fetch|checkout) ;; *) exit 9 ;; esac\n', + 'gh': '#!/bin/sh\n[ "${MOCK_GH_FAILURE:-0}" = 0 ] || exit 1\nprintf "%s\\n" "${MOCK_PR_FILE:-package.json}"\n', + 'npm': '#!/bin/sh\nif [ "$1" = view ]; then echo 0.9.0; else touch "$UNEXPECTED_NPM"; exit 9; fi\n', + }.items(): + script = commands / name + script.write_text(body) + script.chmod(0o755) + marker_file = self.root / 'unexpected-npm' + base = {**self.env, 'PATH': str(commands) + os.pathsep + os.environ['PATH'], + 'PR_HEAD_REF': 'release/patch--trunk', 'PR_NUMBER': '1', + 'USE_TRUSTED_PUBLISHING': 'true', 'UNEXPECTED_NPM': str(marker_file)} + for script, overrides, code in [ + ('publish.sh', {'MOCK_PR_FILE': 'src/changed.js'}, 200), + ('publish.sh', {'MOCK_PR_FILE': 'package.json.bak'}, 200), + ('publish.sh', {'MOCK_GH_FAILURE': '1'}, 1), + ('publish.sh', {'PR_HEAD_REF': 'release/invalid--trunk'}, 201), + ('publish.sh', {'MOCK_BRANCH': 'different'}, 203), + ('publish.sh', {'MOCK_DIRTY': '1'}, 204), + ('publish-prerelease.sh', {'MOCK_DIRTY': '1'}, 204), + ]: + with self.subTest(script=script, overrides=overrides): + result = subprocess.run(['bash', str(ROOT / script)], cwd=self.source, + env={**base, **overrides}, capture_output=True, text=True) + self.assertEqual(result.returncode, code, result.stdout + result.stderr) + self.assertFalse(marker_file.exists()) + + def test_validator_rejects_unsafe_archives(self): + for name, kind, identity in [ + ('package/hard.node', tarfile.LNKTYPE, self.manifest), + ('package/soft.node', tarfile.SYMTYPE, self.manifest), + ('package/../escape', tarfile.REGTYPE, self.manifest), + ('/absolute', tarfile.REGTYPE, self.manifest), + ('package/device', tarfile.CHRTYPE, self.manifest), + ('package/package.json', tarfile.REGTYPE, self.manifest), + ('package/safe', tarfile.REGTYPE, {'name': 'wrong', 'version': '1.0.0'}), + ]: + with self.subTest(name=name, kind=kind): + archive = self.output / 'bad.tgz' + with tarfile.open(archive, 'w:gz') as package: + data = json.dumps(identity).encode() + manifest = tarfile.TarInfo('package/package.json') + manifest.size = len(data) + package.addfile(manifest, io.BytesIO(data)) + entry = tarfile.TarInfo(name) + entry.type = kind + if kind in (tarfile.LNKTYPE, tarfile.SYMTYPE): + entry.linkname = 'package/package.json' + package.addfile(entry) + with self.assertRaises(ValueError): + validator.validate(archive, self.source / 'package.json') + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/scripts/npm-release/validate.py b/.github/scripts/npm-release/validate.py new file mode 100644 index 000000000..3ec6acced --- /dev/null +++ b/.github/scripts/npm-release/validate.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Reject links/unsafe entries and verify the identity of an npm artifact.""" +import json +import sys +import tarfile +from pathlib import PurePosixPath + + +def validate(archive, manifest): + with open(manifest, encoding="utf-8") as source: + expected = json.load(source) + seen = set() + packed = None + with tarfile.open(archive, "r:gz") as package: + for entry in package: + path = PurePosixPath(entry.name) + if (path.is_absolute() or ".." in path.parts or not path.parts + or path.parts[0] != "package" or path.as_posix() in seen): + raise ValueError(f"Unsafe or duplicate archive entry: {entry.name}") + seen.add(path.as_posix()) + if entry.islnk() or entry.issym(): + raise ValueError(f"Forbidden archive link: {entry.name} -> {entry.linkname}") + if not (entry.isfile() or entry.isdir()): + raise ValueError(f"Unsupported archive entry: {entry.name}") + if entry.name == "package/package.json": + packed = json.load(package.extractfile(entry)) + if not packed or any(packed.get(k) != expected.get(k) for k in ("name", "version")): + raise ValueError("Archive package name/version does not match the source") + print(f"Validated {packed['name']}@{packed['version']}: {len(seen)} entries, no links") + + +if __name__ == "__main__": + validate(*sys.argv[1:]) diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index f8bd70757..f8cef00a3 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -33,15 +33,10 @@ jobs: id-token: write pull-requests: write steps: - - uses: Automattic/vip-actions/npm-publish@a21de3c167c01a17bde269771490fda0b8fdbb9c # staged npm artifacts + - uses: actions/checkout@v7 + - uses: ./.github/actions/npm-release with: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - npm-version: '11' - STAGE_PACKAGE: 'true' - SMOKE_SCRIPT: 'smoke:release' - USE_TRUSTED_PUBLISHING: 'true' - PROVENANCE: 'true' - CONVENTIONAL_COMMITS: 'true' + mode: stable recover-stable: name: Recover stable release @@ -72,6 +67,17 @@ jobs: '.tagName == $version and .isDraft == false and .isPrerelease == false' \ "$RUNNER_TEMP/recovery-release.json" + - name: Check out release tooling from the workflow revision + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Preserve release tooling before checking out the old tag + run: | + tools_dir=$(mktemp -d "$RUNNER_TEMP/vip-release-tools.XXXXXXXX") + cp -R "$GITHUB_WORKSPACE/.github/scripts/npm-release/." "$tools_dir/" + printf 'VIP_RELEASE_TOOLS=%s\n' "$tools_dir" >> "$GITHUB_ENV" + - uses: actions/checkout@v7 with: ref: refs/tags/${{ inputs.release_version }} @@ -115,12 +121,13 @@ jobs: npm rebuild npm run prepare --if-present npm test + - name: Stage, pack and validate release id: package - uses: Automattic/vip-actions/npm-pack-staged@a21de3c167c01a17bde269771490fda0b8fdbb9c # staged npm artifacts - with: - tag: latest - smoke-script: smoke:release + run: | + output_dir=$(mktemp -d "$RUNNER_TEMP/npm-artifact.XXXXXXXX") + tarball=$(bash "$VIP_RELEASE_TOOLS/pack.sh" "$GITHUB_WORKSPACE" "$output_dir" latest smoke:release) + printf 'tarball=%s\n' "$tarball" >> "$GITHUB_OUTPUT" - name: Verify publishing the staged tarball env: @@ -163,12 +170,8 @@ jobs: id-token: write pull-requests: write steps: - - uses: Automattic/vip-actions/npm-publish-prerelease@a21de3c167c01a17bde269771490fda0b8fdbb9c # staged npm artifacts + - uses: actions/checkout@v7 + - uses: ./.github/actions/npm-release with: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - npm-version: '11' - STAGE_PACKAGE: 'true' - SMOKE_SCRIPT: 'smoke:release' - USE_TRUSTED_PUBLISHING: 'true' - PROVENANCE: 'true' - NPM_TAG: ${{ inputs.npm_tag }} + mode: prerelease + npm-tag: ${{ inputs.npm_tag }} diff --git a/.github/workflows/npm-release-tests.yml b/.github/workflows/npm-release-tests.yml new file mode 100644 index 000000000..3e41bc61d --- /dev/null +++ b/.github/workflows/npm-release-tests.yml @@ -0,0 +1,28 @@ +name: Test npm release packaging +on: + pull_request: + paths: + - '.github/scripts/npm-release/**' + - '.github/actions/npm-release/**' + - '.github/workflows/npm-publish.yml' + - '.github/workflows/npm-release-tests.yml' + push: + branches: [trunk] + paths: + - '.github/scripts/npm-release/**' + - '.github/actions/npm-release/**' + - '.github/workflows/npm-publish.yml' + - '.github/workflows/npm-release-tests.yml' +permissions: + contents: read +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: '24' + - run: npm install --global npm@11 + - run: shellcheck .github/scripts/npm-release/*.sh + - run: python3 -m unittest discover -s .github/scripts/npm-release/tests -v diff --git a/docs/NPM-RELEASE-RECOVERY.md b/docs/NPM-RELEASE-RECOVERY.md index 5f98ba451..39143eec7 100644 --- a/docs/NPM-RELEASE-RECOVERY.md +++ b/docs/NPM-RELEASE-RECOVERY.md @@ -27,11 +27,14 @@ documentation job fails, rerun only the failed jobs. ## How the release artifact is prepared -Stable releases, prereleases and recovery use the shared `npm-pack-staged` -helper in `Automattic/vip-actions`. After building and testing, it runs the +Stable releases, prereleases and recovery use the local +[release tooling](../.github/scripts/npm-release/README.md) in this repository. +No `vip-actions` changes are required. After building and testing, it runs the prepublish/pack preparation hooks in the source checkout and uses `rsync -a` without `-H` to copy into a fresh staging directory. This turns native dependency -hard links into independent files without changing the source checkout's links. +hard links into independent files without changing the source checkout's links. The workflow saves the current +release tools before checking out an old tag, so that tag need not contain the +new helper. The helper packs with lifecycle scripts disabled, validates the archive and runs `smoke:release` inside an extraction of that exact tarball. Validation rejects