From e40a0c957452da478024c8007f0daba6a083c97d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 24 Sep 2026 10:36:08 -0700 Subject: [PATCH 1/6] quest: open the tooling line Co-Authored-By: Claude Opus 5.5 From 5dda9362257a3a99bd8880fca063dcf7bc70369d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 24 Sep 2026 17:09:14 -0700 Subject: [PATCH 2/6] chore(just): thin justfiles with one impact map (#4058) Co-authored-by: Claude Opus 5.5 --- .github/justfile | 22 +- .github/scripts/package-binary.test.sh | 44 - .github/workflows/alert.yml | 2 +- .github/workflows/docker.yml | 2 +- .github/workflows/interop.yml | 2 +- .github/workflows/libmoq.yml | 14 +- .github/workflows/moq-cli.yml | 26 +- .github/workflows/moq-gst.yml | 16 +- .github/workflows/moq-relay.yml | 24 +- .github/workflows/nightly.yml | 3 +- .github/workflows/obs.yml | 1 + .github/workflows/release-brew.yml | 4 +- .github/workflows/release-dart-ffi.yml | 10 +- .github/workflows/release-dart.yml | 6 +- .github/workflows/release-go-ffi.yml | 14 +- .github/workflows/release-go.yml | 6 +- .github/workflows/release-kt-ffi.yml | 4 +- .github/workflows/release-kt-lib.yml | 12 +- .github/workflows/release-py-ffi.yml | 2 +- .github/workflows/release-py.yml | 10 +- .github/workflows/release-swift-ffi.yml | 12 +- .github/workflows/release-swift-lib.yml | 32 +- .github/workflows/swift.yml | 4 +- .github/workflows/wasm.yml | 1 + .gitignore | 5 +- Cargo.toml | 2 +- bench/justfile | 19 - cpp/obs/justfile | 386 +------- dart/justfile | 15 +- dart/moq_ffi/hook/build.dart | 2 +- dart/moq_ffi/pubspec.yaml | 2 +- demo/justfile | 6 - doc/setup/dev.md | 5 + drafts/justfile | 79 +- flake.nix | 13 +- go/.gitignore | 2 +- go/ffi/README.md | 8 +- go/ffi/moq/cgo.go | 6 +- go/justfile | 28 +- go/wrapper/README.md | 2 +- go/wrapper/go.mod | 2 +- js/common/workers.test.ts | 45 - js/justfile | 98 +- js/wasm/.gitignore | 2 +- js/wasm/README.md | 2 +- justfile | 822 +--------------- kt/.gitignore | 2 +- kt/README.md | 1 - kt/justfile | 32 +- kt/moq-ffi/build.gradle.kts | 2 +- py/justfile | 61 +- quest/m1/browser-benchmarks.md | 2 +- quest/m1/dart-leak.md | 2 +- quest/m1/performance-comparisons.md | 8 +- quest/m1/performance-profiles.md | 2 +- quest/m1/tooling/README.md | 1 - quest/m1/tooling/justfiles.md | 93 -- quest/m1/tooling/workflows-call-just.md | 4 - rs/justfile | 902 ++---------------- rs/moq-gst/package.sh | 2 +- rs/moq-gst/scrub.sh | 2 +- rs/moq-wasm/README.md | 4 +- rs/scripts/doc-names.jq | 31 - rs/scripts/package-nfpm.test.sh | 68 -- sh/clean.sh | 72 ++ {dart/scripts => sh/dart}/check.sh | 11 +- {dart/scripts => sh/dart}/generate.sh | 2 +- {dart/scripts => sh/dart}/package.sh | 2 +- {dart/scripts => sh/dart}/publish.sh | 4 +- sh/dispatch.sh | 194 ++++ sh/drafts/build.sh | 20 + sh/drafts/check.sh | 31 + sh/drafts/publish.sh | 44 + {.github/scripts => sh/gh}/alert.sh | 2 +- {.github/scripts => sh/gh}/install-formula.py | 0 .../scripts => sh/gh}/install-formula.test.py | 0 {.github/scripts => sh/gh}/release.sh | 0 {.github/scripts => sh/gh}/release.test.sh | 0 {.github/scripts => sh/gh}/render-formula.sh | 0 .../scripts => sh/gh}/trigger-repo-publish.sh | 0 {go/scripts => sh/go}/check-errors.sh | 0 {go/scripts => sh/go}/check.sh | 8 +- {go/scripts => sh/go}/package-ffi.sh | 4 +- {go/scripts => sh/go}/package-wrapper.sh | 4 +- {go/scripts => sh/go}/publish-ffi.sh | 0 {go/scripts => sh/go}/publish-wrapper.sh | 0 {go/scripts => sh/go}/publish-wrapper.test.sh | 6 +- {go/scripts => sh/go}/stage.sh | 4 +- sh/js/workers.sh | 16 + {kt/scripts => sh/kt}/check.sh | 4 +- {kt/scripts => sh/kt}/generate.sh | 4 +- {kt/scripts => sh/kt}/package.sh | 4 +- sh/markdown.sh | 69 ++ sh/obs/check.sh | 68 ++ sh/obs/compile.sh | 67 ++ sh/obs/fix.sh | 12 + sh/obs/includes.sh | 76 ++ sh/obs/preset.sh | 22 + sh/obs/test.sh | 43 + sh/obs/unit.sh | 54 ++ sh/rs/c-tests.sh | 60 ++ sh/rs/fuzz.sh | 40 + {rs/scripts => sh/rs}/media-features.sh | 0 {rs/scripts => sh/rs}/package-binary.sh | 2 +- {rs/scripts => sh/rs}/package-nfpm.sh | 0 {rs/scripts => sh/rs}/package-rename.test.py | 2 +- {rs/scripts => sh/rs}/package-windows.sh | 2 +- sh/rs/package.sh | 51 + {rs/scripts => sh/rs}/scrub-macho.sh | 0 sh/rs/select.sh | 136 +++ {rs/scripts => sh/rs}/stats-docs.py | 0 sh/rs/tokio-features.sh | 28 + sh/rs/vulkan-cuda.sh | 17 + sh/shell.sh | 27 + {swift/scripts => sh/swift}/check.sh | 2 +- {swift/scripts => sh/swift}/package-ffi.sh | 4 +- {swift/scripts => sh/swift}/package.sh | 4 +- {swift/scripts => sh/swift}/publish-ffi.sh | 0 {swift/scripts => sh/swift}/publish.sh | 0 {swift/scripts => sh/swift}/verify-ffi.sh | 4 +- {swift/scripts => sh/swift}/verify.sh | 4 +- swift/.gitignore | 2 +- swift/.spi.yml | 2 +- swift/Package.swift | 4 +- swift/Package.swift.template | 2 +- swift/README.md | 7 +- swift/ffi/Package.swift.template | 2 +- swift/justfile | 24 +- test/README.md | 22 - test/interop/README.md | 4 +- test/interop/clients/go/go.mod | 2 +- test/interop/clients/js-native/subscribe.ts | 13 +- test/interop/interop.sh | 14 +- test/justfile | 68 +- test/wasm/README.md | 2 +- test/wasm/run.sh | 2 +- 136 files changed, 1537 insertions(+), 2906 deletions(-) delete mode 100755 .github/scripts/package-binary.test.sh delete mode 100755 bench/justfile delete mode 100644 js/common/workers.test.ts delete mode 100644 quest/m1/tooling/justfiles.md delete mode 100644 rs/scripts/doc-names.jq delete mode 100755 rs/scripts/package-nfpm.test.sh create mode 100755 sh/clean.sh rename {dart/scripts => sh/dart}/check.sh (88%) rename {dart/scripts => sh/dart}/generate.sh (96%) rename {dart/scripts => sh/dart}/package.sh (97%) rename {dart/scripts => sh/dart}/publish.sh (84%) create mode 100755 sh/dispatch.sh create mode 100755 sh/drafts/build.sh create mode 100755 sh/drafts/check.sh create mode 100755 sh/drafts/publish.sh rename {.github/scripts => sh/gh}/alert.sh (99%) rename {.github/scripts => sh/gh}/install-formula.py (100%) rename {.github/scripts => sh/gh}/install-formula.test.py (100%) rename {.github/scripts => sh/gh}/release.sh (100%) rename {.github/scripts => sh/gh}/release.test.sh (100%) rename {.github/scripts => sh/gh}/render-formula.sh (100%) rename {.github/scripts => sh/gh}/trigger-repo-publish.sh (100%) rename {go/scripts => sh/go}/check-errors.sh (100%) rename {go/scripts => sh/go}/check.sh (84%) rename {go/scripts => sh/go}/package-ffi.sh (98%) rename {go/scripts => sh/go}/package-wrapper.sh (97%) rename {go/scripts => sh/go}/publish-ffi.sh (100%) rename {go/scripts => sh/go}/publish-wrapper.sh (100%) rename {go/scripts => sh/go}/publish-wrapper.test.sh (97%) rename {go/scripts => sh/go}/stage.sh (98%) create mode 100755 sh/js/workers.sh rename {kt/scripts => sh/kt}/check.sh (89%) rename {kt/scripts => sh/kt}/generate.sh (96%) rename {kt/scripts => sh/kt}/package.sh (97%) create mode 100755 sh/markdown.sh create mode 100755 sh/obs/check.sh create mode 100755 sh/obs/compile.sh create mode 100755 sh/obs/fix.sh create mode 100755 sh/obs/includes.sh create mode 100755 sh/obs/preset.sh create mode 100755 sh/obs/test.sh create mode 100755 sh/obs/unit.sh create mode 100755 sh/rs/c-tests.sh create mode 100755 sh/rs/fuzz.sh rename {rs/scripts => sh/rs}/media-features.sh (100%) rename {rs/scripts => sh/rs}/package-binary.sh (99%) rename {rs/scripts => sh/rs}/package-nfpm.sh (100%) rename {rs/scripts => sh/rs}/package-rename.test.py (98%) rename {rs/scripts => sh/rs}/package-windows.sh (98%) create mode 100755 sh/rs/package.sh rename {rs/scripts => sh/rs}/scrub-macho.sh (100%) create mode 100755 sh/rs/select.sh rename {rs/scripts => sh/rs}/stats-docs.py (100%) create mode 100755 sh/rs/tokio-features.sh create mode 100755 sh/rs/vulkan-cuda.sh create mode 100755 sh/shell.sh rename {swift/scripts => sh/swift}/check.sh (98%) rename {swift/scripts => sh/swift}/package-ffi.sh (98%) rename {swift/scripts => sh/swift}/package.sh (98%) rename {swift/scripts => sh/swift}/publish-ffi.sh (100%) rename {swift/scripts => sh/swift}/publish.sh (100%) rename {swift/scripts => sh/swift}/verify-ffi.sh (96%) rename {swift/scripts => sh/swift}/verify.sh (96%) diff --git a/.github/justfile b/.github/justfile index f341da7729..8a3fb9ff67 100644 --- a/.github/justfile +++ b/.github/justfile @@ -1,13 +1,13 @@ -set fallback +set working-directory := '..' -# actionlint is skipped when absent locally; the root `_tools` preflight makes -# it a hard requirement under MOQ_STRICT, so CI can't skip it. - -# Lint GitHub Actions workflows. +# Lint GitHub Actions workflows and their alert coverage. check: - python3 {{ source_directory() }}/scripts/install-formula.test.py - {{ source_directory() }}/scripts/release.test.sh - @if command -v nfpm >/dev/null 2>&1 && command -v envsubst >/dev/null 2>&1 && command -v dpkg-deb >/dev/null 2>&1 && command -v rpm >/dev/null 2>&1; then python3 {{ source_directory() }}/../rs/scripts/package-rename.test.py; fi - @if command -v actionlint >/dev/null 2>&1; then actionlint; fi - {{ source_directory() }}/scripts/alert.sh check-coverage - @if command -v rustc >/dev/null 2>&1; then {{ source_directory() }}/scripts/package-binary.test.sh; fi + actionlint + sh/gh/alert.sh check-coverage + +# Tests of the release tooling itself; nightly, not on every pull request. +test: + python3 sh/gh/install-formula.test.py + sh/gh/release.test.sh + python3 sh/rs/package-rename.test.py + sh/go/publish-wrapper.test.sh diff --git a/.github/scripts/package-binary.test.sh b/.github/scripts/package-binary.test.sh deleted file mode 100755 index 2b2554b6c1..0000000000 --- a/.github/scripts/package-binary.test.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -WORKSPACE_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" - -tmp="$(mktemp -d)" -trap 'rm -rf "$tmp"' EXIT - -binary="$tmp/moq-relay" -printf '#!/usr/bin/env sh\necho test\n' >"$binary" -chmod 0755 "$binary" -target="$(rustc -vV | awk '/^host:/ {print $2}')" - -# This test covers archive staging, not the macOS Mach-O rewrite. -mkdir "$tmp/bin" -printf '#!/usr/bin/env sh\nprintf "Linux\\n"\n' >"$tmp/bin/uname" -chmod 0755 "$tmp/bin/uname" - -PATH="$tmp/bin:$PATH" "$WORKSPACE_DIR/rs/scripts/package-binary.sh" \ - --crate moq-relay \ - --bin moq-relay \ - --binary "$binary" \ - --bare \ - --version 0.14.13 \ - --target "$target" \ - --output "$tmp/dist" - -name="moq-relay-v0.14.13-$target" -bare="$tmp/dist/$name" -archive="$tmp/dist/$name.tar.gz" - -[[ -x "$bare" ]] -[[ -f "$archive" ]] - -mkdir "$tmp/extracted" -tar -xzf "$archive" -C "$tmp/extracted" -[[ -x "$tmp/extracted/$name/bin/moq-relay" ]] -[[ -f "$tmp/extracted/$name/LICENSE-MIT" ]] -[[ -f "$tmp/extracted/$name/LICENSE-APACHE" ]] -cmp "$binary" "$bare" -cmp "$binary" "$tmp/extracted/$name/bin/moq-relay" - -echo "release assets package together without path collisions" diff --git a/.github/workflows/alert.yml b/.github/workflows/alert.yml index 4180d6bd84..78d9c48125 100644 --- a/.github/workflows/alert.yml +++ b/.github/workflows/alert.yml @@ -88,4 +88,4 @@ jobs: RUN_REF: ${{ github.event.workflow_run.head_branch }} RUN_TITLE: ${{ github.event.workflow_run.display_title }} RUN_ACTOR: ${{ github.event.workflow_run.triggering_actor.login }} - run: .github/scripts/alert.sh discord + run: sh/gh/alert.sh discord diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2a3c3df447..a512d5696c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -53,7 +53,7 @@ jobs: exit 1 fi echo "target=${target}" >> "$GITHUB_OUTPUT" - .github/scripts/release.sh parse-version "$target" + sh/gh/release.sh parse-version "$target" build: name: Build ${{ matrix.platform }} diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index 7116bd39c1..e7eab95dee 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -24,7 +24,7 @@ on: - "test/justfile" - "package.json" # The Go client builds against the modules this script stages. - - "go/scripts/**" + - "sh/go/**" - ".github/workflows/interop.yml" concurrency: diff --git a/.github/workflows/libmoq.yml b/.github/workflows/libmoq.yml index 7eb12759de..3ad2daf514 100644 --- a/.github/workflows/libmoq.yml +++ b/.github/workflows/libmoq.yml @@ -55,7 +55,7 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version libmoq + run: sh/gh/release.sh parse-version libmoq - name: Build and package shell: bash @@ -87,11 +87,11 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version libmoq + run: sh/gh/release.sh parse-version libmoq - name: Find previous tag id: prev_tag - run: .github/scripts/release.sh prev-tag libmoq + run: sh/gh/release.sh prev-tag libmoq - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -105,7 +105,7 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "libmoq v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: .github/scripts/release.sh create artifacts + run: sh/gh/release.sh create artifacts # The OBS plugin (cpp/obs) statically links libmoq, so it ships with every # libmoq release: rebuild it against the release we just published (no @@ -140,7 +140,7 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version libmoq + run: sh/gh/release.sh parse-version libmoq # Native Xcode / Visual Studio; libobs/Qt6/ffmpeg come from the obs-deps # bundle and libmoq from the release we just published (--libmoq-release). @@ -173,7 +173,7 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version libmoq + run: sh/gh/release.sh parse-version libmoq # Only the obs-moq archives: the libmoq-* artifacts from the build job # belong to the libmoq release, not this one. @@ -208,4 +208,4 @@ jobs: RELEASE_TAG: "obs-moq-v${{ steps.parse.outputs.version }}" RELEASE_TITLE: "obs-moq v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: .github/scripts/release.sh create artifacts + run: sh/gh/release.sh create artifacts diff --git a/.github/workflows/moq-cli.yml b/.github/workflows/moq-cli.yml index 141e47ecb8..9e82930991 100644 --- a/.github/workflows/moq-cli.yml +++ b/.github/workflows/moq-cli.yml @@ -47,7 +47,7 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version moq-cli + run: sh/gh/release.sh parse-version moq-cli - name: Set up packaging tools uses: ./.github/actions/setup-packaging @@ -64,7 +64,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} TARGET: ${{ matrix.target }} run: | - ./rs/scripts/package-binary.sh \ + ./sh/rs/package-binary.sh \ --crate moq-cli \ --bin moq \ --binary "target/${TARGET}/release/moq" \ @@ -80,8 +80,8 @@ jobs: ARCH: ${{ matrix.deb-arch }} BINARY_PATH: target/${{ matrix.target }}/release/moq run: | - rs/scripts/package-nfpm.sh packaging/moq-cli/nfpm.yaml deb dist/ - rs/scripts/package-nfpm.sh packaging/moq-cli/transition.yaml deb dist/ + sh/rs/package-nfpm.sh packaging/moq-cli/nfpm.yaml deb dist/ + sh/rs/package-nfpm.sh packaging/moq-cli/transition.yaml deb dist/ - name: Package .rpm shell: bash @@ -89,7 +89,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} ARCH: ${{ matrix.rpm-arch }} BINARY_PATH: target/${{ matrix.target }}/release/moq - run: rs/scripts/package-nfpm.sh packaging/moq-cli/nfpm.yaml rpm dist/ + run: sh/rs/package-nfpm.sh packaging/moq-cli/nfpm.yaml rpm dist/ - name: Upload artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -121,12 +121,12 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version moq-cli + run: sh/gh/release.sh parse-version moq-cli - name: Build and package tarball shell: bash run: | - ./rs/scripts/package-binary.sh \ + ./sh/rs/package-binary.sh \ --crate moq-cli \ --bin moq \ --target ${{ matrix.target }} \ @@ -169,12 +169,12 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version moq-cli + run: sh/gh/release.sh parse-version moq-cli - name: Build and package zip shell: bash run: | - ./rs/scripts/package-windows.sh \ + ./sh/rs/package-windows.sh \ --crate moq-cli \ --bin moq \ --target ${{ matrix.target }} \ @@ -204,11 +204,11 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version moq-cli + run: sh/gh/release.sh parse-version moq-cli - name: Find previous tag id: prev_tag - run: .github/scripts/release.sh prev-tag moq-cli + run: sh/gh/release.sh prev-tag moq-cli - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -222,10 +222,10 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "moq-cli v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: .github/scripts/release.sh create artifacts + run: sh/gh/release.sh create artifacts - name: Trigger apt/rpm repo publish env: GH_TOKEN: ${{ github.token }} TAG: ${{ github.ref_name }} - run: .github/scripts/trigger-repo-publish.sh + run: sh/gh/trigger-repo-publish.sh diff --git a/.github/workflows/moq-gst.yml b/.github/workflows/moq-gst.yml index 2c93d1ac9a..139d066c0f 100644 --- a/.github/workflows/moq-gst.yml +++ b/.github/workflows/moq-gst.yml @@ -41,7 +41,7 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version moq-gst + run: sh/gh/release.sh parse-version moq-gst - name: Build and package (tarball) shell: bash @@ -123,7 +123,7 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version moq-gst + run: sh/gh/release.sh parse-version moq-gst - name: Install GStreamer dev libraries shell: bash @@ -160,7 +160,7 @@ jobs: PLUGIN_DIR: /usr/lib/${{ matrix.deb-multiarch }}/gstreamer-1.0 run: | mkdir -p dist - rs/scripts/package-nfpm.sh packaging/moq-gst/nfpm.yaml deb dist/ + sh/rs/package-nfpm.sh packaging/moq-gst/nfpm.yaml deb dist/ - name: Package .rpm shell: bash @@ -172,7 +172,7 @@ jobs: PLUGIN_DIR: /usr/lib64/gstreamer-1.0 run: | mkdir -p dist - rs/scripts/package-nfpm.sh packaging/moq-gst/nfpm.yaml rpm dist/ + sh/rs/package-nfpm.sh packaging/moq-gst/nfpm.yaml rpm dist/ - name: Upload artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -197,11 +197,11 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version moq-gst + run: sh/gh/release.sh parse-version moq-gst - name: Find previous tag id: prev_tag - run: .github/scripts/release.sh prev-tag moq-gst + run: sh/gh/release.sh prev-tag moq-gst - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -215,10 +215,10 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "moq-gst v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: .github/scripts/release.sh create artifacts + run: sh/gh/release.sh create artifacts - name: Trigger apt/rpm repo publish env: GH_TOKEN: ${{ github.token }} TAG: ${{ github.ref_name }} - run: .github/scripts/trigger-repo-publish.sh + run: sh/gh/trigger-repo-publish.sh diff --git a/.github/workflows/moq-relay.yml b/.github/workflows/moq-relay.yml index 761ee0aba3..96496d9731 100644 --- a/.github/workflows/moq-relay.yml +++ b/.github/workflows/moq-relay.yml @@ -50,7 +50,7 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version moq-relay + run: sh/gh/release.sh parse-version moq-relay - name: Set up packaging tools uses: ./.github/actions/setup-packaging @@ -67,7 +67,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} TARGET: ${{ matrix.target }} run: | - ./rs/scripts/package-binary.sh \ + ./sh/rs/package-binary.sh \ --crate moq-relay \ --bin moq-relay \ --binary "target/${TARGET}/release/moq-relay" \ @@ -82,7 +82,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} ARCH: ${{ matrix.deb-arch }} BINARY_PATH: target/${{ matrix.target }}/release/moq-relay - run: rs/scripts/package-nfpm.sh packaging/moq-relay/nfpm.yaml deb dist/ + run: sh/rs/package-nfpm.sh packaging/moq-relay/nfpm.yaml deb dist/ - name: Package .rpm shell: bash @@ -90,7 +90,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} ARCH: ${{ matrix.rpm-arch }} BINARY_PATH: target/${{ matrix.target }}/release/moq-relay - run: rs/scripts/package-nfpm.sh packaging/moq-relay/nfpm.yaml rpm dist/ + run: sh/rs/package-nfpm.sh packaging/moq-relay/nfpm.yaml rpm dist/ - name: Upload artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -122,12 +122,12 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version moq-relay + run: sh/gh/release.sh parse-version moq-relay - name: Build and package tarball shell: bash run: | - ./rs/scripts/package-binary.sh \ + ./sh/rs/package-binary.sh \ --crate moq-relay \ --target ${{ matrix.target }} \ --version ${{ steps.parse.outputs.version }} \ @@ -169,12 +169,12 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version moq-relay + run: sh/gh/release.sh parse-version moq-relay - name: Build and package zip shell: bash run: | - ./rs/scripts/package-windows.sh \ + ./sh/rs/package-windows.sh \ --crate moq-relay \ --target ${{ matrix.target }} \ --version ${{ steps.parse.outputs.version }} \ @@ -203,11 +203,11 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version moq-relay + run: sh/gh/release.sh parse-version moq-relay - name: Find previous tag id: prev_tag - run: .github/scripts/release.sh prev-tag moq-relay + run: sh/gh/release.sh prev-tag moq-relay - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -221,10 +221,10 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "moq-relay v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: .github/scripts/release.sh create artifacts + run: sh/gh/release.sh create artifacts - name: Trigger apt/rpm repo publish env: GH_TOKEN: ${{ github.token }} TAG: ${{ github.ref_name }} - run: .github/scripts/trigger-repo-publish.sh + run: sh/gh/trigger-repo-publish.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 9f73f46109..431d89002e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -121,7 +121,8 @@ jobs: matrix: # `rs uring` is a separate feature compile (off the default set) and # kernel-gated below 6.12. The embedding tests for io_uring live there. - recipe: ["rs doctest --workspace", "rs loom", "test drill-sensitivity", "rs uring"] + # `gh test` exercises the release tooling, which no pull request runs. + recipe: ["rs doctest --workspace", "rs loom", "test drill-sensitivity", "rs uring", "gh test"] steps: - name: Free disk space uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # main diff --git a/.github/workflows/obs.yml b/.github/workflows/obs.yml index 139a60cfeb..0f583c95b0 100644 --- a/.github/workflows/obs.yml +++ b/.github/workflows/obs.yml @@ -48,6 +48,7 @@ on: types: [opened, synchronize, reopened, closed] paths: - "cpp/obs/**" + - "sh/obs/**" - "rs/libmoq/**" - "rs/**/Cargo.toml" - "rs/**/build.rs" diff --git a/.github/workflows/release-brew.yml b/.github/workflows/release-brew.yml index 20fe721156..fb06fe444f 100644 --- a/.github/workflows/release-brew.yml +++ b/.github/workflows/release-brew.yml @@ -98,7 +98,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} run: | mkdir -p rendered - ./.github/scripts/render-formula.sh \ + ./sh/gh/render-formula.sh \ --template ".github/homebrew/Formula/${CRATE}.rb.tmpl" \ --version "$VERSION" \ --release-dir release-assets \ @@ -156,7 +156,7 @@ jobs: EOF fi - python3 ../.github/scripts/install-formula.py "$CRATE" "../rendered/${CRATE}.rb" . + python3 ../sh/gh/install-formula.py "$CRATE" "../rendered/${CRATE}.rb" . if git diff --quiet && [[ -z "$(git status --porcelain)" ]]; then echo "No changes to commit (formula already up to date)." diff --git a/.github/workflows/release-dart-ffi.yml b/.github/workflows/release-dart-ffi.yml index 85f2fff849..59f6c7a61c 100644 --- a/.github/workflows/release-dart-ffi.yml +++ b/.github/workflows/release-dart-ffi.yml @@ -16,7 +16,7 @@ on: paths: - ".github/workflows/release-dart-ffi.yml" - "dart/moq_ffi/**" - - "dart/scripts/**" + - "sh/dart/**" - "rs/moq-ffi/**" - "!rs/moq-ffi/Cargo.toml" @@ -48,7 +48,7 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version moq-ffi + run: sh/gh/release.sh parse-version moq-ffi build: name: Build Native Asset (${{ matrix.target }}) @@ -157,7 +157,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./dart/scripts/package.sh \ + ./sh/dart/package.sh \ --package moq_ffi \ --version "$BUILD_VERSION" \ --output release-out @@ -188,7 +188,7 @@ jobs: - name: Find previous tag id: prev_tag - run: .github/scripts/release.sh prev-tag moq-ffi + run: sh/gh/release.sh prev-tag moq-ffi - name: Download Native Assets uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -204,7 +204,7 @@ jobs: RELEASE_TITLE: "moq-ffi v${{ needs.parse-version.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} RELEASE_CHECKSUMS: SHA256SUMS-dart - run: .github/scripts/release.sh create artifacts + run: sh/gh/release.sh create artifacts publish: name: Publish moq_ffi to pub.dev diff --git a/.github/workflows/release-dart.yml b/.github/workflows/release-dart.yml index 2c099bc318..4df8f205f4 100644 --- a/.github/workflows/release-dart.yml +++ b/.github/workflows/release-dart.yml @@ -12,7 +12,7 @@ on: paths: - ".github/workflows/release-dart.yml" - "dart/moq/**" - - "dart/scripts/**" + - "sh/dart/**" permissions: contents: read @@ -47,7 +47,7 @@ jobs: if [[ "$EVENT_NAME" == "pull_request" ]]; then VERSION=$(sed -n 's/^version: //p' dart/moq/pubspec.yaml | head -1) else - .github/scripts/release.sh parse-version moq-dart + sh/gh/release.sh parse-version moq-dart exit 0 fi echo "version=$VERSION" >> "$GITHUB_OUTPUT" @@ -69,7 +69,7 @@ jobs: env: BUILD_VERSION: ${{ steps.version.outputs.version }} run: | - ./dart/scripts/package.sh \ + ./sh/dart/package.sh \ --package moq \ --version "$BUILD_VERSION" \ --output release-out diff --git a/.github/workflows/release-go-ffi.yml b/.github/workflows/release-go-ffi.yml index 33e21c0c02..d1d8477d38 100644 --- a/.github/workflows/release-go-ffi.yml +++ b/.github/workflows/release-go-ffi.yml @@ -16,10 +16,10 @@ on: pull_request: paths: - ".github/workflows/release-go-ffi.yml" - - ".github/scripts/release.sh" + - "sh/gh/release.sh" - "go/ffi/**" - - "go/scripts/package-ffi.sh" - - "go/scripts/publish-ffi.sh" + - "sh/go/package-ffi.sh" + - "sh/go/publish-ffi.sh" - "rs/moq-ffi/build.sh" permissions: @@ -53,7 +53,7 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version moq-ffi + run: sh/gh/release.sh parse-version moq-ffi build: name: Build moq-ffi (${{ matrix.target }}) @@ -208,7 +208,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./go/scripts/package-ffi.sh \ + ./sh/go/package-ffi.sh \ --version "$BUILD_VERSION" \ --source-dir go/ffi \ --lib-dir libs \ @@ -256,7 +256,7 @@ jobs: GO_FFI_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./go/scripts/publish-ffi.sh + run: ./sh/go/publish-ffi.sh publish-dry-run: name: Publish dry-run @@ -278,4 +278,4 @@ jobs: - name: Dry-run publish to mirror env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: ./go/scripts/publish-ffi.sh --dry-run + run: ./sh/go/publish-ffi.sh --dry-run diff --git a/.github/workflows/release-go.yml b/.github/workflows/release-go.yml index 0f87ed9b74..92300e5f13 100644 --- a/.github/workflows/release-go.yml +++ b/.github/workflows/release-go.yml @@ -105,7 +105,7 @@ jobs: if: github.event_name == 'push' env: FFI_VERSION: ${{ steps.ffi.outputs.version }} - run: .github/scripts/release.sh ffi-unreleased "$FFI_VERSION" + run: sh/gh/release.sh ffi-unreleased "$FFI_VERSION" # Skip on unreleased ffi changes, and when the pinned ffi's mirror tag is # missing (release-plz merged but Release Go FFI hasn't published yet): @@ -160,7 +160,7 @@ jobs: LINE: ${{ steps.line.outputs.line }} FFI_VERSION: ${{ steps.ffi.outputs.version }} run: | - ./go/scripts/package-wrapper.sh --line "$LINE" --ffi-version "$FFI_VERSION" --output release-out + ./sh/go/package-wrapper.sh --line "$LINE" --ffi-version "$FFI_VERSION" --output release-out - name: Upload wrapper module if: steps.gate.outputs.skip != 'true' @@ -203,4 +203,4 @@ jobs: GO_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./go/scripts/publish-wrapper.sh + run: ./sh/go/publish-wrapper.sh diff --git a/.github/workflows/release-kt-ffi.yml b/.github/workflows/release-kt-ffi.yml index 2d72acfbf0..31a6979587 100644 --- a/.github/workflows/release-kt-ffi.yml +++ b/.github/workflows/release-kt-ffi.yml @@ -39,7 +39,7 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version moq-ffi + run: sh/gh/release.sh parse-version moq-ffi build: name: Build moq-ffi (${{ matrix.target }}) @@ -199,7 +199,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./kt/scripts/package.sh \ + ./sh/kt/package.sh \ --version "$BUILD_VERSION" \ --lib-dir libs \ --bindings-dir bindings \ diff --git a/.github/workflows/release-kt-lib.yml b/.github/workflows/release-kt-lib.yml index d135754329..1bec21db75 100644 --- a/.github/workflows/release-kt-lib.yml +++ b/.github/workflows/release-kt-lib.yml @@ -30,7 +30,7 @@ on: - "kt/build.gradle.kts" - "kt/settings.gradle.kts" - ".github/workflows/release-kt-lib.yml" - - ".github/scripts/release.sh" + - "sh/gh/release.sh" pull_request: paths: - "kt/moq/**" @@ -38,7 +38,7 @@ on: - "kt/build.gradle.kts" - "kt/settings.gradle.kts" - ".github/workflows/release-kt-lib.yml" - - ".github/scripts/release.sh" + - "sh/gh/release.sh" - "rs/moq-ffi/build.sh" # Manual trigger so a wrapper bump can be published once the bindings it # depends on land on Maven Central (the guard below skips otherwise). @@ -103,18 +103,18 @@ jobs: - name: Check for unreleased FFI changes id: ffi-diff if: github.event_name != 'workflow_run' - run: .github/scripts/release.sh ffi-unreleased "${{ steps.ffi.outputs.version }}" + run: sh/gh/release.sh ffi-unreleased "${{ steps.ffi.outputs.version }}" - name: Check wrapper publication id: wrapper - run: .github/scripts/release.sh maven-exists dev.moq moq "${{ steps.version.outputs.version }}" + run: sh/gh/release.sh maven-exists dev.moq moq "${{ steps.version.outputs.version }}" # The bindings this checkout generates must be on Maven Central, or the # wrapper's POM range resolves to an older moq-ffi without them. - name: Check bindings publication id: bindings if: github.event_name != 'workflow_run' && steps.wrapper.outputs.exists != 'true' && steps.ffi-diff.outputs.unreleased != 'true' - run: .github/scripts/release.sh maven-exists dev.moq moq-ffi "${{ steps.ffi.outputs.version }}" + run: sh/gh/release.sh maven-exists dev.moq moq-ffi "${{ steps.ffi.outputs.version }}" - name: Decide publish id: decide @@ -202,7 +202,7 @@ jobs: - name: Recheck wrapper publication id: recheck if: github.event_name != 'pull_request' - run: .github/scripts/release.sh maven-exists dev.moq moq "${{ needs.check-version.outputs.version }}" + run: sh/gh/release.sh maven-exists dev.moq moq "${{ needs.check-version.outputs.version }}" - name: Publish to Maven Central if: github.event_name != 'pull_request' && steps.recheck.outputs.exists != 'true' diff --git a/.github/workflows/release-py-ffi.yml b/.github/workflows/release-py-ffi.yml index 2a79e7096e..2e1fa1bb49 100644 --- a/.github/workflows/release-py-ffi.yml +++ b/.github/workflows/release-py-ffi.yml @@ -36,7 +36,7 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version moq-ffi + run: sh/gh/release.sh parse-version moq-ffi # Maturin reads the wheel version from rs/moq-ffi/Cargo.toml (via # `dynamic = ["version"]` in pyproject.toml). If a tag is pushed by diff --git a/.github/workflows/release-py.yml b/.github/workflows/release-py.yml index b2ba9f0cd2..67a2741e8b 100644 --- a/.github/workflows/release-py.yml +++ b/.github/workflows/release-py.yml @@ -19,7 +19,7 @@ on: paths: - "py/moq-rs/**" - ".github/workflows/release-py.yml" - - ".github/scripts/release.sh" + - "sh/gh/release.sh" workflow_run: workflows: ["Release Py FFI"] types: @@ -58,7 +58,7 @@ jobs: - name: Read version id: version - run: .github/scripts/release.sh read-version py/moq-rs/pyproject.toml + run: sh/gh/release.sh read-version py/moq-rs/pyproject.toml - name: Resolve FFI version id: ffi @@ -78,14 +78,14 @@ jobs: if: github.event_name == 'push' env: FFI_VERSION: ${{ steps.ffi.outputs.version }} - run: .github/scripts/release.sh ffi-unreleased "$FFI_VERSION" + run: sh/gh/release.sh ffi-unreleased "$FFI_VERSION" - name: Check FFI on PyPI id: ffi-pypi if: github.event_name == 'push' && steps.ffi-gate.outputs.unreleased != 'true' env: FFI_VERSION: ${{ steps.ffi.outputs.version }} - run: .github/scripts/release.sh pypi-exists moq-ffi "$FFI_VERSION" + run: sh/gh/release.sh pypi-exists moq-ffi "$FFI_VERSION" - name: Unpublished FFI notice if: github.event_name == 'push' && steps.ffi-gate.outputs.unreleased != 'true' && steps.ffi-pypi.outputs.exists != 'true' @@ -100,7 +100,7 @@ jobs: id: pypi env: VERSION: ${{ steps.version.outputs.version }} - run: .github/scripts/release.sh pypi-exists moq-rs "$VERSION" + run: sh/gh/release.sh pypi-exists moq-rs "$VERSION" - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main with: diff --git a/.github/workflows/release-swift-ffi.yml b/.github/workflows/release-swift-ffi.yml index 658c06b654..966a1b86c3 100644 --- a/.github/workflows/release-swift-ffi.yml +++ b/.github/workflows/release-swift-ffi.yml @@ -36,7 +36,7 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version moq-ffi + run: sh/gh/release.sh parse-version moq-ffi build: name: Build moq-ffi (${{ matrix.target }}) @@ -149,7 +149,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./swift/scripts/package-ffi.sh \ + ./sh/swift/package-ffi.sh \ --version "$BUILD_VERSION" \ --lib-dir libs \ --bindings-dir bindings \ @@ -183,7 +183,7 @@ jobs: - name: Find previous tag id: prev_tag - run: .github/scripts/release.sh prev-tag moq-ffi + run: sh/gh/release.sh prev-tag moq-ffi - name: Download package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -197,7 +197,7 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "moq-ffi v${{ needs.parse-version.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: .github/scripts/release.sh create artifacts + run: sh/gh/release.sh create artifacts verify: # Gate the mirror push on actually being able to resolve the staged package @@ -223,7 +223,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./swift/scripts/verify-ffi.sh --tarball "artifacts/moq-ffi-${BUILD_VERSION}-swift-ffi.tar.gz" + ./sh/swift/verify-ffi.sh --tarball "artifacts/moq-ffi-${BUILD_VERSION}-swift-ffi.tar.gz" publish: name: Publish to Swift Package mirror @@ -259,4 +259,4 @@ jobs: SWIFT_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./swift/scripts/publish-ffi.sh + run: ./sh/swift/publish-ffi.sh diff --git a/.github/workflows/release-swift-lib.yml b/.github/workflows/release-swift-lib.yml index 6c5b07fc5a..6361566a57 100644 --- a/.github/workflows/release-swift-lib.yml +++ b/.github/workflows/release-swift-lib.yml @@ -22,22 +22,22 @@ on: - "swift/Package.swift.template" - "swift/Sources/Moq/**" - "swift/Tests/**" - - "swift/scripts/package.sh" - - "swift/scripts/publish.sh" - - "swift/scripts/verify.sh" + - "sh/swift/package.sh" + - "sh/swift/publish.sh" + - "sh/swift/verify.sh" - ".github/workflows/release-swift-lib.yml" - - ".github/scripts/release.sh" + - "sh/gh/release.sh" pull_request: paths: - "swift/VERSION" - "swift/Package.swift.template" - "swift/Sources/Moq/**" - "swift/Tests/**" - - "swift/scripts/package.sh" - - "swift/scripts/publish.sh" - - "swift/scripts/verify.sh" + - "sh/swift/package.sh" + - "sh/swift/publish.sh" + - "sh/swift/verify.sh" - ".github/workflows/release-swift-lib.yml" - - ".github/scripts/release.sh" + - "sh/gh/release.sh" workflow_run: workflows: ["Release Swift FFI"] types: @@ -101,13 +101,13 @@ jobs: if: github.event_name != 'workflow_run' env: FFI_VERSION: ${{ steps.version.outputs.ffi_version }} - run: .github/scripts/release.sh ffi-unreleased "$FFI_VERSION" + run: sh/gh/release.sh ffi-unreleased "$FFI_VERSION" - name: Check FFI mirror tag id: ffi-gate env: FFI_VERSION: ${{ steps.version.outputs.ffi_version }} - run: .github/scripts/release.sh git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" + run: sh/gh/release.sh git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" - name: Unpublished FFI notice if: steps.ffi-diff.outputs.unreleased != 'true' && steps.ffi-gate.outputs.exists != 'true' && github.event_name != 'pull_request' @@ -121,12 +121,12 @@ jobs: id: gate env: VERSION: ${{ steps.version.outputs.version }} - run: .github/scripts/release.sh git-tag-exists moq-dev/moq-swift "$VERSION" + run: sh/gh/release.sh git-tag-exists moq-dev/moq-swift "$VERSION" - name: Package env: BUILD_VERSION: ${{ steps.version.outputs.version }} - run: ./swift/scripts/package.sh --version "$BUILD_VERSION" --output release-out + run: ./sh/swift/package.sh --version "$BUILD_VERSION" --output release-out - name: Upload Swift package uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -158,7 +158,7 @@ jobs: # rather than a hard failure, so the first bootstrap (before # moq-swift-ffi exists) skips the cross-package resolve instead of # failing the job. publish.sh stays idempotent regardless. - run: .github/scripts/release.sh git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" || echo "exists=false" >> "$GITHUB_OUTPUT" + run: sh/gh/release.sh git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" || echo "exists=false" >> "$GITHUB_OUTPUT" - name: Download staged package if: steps.ffi.outputs.exists == 'true' @@ -171,7 +171,7 @@ jobs: if: steps.ffi.outputs.exists == 'true' env: BUILD_VERSION: ${{ needs.build.outputs.version }} - run: ./swift/scripts/verify.sh --tarball "artifacts/moq-${BUILD_VERSION}-swift.tar.gz" + run: ./sh/swift/verify.sh --tarball "artifacts/moq-${BUILD_VERSION}-swift.tar.gz" - name: Skip notice if: needs.build.outputs.ffi_unreleased == 'true' @@ -218,7 +218,7 @@ jobs: SWIFT_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./swift/scripts/publish.sh + run: ./sh/swift/publish.sh publish-dry-run: name: Publish dry-run @@ -240,4 +240,4 @@ jobs: - name: Dry-run publish to mirror env: BUILD_VERSION: ${{ needs.build.outputs.version }} - run: ./swift/scripts/publish.sh --dry-run + run: ./sh/swift/publish.sh --dry-run diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index 7092f0d955..3c20d75907 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -14,6 +14,7 @@ on: branches: [main] paths: - "swift/**" + - "sh/swift/**" - "rs/moq-ffi/**" - "rs/moq-video/**" - "Cargo.lock" @@ -24,6 +25,7 @@ on: types: [opened, synchronize, reopened, closed] paths: - "swift/**" + - "sh/swift/**" - "rs/moq-ffi/**" - "rs/moq-video/**" - "Cargo.lock" @@ -77,4 +79,4 @@ jobs: done - name: Swift check - run: ./swift/scripts/check.sh + run: ./sh/swift/check.sh diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 6cfb329d56..a1291df9dd 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -43,6 +43,7 @@ on: - "test/wasm/**" - "test/lib/**" - "justfile" + - "js/justfile" - "test/justfile" - ".cargo/config.toml" - "package.json" diff --git a/.gitignore b/.gitignore index 414686073c..357feca35a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,9 +7,8 @@ *.local.json # Claude Code scratch space. These are untracked working directories, and -# `just _changed` counts untracked files, so leaving them visible pushes the -# changed-file list past its cap and silently turns every scoped `just check` -# into `check --all`. The worktrees also carry their own biome.jsonc, which the +# `just check` counts untracked files, so leaving them visible would put their +# whole tree in scope. The worktrees also carry their own biome.jsonc, which the # unscoped run then rejects as a nested root config. /.claude/cache/ /.claude/tmp/ diff --git a/Cargo.toml b/Cargo.toml index 5a529c9cd6..009381361b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -318,7 +318,7 @@ debug = true # Size-optimized build for the browser/WASM bindings (rs/moq-wasm). Shipping over # the wire, so optimize for size, not speed (~579KB -> ~471KB gzipped vs the -# default release profile). `just wasm` builds with `--profile wasm-release`. +# default release profile). `just js wasm` builds with `--profile wasm-release`. [profile.wasm-release] inherits = "release" opt-level = "z" diff --git a/bench/justfile b/bench/justfile deleted file mode 100755 index b3fa64c392..0000000000 --- a/bench/justfile +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env just --justfile - -set working-directory := '..' - -# Benchmark the current tree, or compare it with a commit. -compare $BASE="": - #!/usr/bin/env bash - exec bench/run.sh "$BASE" - -# Compare the shared Tokio runtime with independent Tokio and io_uring workers. -runtime $ROUNDS="3" $WORKERS="": - #!/usr/bin/env bash - MOQ_BENCH_RUNTIME_ROUNDS="$ROUNDS" MOQ_BENCH_RUNTIME_WORKERS="$WORKERS" \ - exec bench/run.sh --runtime - -# Compile every binary and feature used by the repository-level harness. -check: - cargo check --locked --package moq-relay --package moq-bench \ - --features moq-relay/io-uring diff --git a/cpp/obs/justfile b/cpp/obs/justfile index 99b658406f..eed0d37344 100644 --- a/cpp/obs/justfile +++ b/cpp/obs/justfile @@ -21,393 +21,39 @@ default: # Configure via CMake presets. MOQ_LOCAL defaults to the repo root inside # CMakeLists.txt (so libmoq builds from rs/libmoq); pass `path=/some/moq` to -# override. We deliberately don't compute the path here: `justfile_directory()` -# resolves to the root justfile when this runs as `just obs setup`, so the - -# relative math would point outside the repo. +# override it. setup preset="" path="": - #!/usr/bin/env bash - set -euo pipefail - PRESET=$(just preset "{{ preset }}") - if [ -n "{{ path }}" ]; then - echo "Configuring with preset: $PRESET and MOQ_LOCAL={{ path }}" - cmake --preset "$PRESET" -DMOQ_LOCAL="{{ path }}" - else - echo "Configuring with preset: $PRESET (MOQ_LOCAL defaults to the repo root)" - cmake --preset "$PRESET" - fi + cmake --preset "$(../../sh/obs/preset.sh '{{ preset }}')" {{ if path == "" { "" } else { "-DMOQ_LOCAL=" + quote(path) } }} # Build via CMake presets. Run `just obs setup` first (it configures + downloads - # deps); not chained here because on macOS setup reconfigures OBS, which is slow. build preset="": - #!/usr/bin/env bash - set -euo pipefail - PRESET=$(just preset "{{ preset }}") - cmake --build --preset "$PRESET" - -# Copy the freshly built plugin into the OBS user plugin dir and launch OBS. - -# Uses the installed OBS (no local OBS build required). macOS only for now. -run: - #!/usr/bin/env bash - set -euo pipefail - if [[ "$OSTYPE" != "darwin"* ]]; then - echo "just run is macOS-only for now; copy build_*/obs-moq.* into your OBS plugin dir manually" >&2 - exit 1 - fi - dest="$HOME/Library/Application Support/obs-studio/plugins" - mkdir -p "$dest" - cp -a build_macos/RelWithDebInfo/obs-moq.plugin "$dest/" - RUST_LOG=debug RUST_BACKTRACE=1 OBS_LOG_LEVEL=debug /Applications/OBS.app/Contents/MacOS/OBS + cmake --build --preset "$(../../sh/obs/preset.sh '{{ preset }}')" # Type-check every plugin source and unit test, without linking or an obs-deps -# download. -# -# This is the gate to run in a worktree. `just obs build` needs the -# multi-hundred-MB obs-deps bundle (macOS/Windows) and a full cargo build, per -# worktree; this needs neither, because the dev shell carries all three header -# sets: libobs pinned to the same OBS release buildspec.json downloads (see -# `obs-headers` in flake.nix), Qt6, and ffmpeg. CI links the real thing on Linux -# (.github/workflows/obs.yml). -# -# It compiles the Qt sources too, which the CMake build only does when -# ENABLE_QT and ENABLE_FRONTEND_API are on, so the definitions they gate on are -# set here to match. -# -# test/ is in scope because each test file defines the libmoq entry points the -# plugin calls, so a signature that drifts from the generated moq.h is a -# conflicting C declaration. Catching that needs only headers, which is why it -# belongs here: `just obs ci` finds the same drift, but only where obs.yml's -# path filter reaches, and `just obs test` is manual. +# download. The gate to run in a worktree. compile: - #!/usr/bin/env bash - set -euo pipefail - # Never skips, for the reason `test` doesn't: a compile gate that reports - # success without compiling is worse than no gate. Everything it needs is in - # the dev shell, so the fix is always to enter it. - cxx="${CXX:-c++}" - if ! command -v "$cxx" >/dev/null; then - echo "no C++ compiler at '$cxx'" >&2 - exit 1 - fi - for pkgs in "Qt6Widgets Qt6Gui Qt6Core" "libavcodec libavutil libswscale libswresample"; do - # shellcheck disable=SC2086 - if ! pkg-config --exists $pkgs; then - echo "missing headers for: $pkgs" >&2 - echo "run inside 'nix develop', which supplies Qt6 and ffmpeg on every platform" >&2 - exit 1 - fi - done - - # Assign first so a failure in `_includes` still aborts under `set -e`, - # then split on newlines only: an include path may contain spaces. - includes_raw=$(just _includes) - includes=() - while IFS= read -r flag; do includes+=("$flag"); done <<< "$includes_raw" - - qt=$(pkg-config --cflags Qt6Widgets Qt6Gui Qt6Core) - ffmpeg=$(pkg-config --cflags libavcodec libavutil libswscale libswresample) - - # MOQ_VERSION_STRING only reaches a label in the dock, so any value - # type-checks the same; CMake stamps the real libmoq version. - status=0 - for source in src/*.cpp test/*.cpp; do - # shellcheck disable=SC2086 - # -Wno-unused-command-line-argument: the nix compiler wrapper injects - # link flags, which a syntax-only run reports as unused, once per file. - # $qt and $ffmpeg stay unquoted on purpose: pkg-config hands back one - # space-separated string, and splitting it is the intended reading. - if ! "$cxx" -std=c++17 -fsyntax-only -Wno-unused-command-line-argument \ - "${includes[@]}" $qt $ffmpeg \ - -DMOQ_FRONTEND_ENABLED -DMOQ_VERSION_STRING='"0.0.0"' "$source"; then - status=1 - fi - done - exit $status + ../../sh/obs/compile.sh # Compile and link the plugin the way CI does, against the dev shell's own # libobs/Qt6/ffmpeg, then run the unit tests. Linux-only in practice: it's the -# one platform where nixpkgs has obs-studio, so nothing has to be downloaded. -# Manual elsewhere, like `just rs macos`. -# -# The tests run here without ThreadSanitizer because this is the only recipe CI -# invokes (.github/workflows/obs.yml). `just obs test` is manual, so on its own -# it can't make a build red; the sanitizer finds more, but only what runs in CI -# gates a merge. -# -# CMAKE_BUILD_TYPE overrides the preset's RelWithDebInfo because the Rust cache -# CI restores holds dev-profile artifacts. A release build of libmoq would -# rebuild its whole dependency tree on every run, and the gate is about whether -# the plugin compiles, which the profile doesn't change. +# one platform where nixpkgs has obs-studio. The Debug build type reuses the +# dev-profile Rust artifacts CI restores, where a release libmoq would rebuild +# its whole dependency tree. ci preset="": - #!/usr/bin/env bash - set -euo pipefail - PRESET=$(just preset "{{ preset }}" ci) - cmake --preset "$PRESET" -DCMAKE_BUILD_TYPE=Debug - cmake --build --preset "$PRESET" - just _unit - -# Print the -I flags for libobs and libmoq, shared by `compile` and `_unit`. -# -# moq.h is generated by cbindgen from rs/libmoq/src, so regenerate rather than -# just locating it: a stale header is how a plugin call to a since-changed -# libmoq function type-checks anyway. -[private] -_includes: - #!/usr/bin/env bash - set -euo pipefail - # The dev shell's pinned headers first, so every platform checks against the - # same libobs. Then the obs-deps framework (macOS/Windows), the OBS sources - # CMake unpacks beside it, and a system install (Linux without nix). - obs_include="" - for candidate in \ - "${OBS_INCLUDE_DIR:-}" \ - .deps/Frameworks/libobs.framework/Versions/A/Headers \ - .deps/obs-studio-*/libobs; do - if [ -n "$candidate" ] && [ -f "$candidate/obs.h" ]; then - obs_include="$candidate" - break - fi - done - if [ -z "$obs_include" ] && command -v pkg-config >/dev/null && pkg-config --exists libobs; then - obs_include=$(pkg-config --variable=includedir libobs)/obs - fi - if [ -z "$obs_include" ]; then - echo "libobs headers not found; run inside 'nix develop' (or set OBS_INCLUDE_DIR)" >&2 - exit 1 - fi - - # obs.h reaches as of OBS 32, which used to be vendored - # under libobs/util/ and so came along with the headers above. The dev shell - # carries it (see flake.nix); the obs-deps bundle ships its own copy beside - # libobs, which is how the macOS and Windows builds get it. - simde_include="" - if command -v pkg-config >/dev/null && pkg-config --exists simde; then - simde_include=$(pkg-config --variable=includedir simde) - else - for candidate in .deps/obs-deps-*/include; do - if [ -f "$candidate/simde/x86/sse2.h" ]; then - simde_include="$candidate" - break - fi - done - fi - if [ -z "$simde_include" ]; then - echo "simde headers not found; run inside 'nix develop', which supplies them" >&2 - exit 1 - fi - - (cd ../.. && cargo check --locked --quiet -p libmoq) - - # Ask cargo where its build script ran rather than reconstructing the path. - # CARGO_TARGET_DIR, a `--target` triple and a custom profile each move the - # generated header, and guessing wrong doesn't fail: it type-checks against - # whatever stale copy the default directory still holds. Replays the check - # above from cache, so it costs a process rather than a compile. - out_dir=$(cd ../.. && cargo check --locked -p libmoq --message-format=json | - jq -r 'select(.reason == "build-script-executed") | select(.package_id | tostring | test("libmoq")) | .out_dir' | - tail -1) - if [ -z "$out_dir" ]; then - echo "cargo reported no build script output for libmoq" >&2 - exit 1 - fi - # //build/libmoq-/out, and the header sits beside the - # profile directory. rs/libmoq/build.rs derives it from OUT_DIR the same way. - moq_include="$(dirname "$(dirname "$(dirname "$(dirname "$out_dir")")")")/include" - if [ ! -f "$moq_include/moq.h" ]; then - echo "$moq_include/moq.h missing after 'cargo check --locked -p libmoq'" >&2 - exit 1 - fi - - # One flag per line, path glued to the -I, so a caller can rebuild the - # arguments without splitting on the spaces in a path. - printf -- '-I%s\n' src "$obs_include" "$simde_include" "$moq_include" + cmake --preset "$(../../sh/obs/preset.sh '{{ preset }}' ci)" -DCMAKE_BUILD_TYPE=Debug + cmake --build --preset "$(../../sh/obs/preset.sh '{{ preset }}' ci)" + ../../sh/obs/unit.sh # Unit-test the plugin sources against stubbed libobs/libmoq/ffmpeg, under -# ThreadSanitizer. -# -# Manual, like `just rs macos` and `just rs windows`, because ThreadSanitizer -# needs its own build. `just obs ci` runs the same tests without it, so a -# regression these assertions catch still turns PR CI red; the sanitizer is what -# adds the races on top. Run this whenever you touch src/, especially the session -# callback plumbing, whose orderings the build can't check. +# ThreadSanitizer. Manual; `just obs ci` runs them without the sanitizer. test: - #!/usr/bin/env bash - set -euo pipefail - # Unlike the lint recipes, this one never skips: a test gate that reports - # success without running anything is worse than no gate. - cxx="${CXX:-c++}" - unsupported="'$cxx' cannot build and run -fsanitize=thread binaries. These tests need a - Clang or GCC whose ThreadSanitizer runtime works on this host: set CXX, or on Windows run - them from WSL, since neither MSVC nor Clang on Windows implements ThreadSanitizer. - They also run without the sanitizer as part of 'just obs ci'." - if ! command -v "$cxx" >/dev/null; then - echo "no C++ compiler at '$cxx'." >&2 - echo "$unsupported" >&2 - exit 1 - fi - - probe=$(mktemp -d) - trap 'rm -rf "$probe"' EXIT - - # Run the probe, don't just link it: a mismatch between the compiler's TSan - # runtime and the host kernel (macOS is where this shows up) links fine and - # then segfaults before main, which is an unreadable way for the real test - # to fail. - # The subshell keeps bash's own "Segmentation fault" job message out of the - # output, so the explanation below is all the reader gets. - if ! printf 'int main(){}\n' | "$cxx" -x c++ -fsanitize=thread -o "$probe/probe" - >/dev/null 2>&1 \ - || ! ("$probe/probe" >/dev/null 2>&1) 2>/dev/null; then - echo "$unsupported" >&2 - exit 1 - fi - - just _unit tsan + ../../sh/obs/test.sh -# Compile and run the unit tests, plain or under ThreadSanitizer. Shared by -# `test` and `ci` so both run the same assertions. -[private] -_unit mode="plain": - #!/usr/bin/env bash - set -euo pipefail - cxx="${CXX:-c++}" - if ! command -v "$cxx" >/dev/null; then - echo "no C++ compiler at '$cxx'" >&2 - exit 1 - fi - # moq-source.cpp decodes with ffmpeg, so its test needs the headers even - # though it stubs every function out of them. - if ! pkg-config --exists libavcodec libavutil libswscale; then - echo "missing ffmpeg headers" >&2 - echo "run inside 'nix develop', which supplies ffmpeg on every platform" >&2 - exit 1 - fi - - includes_raw=$(just _includes) - includes=() - while IFS= read -r flag; do includes+=("$flag"); done <<< "$includes_raw" - - ffmpeg=$(pkg-config --cflags libavcodec libavutil libswscale) - - flags=(-std=c++17 -g -O0 -pthread) - if [ "{{ mode }}" = "tsan" ]; then - flags+=(-fsanitize=thread) - fi - - out=$(mktemp -d) - trap 'rm -rf "$out"' EXIT - - # One binary per source under test: each test file defines its own libobs and - # libmoq stubs, so two of them can't share a link. Header-only helpers - # (quality defaults) build as a third binary with no plugin source. - for name in moq-output moq-source; do - # shellcheck disable=SC2086 - # $ffmpeg stays unquoted on purpose: pkg-config hands back one - # space-separated string, and splitting it is the intended reading. - "$cxx" "${flags[@]}" "${includes[@]}" $ffmpeg \ - -o "$out/$name-test" "test/$name-test.cpp" "src/$name.cpp" - TSAN_OPTIONS="halt_on_error=1" "$out/$name-test" - done - for name in moq-quality-defaults moq-dock-stop moq-error moq-encoder-latency moq-dial moq-spark; do - "$cxx" "${flags[@]}" "${includes[@]}" \ - -o "$out/$name-test" "test/$name-test.cpp" - TSAN_OPTIONS="halt_on_error=1" "$out/$name-test" - done - -# Lint formatting. Skips a tool silently if it isn't on $PATH (matches repo convention). +# Lint formatting and the OBS version pins. check: - #!/usr/bin/env bash - set -euo pipefail - if grep -Fq '"MOQ_VERSION"' CMakePresets.json; then - echo "CMakePresets.json must not pin MOQ_VERSION" >&2 - exit 1 - fi - # The headers `just obs compile` type-checks against are a second copy of - # the OBS version this spec downloads, so a bump here has to reach flake.nix - # or the two gates check different libobs. - spec_obs=$(jq -r '.dependencies["obs-studio"].version // ""' buildspec.json) - flake_obs=$(sed -n '/pname = "libobs-headers"/,/}/p' ../../flake.nix | sed -n 's/.*version = "\([^"]*\)".*/\1/p') - # Both are pattern matches against files this recipe doesn't own the shape - # of. An empty parse compares equal to an empty parse, so the guard would - # pass by reporting nothing rather than by finding the versions in sync. - if [ -z "$spec_obs" ] || [ -z "$flake_obs" ]; then - echo "couldn't read the obs-studio version: '$spec_obs' from buildspec.json, '$flake_obs' from flake.nix" >&2 - exit 1 - fi - if [ "$spec_obs" != "$flake_obs" ]; then - echo "obs-studio is $spec_obs in buildspec.json but $flake_obs in flake.nix (libobs-headers)" >&2 - exit 1 - fi - # And a third: `just obs ci` links against nixpkgs' obs-studio, the one OBS - # this repo doesn't pick. Without this the released macOS/Windows binaries - # could be a whole libobs API apart from what CI ever compiles. Patch - # releases carry no API change, so only the major.minor has to agree; that - # also keeps the guard quiet until flake.lock actually moves OBS, which is - # the change that opens the gap and so the one that should fail. - # OBS_LINKED_VERSION comes from the dev shell (see flake.nix), so this leg - # is skipped outside it, where there is no nixpkgs obs-studio to compare. - # MOQ_STRICT turns that skip into an error for the same reason it turns a - # missing tool into one: in CI a leg that checks nothing still reports green. - if [ -z "${OBS_LINKED_VERSION:-}" ]; then - if [ -n "${MOQ_STRICT:-}" ]; then - echo "MOQ_STRICT is set but OBS_LINKED_VERSION is unset; run inside 'nix develop', which exports it" >&2 - exit 1 - fi - else - spec_api=$(echo "$spec_obs" | cut -d. -f1,2) - linked_api=$(echo "$OBS_LINKED_VERSION" | cut -d. -f1,2) - if [ "$spec_api" != "$linked_api" ]; then - echo "obs-studio is $spec_obs in buildspec.json but $OBS_LINKED_VERSION in nixpkgs, which is what 'just obs ci' links" >&2 - echo "bump buildspec.json and flake.nix's libobs-headers to $OBS_LINKED_VERSION, or pin nixpkgs back" >&2 - exit 1 - fi - fi - if command -v cmake >/dev/null; then - build_dir=$(mktemp -d) - trap 'rm -rf "$build_dir"' EXIT - if cmake -S . -B "$build_dir" -DMOQ_LOCAL= >"$build_dir/configure.log" 2>&1; then - echo "CMake accepted a release build without MOQ_VERSION" >&2 - exit 1 - fi - if ! grep -Fq "MOQ_VERSION is required when MOQ_LOCAL does not contain rs/libmoq" "$build_dir/configure.log"; then - cat "$build_dir/configure.log" >&2 - exit 1 - fi - fi - if command -v clang-format >/dev/null; then - git ls-files 'src/*.cpp' 'src/*.h' 'test/*.cpp' | xargs clang-format --dry-run --Werror - fi - if command -v gersemi >/dev/null; then - gersemi --check CMakeLists.txt cmake - fi + ../../sh/obs/check.sh # Auto-fix formatting. fix: - #!/usr/bin/env bash - set -euo pipefail - if command -v clang-format >/dev/null; then - git ls-files 'src/*.cpp' 'src/*.h' 'test/*.cpp' | xargs clang-format -i - fi - if command -v gersemi >/dev/null; then - gersemi --in-place CMakeLists.txt cmake - fi - -# Detect the CMake preset for the current platform (or use the override). -# Pass `ci` as the second argument for the warnings-as-errors variant, whose -# name upstream spells differently on each platform. -preset override="" variant="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "{{ override }}" ]]; then - echo "{{ override }}" - elif [[ "$OSTYPE" == "darwin"* ]]; then - [[ "{{ variant }}" == "ci" ]] && echo "macos-ci" || echo "macos" - elif [[ "$OSTYPE" == "linux-gnu"* ]]; then - [[ "{{ variant }}" == "ci" ]] && echo "ubuntu-ci-x86_64" || echo "ubuntu-x86_64" - elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then - [[ "{{ variant }}" == "ci" ]] && echo "windows-ci-x64" || echo "windows-x64" - else - echo "Unknown platform: $OSTYPE" >&2 - exit 1 - fi + ../../sh/obs/fix.sh diff --git a/dart/justfile b/dart/justfile index 9823e8cf32..4387f113c8 100644 --- a/dart/justfile +++ b/dart/justfile @@ -1,18 +1,13 @@ #!/usr/bin/env just --justfile # Check the generated bindings and both Dart packages. -check $FILES="": - ./scripts/check.sh "{{ FILES }}" +check: + ../sh/dart/check.sh # Format Dart sources. -fix $FILES="": - ./scripts/check.sh --fix "{{ FILES }}" +fix: + ../sh/dart/check.sh --fix # Regenerate the raw UniFFI Dart bindings. generate: - ./scripts/generate.sh - -# Remove Dart tool state and build output. -clean: - rm -rf moq/.dart_tool moq_ffi/.dart_tool - rm -rf moq/build moq_ffi/build + ../sh/dart/generate.sh diff --git a/dart/moq_ffi/hook/build.dart b/dart/moq_ffi/hook/build.dart index 6916ea46a7..bfb465d603 100644 --- a/dart/moq_ffi/hook/build.dart +++ b/dart/moq_ffi/hook/build.dart @@ -7,7 +7,7 @@ import 'package:http/http.dart' as http; /// The moq-ffi release whose native assets this package downloads. /// -/// dart/scripts/package.sh rewrites this when staging a release. The sentinel +/// sh/dart/package.sh rewrites this when staging a release. The sentinel /// is never used to download: a checkout builds from source instead, and a /// package built without the injection fails loudly rather than fetching a /// library that does not match these bindings. diff --git a/dart/moq_ffi/pubspec.yaml b/dart/moq_ffi/pubspec.yaml index cf7a8ecadc..0f53a4f24c 100644 --- a/dart/moq_ffi/pubspec.yaml +++ b/dart/moq_ffi/pubspec.yaml @@ -1,6 +1,6 @@ name: moq_ffi description: Raw Dart and Flutter bindings for Media over QUIC. -# Overridden by dart/scripts/package.sh from the moq-ffi-v* tag. Committing a +# Overridden by sh/dart/package.sh from the moq-ffi-v* tag. Committing a # real version would go stale the moment release-plz bumps the crate. version: 0.0.0-dev repository: https://github.com/moq-dev/moq/tree/main/dart/moq_ffi diff --git a/demo/justfile b/demo/justfile index bfd725f82d..b612873465 100644 --- a/demo/justfile +++ b/demo/justfile @@ -1,11 +1,5 @@ set fallback -mod boy -mod pub -mod relay -mod sub -mod web - # Run the web demo (default). # # Picks the first free port at/after 4443 (both UDP for QUIC and TCP for HTTP) diff --git a/doc/setup/dev.md b/doc/setup/dev.md index 9aaf57bad6..0eb3852d7f 100644 --- a/doc/setup/dev.md +++ b/doc/setup/dev.md @@ -63,6 +63,11 @@ just check just test ``` +These diff the branch against its upstream (or `origin/main`) and run only the +modules the diff reaches. The map from paths to modules lives in +`sh/dispatch.sh`; every recipe is one line, and any logic lives in a script +under `sh/`. + See [CONTRIBUTING.md](https://github.com/moq-dev/moq/blob/main/CONTRIBUTING.md) for branch targeting, commit messages, and reviews, and [Agent setup](/setup/agent) if an AI coding agent is doing the work. diff --git a/drafts/justfile b/drafts/justfile index 8ca61beadc..bdb39c6e4d 100644 --- a/drafts/justfile +++ b/drafts/justfile @@ -5,87 +5,28 @@ # # The toolchain (kramdown-rfc, xml2rfc, mmark) comes from the nix dev shell # (see `draftsDeps` in flake.nix), so `nix develop --command just drafts build` -# renders with the same tool versions everywhere. There is no submodule or -# venv to bootstrap: `kramdown-rfc` turns the markdown into RFC XML, `xml2rfc` -# renders the txt/html. +# renders with the same tool versions everywhere. `kramdown-rfc` turns the +# markdown into RFC XML, `xml2rfc` renders the txt/html. set working-directory := '.' -# Common xml2rfc flags. --allow-local-file-access lets it read the CSS/refs. -xml2rfc := "xml2rfc -q --allow-local-file-access" - # List the draft names (the argument to `build` / `publish`). default: - @for f in draft-*.md; do echo "${f%.md}"; done + @ls draft-*.md | sed 's/\.md$//' -# Render one draft to .txt and .html (local editor's copy, -# gitignored). Renders the -latest docname as-is; use `publish` for a version. +# Render one draft to .txt and .html (local editor's copy, gitignored). build name: - kramdown-rfc --v3 < {{ name }}.md > {{ name }}.xml - {{ xml2rfc }} --text {{ name }}.xml -o {{ name }}.txt - {{ xml2rfc }} --html {{ name }}.xml -o {{ name }}.html - @echo "built {{ name }}.txt and {{ name }}.html" + ../sh/drafts/build.sh {{ name }} # Render every draft. all: - @for f in draft-*.md; do just build "${f%.md}"; done + ../sh/drafts/build.sh # Validate that every draft parses and that its generated XML passes validation. -# -# Both halves are needed. kramdown-rfc catches markdown and frontmatter errors, -# but happily emits XML for a cross-reference that points at no such section; -# xml2rfc is what rejects the dangling IDREF. Checking only the first half lets -# a broken `[text](#anchor)` pass here and then fail at `publish` time, which is -# the worst place to find it. -# -# --preptool stops after validation instead of rendering txt/html, and -N keeps -# it off the network (kramdown-rfc already inlined the references), which is -# what makes this a couple of seconds per draft rather than a minute. check: - bun install --frozen-lockfile --cwd .. - @for f in draft-*.md; do \ - echo "checking $f"; \ - kramdown-rfc --v3 < "$f" > "${f%.md}.xml" || exit 1; \ - {{ xml2rfc }} --preptool -N "${f%.md}.xml" -o "${f%.md}.prepped.xml" \ - || { rm -f "${f%.md}.prepped.xml"; exit 1; }; \ - rm -f "${f%.md}.prepped.xml"; \ - done - @for f in moq-e2ee-*.ts; do \ - bun tsc --noEmit --skipLibCheck --target esnext --module esnext --moduleResolution bundler --types bun "$f" || exit 1; \ - bun "$f" || exit 1; \ - done - -# Remove generated files and the reference cache. -clean: - rm -f draft-*.xml draft-*.txt draft-*.html - rm -rf .refcache + ../sh/drafts/check.sh -# Submit a new draft version to the IETF datatracker. The datatracker emails -# the submitter a confirmation link; the submission is not final until that -# link is clicked. For a brand-new draft (-00) set "Replaces" on the -# confirmation page. -# Example: just drafts publish draft-lcurley-moq-lite 05 you@example.com +# Submit a new draft version to the IETF datatracker, e.g. +# `just drafts publish draft-lcurley-moq-lite 05 you@example.com`. publish name version email: - #!/usr/bin/env bash - set -euo pipefail - case "{{ version }}" in - [0-9][0-9]) ;; - *) echo "version must be two digits, e.g. 05" >&2; exit 1 ;; - esac - doc="{{ name }}-{{ version }}" - if [ ! -f "{{ name }}.md" ]; then - echo "no such draft: {{ name }}.md" >&2; exit 1 - fi - echo "Building $doc.xml" - sed "s/{{ name }}-latest/$doc/g" "{{ name }}.md" | kramdown-rfc --v3 > "$doc.xml" - echo "Submitting $doc.xml to the datatracker as {{ email }}" - resp="$(mktemp)" - code="$(curl -sS -o "$resp" -w '%{http_code}' \ - -F "user={{ email }}" -F "xml=@$doc.xml" \ - https://datatracker.ietf.org/api/submission)" - echo "HTTP $code" - cat "$resp"; echo; rm -f "$resp" - case "$code" in - 200|201) echo "Submitted. Check {{ email }} for the confirmation link." ;; - *) echo "Submission failed." >&2; exit 1 ;; - esac + ../sh/drafts/publish.sh {{ name }} {{ version }} {{ email }} diff --git a/flake.nix b/flake.nix index 48971bef39..7e55731225 100644 --- a/flake.nix +++ b/flake.nix @@ -121,7 +121,7 @@ # unification would otherwise hide a broken single-crate build. cargo-hack cargo-nextest - # Browser/WASM bindings (rs/moq-wasm -> @moq/wasm via `just wasm`). + # Browser/WASM bindings (rs/moq-wasm -> @moq/wasm via `just js wasm`). # wasm-bindgen-cli must match the `wasm-bindgen` crate version (the # crate is pinned to nixpkgs' CLI version); bump both together. wasm-bindgen-cli @@ -231,10 +231,9 @@ ]; # Linters / formatters used by `just check` and `just fix`, which - # guard each tool with `command -v` so they skip silently when the - # binary isn't on $PATH. CI sets MOQ_STRICT=1, which turns that skip - # into an error (see `_tools` in the root justfile), so this list and - # that one have to stay in step. + # skip a module whose tools aren't on $PATH. CI sets MOQ_STRICT=1, + # which turns that skip into an error (see the tools map in + # sh/dispatch.sh), so this list and that one have to stay in step. lintDeps = with pkgs; [ shellcheck shfmt @@ -263,8 +262,8 @@ # dependency in rs/moq-ffi/Cargo.toml. Five other places name the same # generator version and must be bumped together: the repo and revision # in release-go-ffi.yml, and the `cargo install` line in - # rs/moq-ffi/build.sh, go/ffi/README.md, go/scripts/check.sh, and - # go/scripts/stage.sh. + # rs/moq-ffi/build.sh, go/ffi/README.md, sh/go/check.sh, and + # sh/go/stage.sh. # # This points at a fork rather than NordSecurity because upstream has no # uniffi 0.32 generator: the metadata encoding changed in 0.32 even diff --git a/go/.gitignore b/go/.gitignore index 9d1c2961a2..4e2b41f06e 100644 --- a/go/.gitignore +++ b/go/.gitignore @@ -1,4 +1,4 @@ -# Generated by scripts/{check,package-ffi}.sh; the main repo stays binary-free. +# Generated by sh/go/{check,package-ffi}.sh; the main repo stays binary-free. ffi/moq/moq.go ffi/moq/moq.h ffi/moq/lib/ diff --git a/go/ffi/README.md b/go/ffi/README.md index 23aae8b025..c45b1e7026 100644 --- a/go/ffi/README.md +++ b/go/ffi/README.md @@ -20,9 +20,9 @@ The published module ships prebuilt `libmoq_ffi.a` for `linux/amd64`, `linux/arm ## Local development -`go/scripts/stage.sh` builds `moq-ffi` for the host, runs `uniffi-bindgen-go` to regenerate `moq.go`, and stages this module plus the wrapper into `dist/` with the wrapper wired to the local ffi by a `replace`. It prints the two staged module directories, so anything that needs to build Go against this checkout (`go/scripts/check.sh`, `test/interop`) shares one staging path. +`sh/go/stage.sh` builds `moq-ffi` for the host, runs `uniffi-bindgen-go` to regenerate `moq.go`, and stages this module plus the wrapper into `dist/` with the wrapper wired to the local ffi by a `replace`. It prints the two staged module directories, so anything that needs to build Go against this checkout (`sh/go/check.sh`, `test/interop`) shares one staging path. -`go/scripts/check.sh` stages that way and then runs `go build`/`go vet`/`go test`. It also runs `publish-wrapper.test.sh`, which drives the wrapper publisher against a scratch bare repo (no cargo, no network). Run via `just go check`. Skips cleanly without `cargo`, `go`, or `uniffi-bindgen-go`. +`sh/go/check.sh` stages that way and then runs `go build`/`go vet`/`go test`. Run via `just go check`. Skips cleanly without `cargo`, `go`, or `uniffi-bindgen-go`. The dev shell provides both `go` and `uniffi-bindgen-go`, so `nix develop --command just go check` needs no setup. Without Nix, install `uniffi-bindgen-go` once: @@ -49,8 +49,8 @@ go/ffi/ lib/_/libmoq_ffi.a (gitignored, staged at release time) ``` -Compiled binaries never live in the source tree. `scripts/check.sh` stages into the gitignored `dist/` working dir under the repo root; CI does the equivalent assembly into the [moq-dev/moq-go-ffi](https://github.com/moq-dev/moq-go-ffi) mirror. +Compiled binaries never live in the source tree. `sh/go/check.sh` stages into the gitignored `dist/` working dir under the repo root; CI does the equivalent assembly into the [moq-dev/moq-go-ffi](https://github.com/moq-dev/moq-go-ffi) mirror. ## Release -The `release-go-ffi.yml` workflow fires on every `moq-ffi-v*` tag, builds per-target static libraries, runs `uniffi-bindgen-go`, calls `go/scripts/package-ffi.sh` to assemble the module, and `go/scripts/publish-ffi.sh` to push the result to `moq-dev/moq-go-ffi` with a bare-semver tag (e.g. `v0.2.18`) lockstep with the crate. Go's module proxy picks up the new tag automatically. +The `release-go-ffi.yml` workflow fires on every `moq-ffi-v*` tag, builds per-target static libraries, runs `uniffi-bindgen-go`, calls `sh/go/package-ffi.sh` to assemble the module, and `sh/go/publish-ffi.sh` to push the result to `moq-dev/moq-go-ffi` with a bare-semver tag (e.g. `v0.2.18`) lockstep with the crate. Go's module proxy picks up the new tag automatically. diff --git a/go/ffi/moq/cgo.go b/go/ffi/moq/cgo.go index 8be6f790a6..50dc423b50 100644 --- a/go/ffi/moq/cgo.go +++ b/go/ffi/moq/cgo.go @@ -3,12 +3,12 @@ // moq.dev/moq wrapper, which builds on this package. // // The exported API is generated from rs/moq-ffi via uniffi-bindgen-go and -// dropped in alongside this file (moq.go) by scripts/package-ffi.sh; the in-tree -// source therefore does not build on its own. Run scripts/check.sh to stage a +// dropped in alongside this file (moq.go) by sh/go/package-ffi.sh; the in-tree +// source therefore does not build on its own. Run sh/go/check.sh to stage a // complete copy into dist/ and exercise it. // // The per-platform static archive is loaded from moq/lib/_/ -// inside the staged module, populated by scripts/package-ffi.sh from the release +// inside the staged module, populated by sh/go/package-ffi.sh from the release // build matrix. package moq diff --git a/go/justfile b/go/justfile index d842e27d10..ad26a9875d 100644 --- a/go/justfile +++ b/go/justfile @@ -10,36 +10,18 @@ default: # Stages both the ffi and wrapper modules into dist/ (wiring the wrapper to the # local ffi via replace). Skips cleanly if cargo, go, or uniffi-bindgen-go is -# missing. -# -# This runs the tests too, so there is no separate `just go test`. Takes an -# optional newline-separated list of changed files and skips when none are -# Go-scoped; `just go check` (no FILES) always runs. +# missing. This runs the tests too, so there is no separate `just go test`. # Build moq-ffi, run uniffi-bindgen-go, then `go build`/`go vet`/`go test`. -check $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '^(go/|rs/moq-ffi/)' <<< "$FILES"; then - echo "go: no Go changes; skipping." - exit 0 - fi - bash scripts/check.sh +check: + bash ../sh/go/check.sh # Stage the in-tree go/ffi source + per-target moq-ffi libs + generated # bindings into the moq.dev/moq-ffi module ready for publish. package-ffi *args: - bash scripts/package-ffi.sh {{ args }} + bash ../sh/go/package-ffi.sh {{ args }} # Stage the in-tree go/wrapper source into the moq.dev/moq module, rewriting its # moq.dev/moq-ffi require to the target ffi version, ready for publish. package-wrapper *args: - bash scripts/package-wrapper.sh {{ args }} - -# The paths are the gitignored ones in go/.gitignore. `just go check` stages -# into the workspace dist/ instead, which `just js clean` sweeps, so these only -# show up when uniffi-bindgen-go or `go mod` is pointed at the tree directly. - -# Remove the generated bindings, native libs, and vendoring from go/. -clean: - rm -rf ffi/moq/moq.go ffi/moq/moq.h ffi/moq/lib ffi/go.sum ffi/vendor wrapper/go.sum wrapper/vendor + bash ../sh/go/package-wrapper.sh {{ args }} diff --git a/go/wrapper/README.md b/go/wrapper/README.md index 52ed3ce69d..9a69365d69 100644 --- a/go/wrapper/README.md +++ b/go/wrapper/README.md @@ -117,4 +117,4 @@ The committed `go.mod` carries a `require moq.dev/moq-ffi v0.0.0` **placeholder* ## Local development -Run `just go check`: it builds `moq-ffi` for the host, regenerates the bindings, stages both modules into `dist/` with a `replace` wiring the wrapper to the local ffi, and runs `go build`/`go vet`/`go test`. It also runs `scripts/publish-wrapper.test.sh`, which exercises the publisher's release/no-op/recovery paths against a scratch bare repo standing in for the mirror. See [../ffi/README.md](../ffi/README.md) for the `uniffi-bindgen-go` install. +Run `just go check`: it builds `moq-ffi` for the host, regenerates the bindings, stages both modules into `dist/` with a `replace` wiring the wrapper to the local ffi, and runs `go build`/`go vet`/`go test`. See [../ffi/README.md](../ffi/README.md) for the `uniffi-bindgen-go` install. diff --git a/go/wrapper/go.mod b/go/wrapper/go.mod index e306de284d..eba975c62c 100644 --- a/go/wrapper/go.mod +++ b/go/wrapper/go.mod @@ -3,7 +3,7 @@ module moq.dev/moq go 1.23 // The require below is a placeholder. The real version is rewritten at release -// time by go/scripts/package-wrapper.sh to the latest published moq-ffi, and +// time by sh/go/package-wrapper.sh to the latest published moq-ffi, and // `just go check` injects a local `replace` to the freshly-generated bindings. // Do not "fix" this by hand or add a replace directive to the committed file. require moq.dev/moq-ffi v0.0.0 diff --git a/js/common/workers.test.ts b/js/common/workers.test.ts deleted file mode 100644 index 5b11c5e1fb..0000000000 --- a/js/common/workers.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { expect, test } from "bun:test"; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; - -test("worker checks select changed projects and propagate build failures", () => { - const temp = mkdtempSync(join(tmpdir(), "moq-workers-")); - const log = join(temp, "calls"); - writeFileSync( - join(temp, "bun"), - '#!/bin/sh\nprintf "%s %s\\n" "$PWD" "$*" >> "$WORKER_TEST_LOG"\nif [ "$*" = "run deploy --dry-run" ]; then exit "$WORKER_TEST_EXIT"; fi\n', - { mode: 0o755 }, - ); - try { - for (const project of ["infra/apt", "infra/rpm", "demo/pub"]) { - writeFileSync(log, ""); - const result = Bun.spawnSync(["just", "js", "check", `${project}/bun.lock`], { - cwd: resolve(import.meta.dir, "../.."), - env: { - ...process.env, - PATH: `${temp}:${process.env.PATH}`, - WORKER_TEST_LOG: log, - WORKER_TEST_EXIT: "0", - }, - }); - expect(result.exitCode).toBe(0); - expect(readFileSync(log, "utf8").trim().split("\n")).toEqual([ - `${resolve(import.meta.dir, "../..", project)} install --frozen-lockfile`, - `${resolve(import.meta.dir, "../..", project)} run deploy --dry-run`, - ]); - } - const failed = Bun.spawnSync(["just", "js", "check", "infra/apt/bun.lock"], { - cwd: resolve(import.meta.dir, "../.."), - env: { - ...process.env, - PATH: `${temp}:${process.env.PATH}`, - WORKER_TEST_LOG: log, - WORKER_TEST_EXIT: "1", - }, - }); - expect(failed.exitCode).not.toBe(0); - } finally { - rmSync(temp, { recursive: true, force: true }); - } -}); diff --git a/js/justfile b/js/justfile index bcad24af59..3cc52a9e22 100644 --- a/js/justfile +++ b/js/justfile @@ -6,108 +6,48 @@ set working-directory := '.' -# Paths a JS check depends on: every bun workspace (they are not all under js/, -# see the root package.json) plus the config shared by all of them. `check`, -# `fix`, and `ci` all skip themselves when a diff misses this. -scope := '^(js/|doc/|drafts/|demo/(boy|web)/|test/interop/clients/js|test/wasm/|package\.json$|bun\.lock(b)?$|biome\.jsonc$)' - # Run all checks by default. default: just check -install: - bun install - -# Takes an optional newline-separated list of changed files and skips if none -# match the JS scope; `just js check` (no FILES) always runs. -# -# FILES is an exported parameter, not `{{ FILES }}`: just interpolates the -# latter into the recipe source, where a filename like `$(...)` would run as a -# command. Git-derived paths are untrusted input, so they stay data. - # `build` runs here rather than as its own gate: the per-package `check` script # is `tsc --noEmit`, which never runs declaration emit, so the errors that only # surface when emitting a `.d.ts` (TS4023 and friends) pass `check` and then # break a JSR/npm publish. It costs about 20s for every package. - +# # `common/deps.ts` runs first and cheaply: bundlers resolve an undeclared # workspace import through hoisting, so nothing else in `check` notices one. +# doc/ is in scope, so the binding pages' stats fields are checked here too. # Lint, type-check, biome-format, and build all packages. -check $FILES="": - #!/usr/bin/env bash - set -euo pipefail - # These Workers have independent lockfiles outside the Bun workspace. - # Validate the deploy bundle without publishing or requiring credentials. - for project in infra/apt infra/rpm demo/pub; do - if [[ -z "$FILES" ]] || grep -qE "^($project/|js/justfile$)" <<< "$FILES"; then - ( - cd "../$project" - bun install --frozen-lockfile - bun run deploy --dry-run - ) - fi - done - if [[ -n "$FILES" ]] && ! grep -qE '{{ scope }}' <<< "$FILES"; then - echo "js: no JS changes; skipping." - exit 0 - fi - # doc/ is in this scope, so the binding pages are checked here. - python3 ../rs/scripts/stats-docs.py +check: + python3 ../sh/rs/stats-docs.py bun install --frozen-lockfile bun common/deps.ts - if tty -s; then - bun run --filter='*' --elide-lines=0 check - else - bun run --filter='*' check - fi + bun run --filter='*' --elide-lines=0 check bun biome check just build -# Auto-fix lint and formatting issues. Skips on the same terms as `check`. -fix $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '{{ scope }}' <<< "$FILES"; then - echo "js: no JS changes; skipping." - exit 0 - fi +# Validate the Worker deploy bundles outside the Bun workspace. +workers: + ../sh/js/workers.sh + +# Auto-fix lint and formatting issues. +fix: bun install bun biome check --write -# Run all unit tests. Skips on the same terms as `check`. -test $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '{{ scope }}' <<< "$FILES"; then - echo "js: no JS changes; skipping tests." - exit 0 - fi +# Run all unit tests. +test: bun install --frozen-lockfile - bun test common/deps.test.ts common/workers.test.ts - if tty -s; then - bun run --filter='*' --elide-lines=0 test - else - bun run --filter='*' test - fi + bun test common/deps.test.ts + bun run --filter='*' --elide-lines=0 test # Build all packages. build: bun run --filter='*' build -# Remove node_modules and build output. The bun workspace spans the repo - -# (deps hoist to the root, not into js/), so sweep from the workspace root. -clean: - #!/usr/bin/env bash - set -euo pipefail - cd .. - find . -name .claude -prune -o \ - -type d \( -name node_modules -o -name dist -o -name out -o -name pkg \) \ - -prune -exec rm -rf {} + - find . -name .claude -prune -o -type f -name '*.tsbuildinfo' -exec rm -f {} + - -# Pull in new versions and report what's outdated. -update: - bun update - bun outdated +# Build browser/WASM bindings into @moq/wasm using the pinned wasm-bindgen toolchain. +wasm: + cargo build --locked -p moq-wasm --target wasm32-unknown-unknown --profile wasm-release + wasm-bindgen --target web --out-name moq --out-dir wasm/dist "${CARGO_TARGET_DIR:-../target}/wasm32-unknown-unknown/wasm-release/moq_wasm.wasm" diff --git a/js/wasm/.gitignore b/js/wasm/.gitignore index d3d540b01f..b7322370ae 100644 --- a/js/wasm/.gitignore +++ b/js/wasm/.gitignore @@ -1,3 +1,3 @@ -# Generated by `just wasm` (wasm-bindgen). The .wasm binary and JS glue are +# Generated by `just js wasm` (wasm-bindgen). The .wasm binary and JS glue are # build artifacts, not source: build them, never commit them. /dist/ diff --git a/js/wasm/README.md b/js/wasm/README.md index b836032e6c..bd554effd7 100644 --- a/js/wasm/README.md +++ b/js/wasm/README.md @@ -29,7 +29,7 @@ The classes (`Moq.Session`, `Moq.Broadcast`, `Moq.Track`, `Moq.Group`) drop the `dist/` is generated, not committed. Build it from the repo root: ```bash -just wasm +just js wasm ``` That compiles `rs/moq-wasm` for `wasm32-unknown-unknown`, runs `wasm-bindgen` diff --git a/justfile b/justfile index a48833cafe..4da8f933ac 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,8 @@ #!/usr/bin/env just --justfile # Using Just: https://github.com/casey/just?tab=readme-ov-file#installation +# +# Every recipe is a menu entry: one line that runs a tool, or a script under +# sh/. Logic belongs in the script. set unstable @@ -29,10 +32,6 @@ mod relay 'demo/relay' mod sub 'demo/sub' mod web 'demo/web' -# Byte budget for a changed-file list, sized so it survives being passed as a -# single argv/env string on every hop of the dispatch. See `_changed`. -changed_max := '65536' - # Run the demo by default. default: just demo @@ -43,826 +42,39 @@ dev: # Benchmark the current tree, or compare it with a commit: `just bench origin/main`. bench $BASE="": - #!/usr/bin/env bash - exec just --justfile bench/justfile compare "$BASE" + bench/run.sh "$BASE" # Compare one multi-threaded Tokio runtime with the same number of independent -# Tokio/epoll and io_uring workers. Defaults to every logical CPU. -bench-runtime $ROUNDS="3" $WORKERS="": - #!/usr/bin/env bash - exec just --justfile bench/justfile runtime "$ROUNDS" "$WORKERS" - -# A linked worktree's Git metadata does not live under its own root: the -# per-worktree directory is `--git-dir` and everything shared (objects, remote -# refs, the branch namespace) is under `--git-common-dir`, which for an agent -# checkout is inside the main repository. Write access to the source tree -# therefore says nothing about whether this checkout can fetch, branch, or -# rebase; the answer is a property of those two directories, and finding out by -# running `git fetch` and reading the error is the slow way. -# -# `check` reports; `setup` fetches, points the branch at its base, and records -# the SHA it fetched under the per-worktree Git directory, where `check` reads it -# back to say how stale the recorded base has become. Neither ever resets, -# rebases, cleans, or checks anything out: a dirty tree is someone's work in -# progress, and adopting a checkout must not be able to destroy it. -# -# BASE follows the same rule as the rest of the repo (see `_base`): `main` -# unless the branch's upstream says otherwise. - -# Report a worktree's base, Git metadata access, and state; `setup` also fetches. -worktree ACTION="check" $BASE="": - #!/usr/bin/env bash - set -euo pipefail - - case "{{ ACTION }}" in - check | setup) ;; - *) - echo "usage: just worktree [check|setup] [BASE]" >&2 - exit 2 - ;; - esac - - root=$(git rev-parse --show-toplevel) - git_dir=$(cd "$(git rev-parse --git-dir)" && pwd) - common_dir=$(cd "$(git rev-parse --git-common-dir)" && pwd) - branch=$(git branch --show-current || true) - - # A written probe rather than `[[ -w ]]`: the directory can be readable and - # nominally writable while the sandbox, a read-only mount, or an ACL refuses - # the create, and it is the create that fetch and branch creation need. - access() { - local dir="$1" probe - [[ -d "$dir" ]] || { echo missing; return; } - [[ -r "$dir" ]] || { echo denied; return; } - probe="$dir/.moq-access-probe.$$" - if (umask 077 && : > "$probe") 2>/dev/null; then - rm -f "$probe" - echo write - else - echo read-only - fi - } - - # A directory git will create on demand is only as writable as its nearest - # existing parent, so probe upward instead of reporting it missing. - access_or_parent() { - local dir="$1" parent - while [[ ! -d "$dir" ]]; do - parent=$(dirname "$dir") - [[ "$parent" != "$dir" ]] || { echo missing; return; } - dir="$parent" - done - access "$dir" - } - - base=$(just _base "$BASE") - stamp="$git_dir/moq-base" - # Whichever remote provides the base, not always origin. A base whose first - # segment names no remote (a local branch, a tag) leaves origin. - remote="${base%%/*}" - git remote | grep -qx "$remote" || remote=origin - - objects=$(access "$common_dir/objects") - heads=$(access "$common_dir/refs/heads") - worktree_meta=$(access "$git_dir") - worktree=$(access "$root") - # Tracking is `branch..remote`/`.merge` in the repository config, which - # lives in the common directory alongside the `config.lock` the write needs, - # not in the branch's ref. Probing refs/heads for it would refuse on a - # writable config and, worse, proceed on a read-only one. - config=$(access "$common_dir") - # A fetch writes three places, not one: objects, the remote-tracking refs it - # updates, and FETCH_HEAD in the per-worktree directory. The refs are the - # narrow one: `refs/remotes/` is where `.lock` is created, so - # probing `refs/remotes` stops a directory short and passes a split that git - # then fails on. - tracking=$(access_or_parent "$common_dir/refs/remotes/$remote") - - echo "worktree: $root" - echo "branch: ${branch:-(detached)} $(git rev-parse --short HEAD)" - echo "git-dir: $git_dir ($worktree_meta)" - echo "common-dir: $common_dir ($config)" - echo " fetch needs $common_dir/objects: $objects" - echo " $common_dir/refs/remotes/$remote: $tracking" - echo " $git_dir (FETCH_HEAD): $worktree_meta" - echo " branch needs $common_dir/refs/heads: $heads" - echo " upstream needs $common_dir/config: $config" - echo " rebase needs $git_dir: $worktree_meta" - echo " $root: $worktree" - echo " $common_dir/refs/heads: $heads" +# Tokio/epoll and io_uring workers; WORKERS defaults to every logical CPU. - dirty=$(git status --porcelain | wc -l | tr -d ' ') - echo "dirty: $dirty tracked/untracked path(s)" - - if [[ "{{ ACTION }}" == setup ]]; then - blocked="" - [[ "$objects" == write ]] || blocked="$blocked $common_dir/objects ($objects)" - [[ "$tracking" == write ]] || blocked="$blocked $common_dir/refs/remotes/$remote ($tracking)" - [[ "$worktree_meta" == write ]] || blocked="$blocked $git_dir ($worktree_meta)" - if [[ -n "$blocked" ]]; then - echo "error: cannot fetch;$blocked" >&2 - echo " grant write access to the main repository's Git directory, not just this worktree" >&2 - exit 1 - fi - # The remote resolved above, not always origin: recording a stamp against a - # ref nobody refreshed is worse than recording none, and `just check` would - # scope the branch against a base that has since moved. - git fetch --quiet "$remote" - # Repointing an upstream someone chose would silently change what `just - # check` scopes against, so only three cases write it: no upstream, an - # upstream `_base` discards anyway (the branch's own remote copy, which - # `git push -u` leaves behind and which says nothing about what the branch - # merges into), and a base the caller named on the command line. - upstream=$(git rev-parse --abbrev-ref '@{upstream}' 2> /dev/null || true) - # Two things stop the write, and they fail identically: a detached HEAD has - # no branch to hang an upstream on, and the config it lands in may be - # read-only. Skipping either silently would report a setup that recorded - # the caller's base while `just check` still scoped against origin/main. - blocker="" - if [[ -z "$branch" ]]; then - blocker="HEAD is detached" - elif [[ "$config" != write ]]; then - blocker="$common_dir/config is $config" - fi - if [[ -z "$upstream" ]] || [[ "$upstream" == */"$branch" ]] || [[ -n "$BASE" ]]; then - if [[ -z "$blocker" ]]; then - git branch --set-upstream-to "$base" "$branch" - elif [[ "$base" == origin/main ]]; then - # Nothing is lost: with no upstream `_base` falls back to - # origin/main, which is what this would have written. - echo "warning: cannot record the upstream; $blocker" >&2 - else - # The upstream is the only place this choice survives, so a setup - # that could not write it did not do what it was asked. - echo "error: cannot set the upstream to $base; $blocker" >&2 - echo " the branch would keep scoping against origin/main" >&2 - exit 1 - fi - fi - # Resolved, written, then renamed into place. A redirect straight into the - # stamp truncates it before git runs, so a failure there would leave an - # empty file that reads back as a recorded base that never existed. Each - # setup needs its own temporary file so concurrent runs cannot rename or - # overwrite each other's in-progress stamp. - if [[ "$worktree_meta" == write ]]; then - stamp_tmp=$(mktemp "$git_dir/.moq-base.XXXXXXXX") - if ! git rev-parse "$base" > "$stamp_tmp"; then - rm -f "$stamp_tmp" - exit 1 - fi - mv "$stamp_tmp" "$stamp" - fi - fi - - if ! git rev-parse --verify --quiet "$base^{commit}" > /dev/null; then - echo "base: $base (NOT FETCHED; run 'just worktree setup')" - exit 0 - fi - - head=$(git rev-parse "$base") - echo "base: $base $(git rev-parse --short "$base")" - echo "upstream: $(git rev-parse --abbrev-ref '@{upstream}' 2> /dev/null || echo '(unset)')" - echo "behind: $(git rev-list --count "HEAD..$base") commit(s)" - - recorded="" - [[ -f "$stamp" ]] && recorded=$(cat "$stamp") - - # A stamp that no longer names a commit is worse than none: reporting it would - # abort here on the `rev-parse --short` rather than say what to do about it. - # An interrupted setup, or a base garbage-collected out of the repository. - if [[ -z "$recorded" ]]; then - echo "recorded: (none; run 'just worktree setup')" - elif ! git rev-parse --verify --quiet "$recorded^{commit}" > /dev/null; then - echo "recorded: $recorded (UNKNOWN COMMIT; run 'just worktree setup')" - elif [[ "$recorded" == "$head" ]]; then - echo "recorded: $(git rev-parse --short "$recorded") (current)" - else - echo "recorded: $(git rev-parse --short "$recorded") (STALE; $base has moved since setup)" - fi +# Compare a shared Tokio runtime with independent workers: `just bench-runtime 5 16`. +bench-runtime $ROUNDS="3" $WORKERS="": + MOQ_BENCH_RUNTIME_ROUNDS="$ROUNDS" MOQ_BENCH_RUNTIME_WORKERS="$WORKERS" bench/run.sh --runtime # Install repo-wide tooling. Per-language deps install on first check. install: bun install - cargo install --locked cargo-shear cargo-sort cargo-upgrades cargo-edit cargo-semver-checks release-plz - -# Resolve BASE: arg > $GITHUB_BASE_REF > upstream > origin/main. A branch's -# upstream is the branch it merges into, which is the base a `dev`-targeted -# branch needs. `git push -u` repoints upstream at the branch's own remote copy, -# which would diff HEAD against itself, so ignore that case (see CLAUDE.md). -# GITHUB_BASE_REF outranks the upstream because a PR checkout has no upstream -# configured, and the branch being merged into is exactly the base GitHub is -# asking about. -# -# Shared by `_changed` and `worktree`, so a checkout's scope and its reported -# base can never disagree. - -# Print the ref this branch is based on. -[private] -_base $BASE="": - #!/usr/bin/env bash - set -euo pipefail - - base="$BASE" - if [[ -z "$base" && -n "${GITHUB_BASE_REF:-}" ]]; then - base="origin/${GITHUB_BASE_REF}" - fi - if [[ -z "$base" ]]; then - base=$(git rev-parse --abbrev-ref '@{upstream}' 2>/dev/null || true) - if [[ -z "$base" || "$base" == */"$(git branch --show-current)" ]]; then - base="origin/main" - fi - fi - printf '%s\n' "$base" - -# Reports the base it picked on stderr, so a surprising scope is traceable. -# -# LIMIT is the byte budget for the printed list, and exists as a parameter so -# `_changed-test` can force the oversized path without a synthetic 30k-file diff. - -# Print the files this branch changed relative to BASE, one per line, or `ALL`. -[private] -_changed $BASE $LIMIT=changed_max: - #!/usr/bin/env bash - set -euo pipefail - - base=$(just _base "$BASE") - - merge_base=$(git merge-base "$base" HEAD) || { - echo "error: cannot resolve merge-base against $base (is full history fetched?)" >&2 - exit 1 - } - echo "base: $base" >&2 - - # Untracked files count too: a brand new crate or module is the whole change. - files=$({ - git diff --name-only "$merge_base" - git ls-files --others --exclude-standard - } | sort -u) - - # Assigned rather than tested inline, so a failing `_changed-cap` aborts here - # under `set -e`. Inside `[[ ]]` its exit status is discarded, and the caller - # would then scope to a list that never got budgeted -- straight back into the - # E2BIG this exists to prevent. - cap=$(printf '%s' "$files" | just _changed-cap "$LIMIT") - if [[ "$cap" == ALL ]]; then - echo ALL - exit 0 - fi - - # Guarded because a bare printf of an empty list prints a newline, and the - # callers test the result for emptiness to decide whether anything changed. - if [[ -n "$files" ]]; then - printf '%s\n' "$files" - fi - -# Every hop of the dispatch takes the changed-file list as one argument, and -# just exports recipe parameters into the child's environment, so the whole list -# has to fit in a single execve string. Linux caps one string at MAX_ARG_STRLEN -# (32 pages, 131072 bytes) however large ARG_MAX is, so past that each hop dies -# with E2BIG, which just reports as exit code 126 and no mention of the diff. A -# diff that large selects most of the workspace anyway, so the callers widen to -# the unscoped suite, which passes no list at all. -# -# Split out from `_changed` so `_changed-test` can drive the decision with a -# synthetic list, rather than needing the working tree to hold a diff of a -# particular size. -# -# The list arrives on stdin, which is both the only channel that can carry an -# oversized one and the only way to measure it honestly: `${#var}` counts -# CHARACTERS under a UTF-8 locale while execve counts BYTES, so a path set of -# 3-byte characters would read as a third of its real size and sail past a -# budget it actually blows. - -# Print `ALL` when the changed-file list on stdin is too long for one argument. -[private] -_changed-cap $LIMIT=changed_max: - #!/usr/bin/env bash - set -euo pipefail - - [[ "$LIMIT" =~ ^[0-9]+$ ]] || { - echo "changed: not a byte count: $LIMIT" >&2 - exit 2 - } - - bytes=$(wc -c | tr -d '[:space:]') - if ((bytes > LIMIT)); then - echo "changed: $bytes bytes of paths exceeds the $LIMIT budget; selecting everything." >&2 - echo ALL - fi - -# Guards the thing that fails LOUDLY but unhelpfully: past the budget every -# `just` hop dies with "Argument list too long" and exit 126, naming neither the -# diff nor the recipe that could not receive it. Both halves matter -- a budget -# that never trips scopes nothing, and one above what execve accepts still dies. - -# Check that an oversized diff widens to `ALL`, and that the budget fits in argv. -[private] -_changed-test $LIMIT=changed_max: - #!/usr/bin/env bash - set -euo pipefail - - fail() { echo "changed: _changed-test: $1" >&2; exit 1; } - - # Synthetic sizes rather than whatever the working tree happens to hold: a - # clean checkout has no diff at all, and `check --all` runs there (cache.yml - # warms the cache from `main`), so a test keyed on the real list would take - # down the one job allowed to write the shared Rust cache. - [[ "$(printf 'aaaa' | just _changed-cap 3)" == ALL ]] || fail "over budget must print ALL" - [[ -z "$(printf 'aaa' | just _changed-cap 3)" ]] || fail "at budget must print nothing" - [[ -z "$(printf '' | just _changed-cap 3)" ]] || fail "an empty diff must print nothing" - - # The smallest nonempty list there is, against the only budget below it. - [[ "$(printf 'x' | just _changed-cap 0)" == ALL ]] || fail "a 1-byte list must exceed a 0 budget" - - # execve counts bytes, so the budget has to as well. Spelled as raw bytes - # rather than as characters: this is one CJK character, 3 bytes wide, which - # `${#var}` would count as 1 under a UTF-8 locale and let past a 2-byte - # budget it actually blows. - [[ "$(printf '\xe6\x97\xa5' | just _changed-cap 2)" == ALL ]] \ - || fail "the budget must count bytes, not characters" - - # A byte count is the whole input, so anything else is a caller bug, not a - # reason to silently scope to nothing. - ! printf '' | just _changed-cap not-a-number 2> /dev/null || fail "a bad budget must be rejected" - - # ...and that rejection has to reach the caller. A swallowed one would hand - # back an unbudgeted list, which is the failure this whole recipe prevents. - ! just _changed "" not-a-number 2> /dev/null || fail "a rejected budget must fail _changed" - - # Linux caps a single argv/env string at MAX_ARG_STRLEN (32 pages), whatever - # ARG_MAX says, and the list travels as one string. Asserted rather than - # probed because this repo's CI is the Linux host and a dev box may be laxer. - ((LIMIT <= 131072)) || fail "budget $LIMIT exceeds Linux MAX_ARG_STRLEN" - - # The budget still has to survive the hop it was sized for, which the checks - # above cannot show: they never pass a list that big to anything. - payload=$(head -c "$LIMIT" /dev/zero | tr '\0' x) - [[ "$(just _echo "$payload" | wc -c)" -eq $((LIMIT + 1)) ]] \ - || fail "a $LIMIT-byte list does not survive an argv hop" - - # End to end, but only when there is a diff to be oversized: see above. The - # budget is zero rather than one because the real list is whatever the - # checkout holds, and a single one-character root path is a one-byte list - # that a one-byte budget does not exceed. Zero is below every nonempty list. - if [[ -n "$(just _changed "" 100000000)" ]]; then - [[ "$(just _changed "" 0)" == ALL ]] || fail "a diff over budget must print ALL" - fi - - echo "changed: budget ok" - -# Print an argument back, to measure what survives a `just` invocation. -[private] -_echo $VALUE: - @printf '%s\n' "$VALUE" - -# Tools every scope guards with `command -v`, so an incomplete local toolchain -# checks less instead of failing. That trade is wrong in CI, where a skip is -# indistinguishable from a pass, so CI exports MOQ_STRICT=1 and this turns the -# required set into a precondition. Checked up front, and as one list, so a -# missing tool is reported before a long compile rather than after it. -# -# Required per scope, mirroring what `check` actually dispatches for a given -# diff: demanding gradle on a docs-only PR would fail a run that was never going -# to invoke it. Takes the same file list as the dispatch, or `ALL` to require -# everything (`check --all`). -# -# One deliberate absence: swift exists only on macOS, and `swift check` skips -# off-macOS by design; swift.yml is its real gate. - -# Fail when a tool the diff's scopes need is missing. No-op unless MOQ_STRICT. -[private] -_tools $FILES="": - #!/usr/bin/env bash - set -euo pipefail - [[ -n "${MOQ_STRICT:-}" ]] || exit 0 - - scoped() { [[ "$FILES" == ALL ]] || grep -qE "$1" <<< "$FILES"; } - - # `_check-common` runs on every invocation, so its tools are unconditional. - tools=(actionlint bun jq nix nixfmt shellcheck shfmt taplo python3 nfpm dpkg-deb envsubst rpm) - scoped '^(drafts/|doc/\.vitepress/drafts\.ts$)' && tools+=(kramdown-rfc xml2rfc) - scoped '^(bench/|quest/|rs/|Cargo\.(toml|lock)$|rust-toolchain\.toml$)' && tools+=(cargo envsubst) - scoped '^(py/|pyproject\.toml$|uv\.lock$|rs/moq-ffi/)' && tools+=(uv) - scoped '^(kt/|rs/moq-ffi/)' && tools+=(gradle java) - # cargo because `go check` builds moq-ffi for the host, and skips on a - # missing cargo the same way it skips on a missing go. rsync because the - # publish scripts stage the mirror tree with it, so the publisher test skips - # without it, and a skip that keeps `just check` green is what MOQ_STRICT is - # here to prevent. - scoped '^(go/|rs/moq-ffi/)' && tools+=(go uniffi-bindgen-go cargo rsync) - scoped '^(dart/|rs/moq-ffi/)' && tools+=(cargo dart uniffi_bindgen_dart) - # Two obs recipes with two dispatch scopes, so two lines: over-requiring - # would fail a diff that never runs the recipe. `just obs compile` needs - # cargo to regenerate moq.h and pkg-config to locate Qt6 and ffmpeg. Every - # platform: the plugin type-checks against headers, and the dev shell ships - # those even on Darwin, where obs-studio can't build. - scoped '^(cpp/obs/|rs/libmoq/|flake\.nix$)' && tools+=(pkg-config cargo) - # `just obs check` lints with clang-format and gersemi, validates the CMake - # release configuration, and compares the three OBS pins, one of which moves - # on a flake.lock bump alone. - scoped '^(cpp/obs/|flake\.(nix|lock)$)' && tools+=(clang-format gersemi cmake) - - # Scopes overlap (rs/moq-ffi/ is in five of them), so the same tool can land - # in the list twice and be reported missing twice. Splitting on whitespace is - # safe: every entry is a bare command name. - tools=($(printf '%s\n' "${tools[@]}" | sort -u)) - - missing=() - for tool in "${tools[@]}"; do - command -v "$tool" >/dev/null 2>&1 || missing+=("$tool") - done - if ((${#missing[@]})); then - echo "error: MOQ_STRICT is set but these tools are missing: ${missing[*]}" >&2 - echo " run inside 'nix develop', or unset MOQ_STRICT to skip what isn't installed" >&2 - exit 1 - fi + cargo install --locked cargo-shear cargo-sort cargo-semver-checks release-plz # Lints and compiles only the packages the branch changed plus everything -# depending on them, so several worktrees can build at once. This is also what -# CI runs (with MOQ_STRICT=1), so there is no second, drifting definition of -# "checked". Tests are the sibling `just test`; `check --all` is the unscoped suite. +# depending on them; `check --all` is the unscoped suite. CI runs the same +# thing with MOQ_STRICT=1. BASE defaults to the branch's upstream. # Lint and compile what the branch changed since BASE, plus its dependents. -check $BASE="" *args: - #!/usr/bin/env bash - set -euo pipefail - - if [[ "$BASE" == --all ]]; then - files=ALL - else - files=$(just _changed "$BASE") - fi - - # `_changed` says ALL when the list outgrew what argv can carry. The unscoped - # suite is the path that passes no list at all, so it is the one that works. - # The dispatch below lives in these two files, and neither matches any - # language scope, so a PR that rewrites how CI dispatches would otherwise - # validate none of it. Widen to the unscoped suite instead. - if [[ "$files" != ALL ]] && grep -qE '^(justfile|test/justfile)$' <<< "$files"; then - echo "check: root orchestration changed; checking everything." >&2 - files=ALL - fi - - just _tools "$files" - - if [[ "$files" == ALL ]]; then - just js check - just drafts check - just rs check --workspace --exclude moq-net-fuzz {{ args }} - just rs tokio-features - just --justfile bench/justfile check - cargo run --quiet --locked --package quest -- check - # Not covered by the line above: moq-wasm only exists on the wasm32 target. - just rs wasm - just py check - just kt check - just swift check - just go check - just dart check - just obs check - just obs compile - just _flake - elif [[ -n "$files" ]]; then - just js check "$files" - just rs check-changed "$files" - if echo "$files" | grep -q '^bench/'; then - just --justfile bench/justfile check - fi - # Draft sources render into the doc site and carry their own kramdown-rfc - # plus vector checks; a drafts-only diff would otherwise skip both. - if echo "$files" | grep -qE '^(drafts/|doc/\.vitepress/drafts\.ts$)'; then - just drafts check - fi - # Quest documents form one graph, so validate the whole living tree when - # either a quest or its validator changes. - if echo "$files" | grep -qE '^(quest/|rs/quest/)'; then - cargo run --quiet --locked --package quest -- check - fi - just py check "$files" - just kt check "$files" - just swift check "$files" - just go check "$files" - just dart check "$files" - # Type-checking the plugin and its unit tests needs only headers, so it - # runs here rather than waiting for obs.yml to link them on Linux. libmoq - # is in scope because the plugin calls through its generated C header, and - # the tests restate those entry points as stubs, so an ABI change breaks - # both. flake.nix because it owns the libobs headers this compiles - # against -- obs.yml links against nixpkgs' obs-studio instead, so nothing - # else would notice that package going bad. - if echo "$files" | grep -qE '^(cpp/obs/|rs/libmoq/|flake\.nix$)'; then - just obs compile - fi - # flake.nix is in scope because `just obs check` is what compares the OBS - # version pinned there against buildspec.json, and either side can move. - # flake.lock too, because the third OBS the guard compares is nixpkgs' - # obs-studio, the one `just obs ci` links: it moves on a lock bump alone, - # and that bump is the change that opens the gap. - if echo "$files" | grep -qE '^(cpp/obs/|flake\.(nix|lock)$)'; then - just obs check - fi - # Validates flake eval + dev shell build; it no longer compiles the - # workspace, so it's cheap. Gated anyway: a pure doc/JS PR can't - # affect flake eval. - if echo "$files" | grep -qE '(^rs/|^Cargo\.(toml|lock)$|^flake\.lock$|\.nix$)'; then - just _flake - fi - else - echo "check: nothing changed." - fi - - just _check-common - -# Skips when nix is absent: the flake is not a precondition for working on the -# repo, and `_tools` already makes it required under MOQ_STRICT. - -# Validate flake evaluation and the dev shell build. -[private] -_flake: - @if command -v nix >/dev/null 2>&1; then nix flake check; fi - -# Repository-wide non-compiling checks shared by scoped and unscoped `check`. -# Optional shell, workflow, TOML, Nix, and justfile lints skip if missing. -# -# `bun install` because remark-cli lives in node_modules and `just js check` is -# where it would otherwise be installed, which a Rust-only diff skips. - -# Run shell checks or formatting over tracked files that exist in the worktree. -[private] -_shell $ACTION: - #!/usr/bin/env bash - set -euo pipefail - - if ! command -v shfmt >/dev/null 2>&1; then - exit 0 - fi - if [[ "$ACTION" == "check" ]] && ! command -v shellcheck >/dev/null 2>&1; then - exit 0 - fi - - scripts_file=$(mktemp) - trap 'rm -f "$scripts_file"' EXIT - shfmt -f=0 . > "$scripts_file" - - scripts=() - while IFS= read -r -d '' file; do - if git --literal-pathspecs ls-files --error-unmatch -- "$file" >/dev/null 2>&1; then - scripts+=("$file") - fi - done < "$scripts_file" - ((${#scripts[@]})) || exit 0 - - case "$ACTION" in - check) - shfmt --diff "${scripts[@]}" - shellcheck "${scripts[@]}" - ;; - fix) - shfmt --write "${scripts[@]}" - ;; - *) - echo "invalid shell action: $ACTION" >&2 - exit 2 - ;; - esac - -# remark-cli has no `--check`. `--frail` raises the exit code on lint messages, -# and only `--output` formats, so a file that is merely misformatted passes both -# ways. Format a scratch mirror and diff that, so the check stays read-only -# while `fix` still writes in place. - -# Run Markdown lints over the worktree, either checking formatting or applying it. -[private] -_markdown $ACTION: - #!/usr/bin/env bash - set -euo pipefail - - case "$ACTION" in - check) ;; - fix) - bun remark . --quiet --output - exit 0 - ;; - *) - echo "invalid markdown action: $ACTION" >&2 - exit 2 - ;; - esac - - mirror=$(mktemp -d) - trap 'rm -rf "$mirror"' EXIT - - extension_list=$(bun -e \ - 'import extensions from "markdown-extensions"; console.log(extensions.join("\n"))') - patterns=() - while IFS= read -r extension; do - [[ -n "$extension" ]] && patterns+=("*.$extension") - done <<< "$extension_list" - ((${#patterns[@]})) || { - echo "error: remark reported no Markdown extensions" >&2 - exit 1 - } - - # Untracked files are in scope so a new doc is linted before it is staged; - # --exclude-standard keeps build output out. - files=() - while IFS= read -r -d '' file; do - [[ -f "$file" ]] || continue - files+=("$file") - mkdir -p "$mirror/$(dirname "$file")" - cp "$file" "$mirror/$file" - done < <(git ls-files -z --cached --others --exclude-standard -- "${patterns[@]}") - ((${#files[@]})) || exit 0 - - # The config rides along so every .remarkignore pattern resolves against the - # mirror root exactly as it does here, and node_modules is where the plugins - # named by .remarkrc.mjs come from. - cp .remarkrc.mjs .remarkignore "$mirror/" - ln -s "$PWD/node_modules" "$mirror/node_modules" - - status=0 - (cd "$mirror" && bun remark . --quiet --frail --output) || status=$? - - stale=() - for file in "${files[@]}"; do - cmp -s "$file" "$mirror/$file" || stale+=("$file") - done - - if ((${#stale[@]})); then - echo "error: these files are not formatted, run 'just fix':" >&2 - printf ' %s\n' "${stale[@]}" >&2 - status=1 - fi - - exit "$status" - -# Check Markdown formatting detection without touching the caller's worktree. -[private] -_markdown-test: - #!/usr/bin/env bash - set -euo pipefail - - fail() { echo "markdown: _markdown-test: $1" >&2; exit 1; } - - repo=$PWD - fixture=$(mktemp -d) - trap 'rm -rf "$fixture"' EXIT - - git -C "$fixture" init --quiet - cp .remarkrc.mjs .remarkignore "$fixture/" - ln -s "$repo/node_modules" "$fixture/node_modules" - for file in dirty.md dirty.markdown; do - printf '# dirty_heading\n' > "$fixture/$file" - git -C "$fixture" add "$file" - done - - original='# dirty_heading' - if just --justfile "$repo/justfile" --working-directory "$fixture" \ - _markdown check > "$fixture/check.log" 2>&1; then - fail "formatter-only drift must fail the check" - fi - for file in dirty.md dirty.markdown; do - [[ "$(<"$fixture/$file")" == "$original" ]] \ - || fail "the check modified $file" - grep -q "$file" "$fixture/check.log" \ - || fail "the check did not name $file" - done - - just --justfile "$repo/justfile" --working-directory "$fixture" _markdown fix - for file in dirty.md dirty.markdown; do - [[ "$(<"$fixture/$file")" == '# dirty\_heading' ]] \ - || fail "the fix did not format $file" - done - just --justfile "$repo/justfile" --working-directory "$fixture" _markdown check - - echo "markdown: check/fix regression ok" - -# Repository-wide lints, shared by scoped and unscoped `check`. -[private] -_check-common: - just _changed-test - bun install --frozen-lockfile - just _markdown-test - just _markdown check - just _shell check - @if command -v taplo >/dev/null 2>&1; then RUST_LOG=error taplo format --check; fi - @if command -v nixfmt >/dev/null 2>&1; then nixfmt --check $(find . -name '*.nix' -not -path './node_modules/*' -not -path './target/*' -not -path './.venv/*' -not -path './.direnv/*'); fi - @for f in $(find . -name justfile -not -path './node_modules/*' -not -path './target/*' -not -path './.venv/*' -not -path './.direnv/*'); do just --fmt --check --justfile "$f"; done - just gh check - -# Scoped exactly like `check`, because `clippy --fix` compiles what it fixes. -# Pass `--all` for the unscoped version. +check $BASE="": + sh/dispatch.sh check "$BASE" # Auto-fix lint and formatting for what the branch changed since BASE. fix $BASE="": - #!/usr/bin/env bash - set -euo pipefail - - if [[ "$BASE" == --all ]]; then - files=ALL - else - files=$(just _changed "$BASE") - fi - - # Mirrors `check`: too long for argv means fix everything instead. - if [[ "$files" == ALL ]]; then - just js fix - just rs fix --workspace --exclude moq-net-fuzz - just py fix - just dart fix - just obs fix - elif [[ -n "$files" ]]; then - just js fix "$files" - just rs fix-changed "$files" - just py fix "$files" - just dart fix "$files" - if echo "$files" | grep -q '^cpp/obs/'; then - just obs fix - fi - else - echo "fix: nothing changed." - fi - - just _fix-common - -# Optional tools skip if missing locally. `bun install` for the same reason as -# `_check-common`. - -# Repository-wide fixes, shared by scoped and unscoped `fix`. -[private] -_fix-common: - bun install - just _markdown fix - just _shell fix - @if command -v taplo >/dev/null 2>&1; then RUST_LOG=error taplo format; fi - @if command -v nixfmt >/dev/null 2>&1; then nixfmt $(find . -name '*.nix' -not -path './node_modules/*' -not -path './target/*' -not -path './.venv/*' -not -path './.direnv/*'); fi - @for f in $(find . -name justfile -not -path './node_modules/*' -not -path './target/*' -not -path './.venv/*' -not -path './.direnv/*'); do just --fmt --justfile "$f"; done + sh/dispatch.sh fix "$BASE" # Build the packages. build: just js build just rs build - if command -v uv &> /dev/null; then just py build; fi - if command -v wasm-bindgen &> /dev/null; then just wasm; fi - -# Build browser/WASM bindings into @moq/wasm using the pinned wasm-bindgen toolchain. -wasm: - cargo build --locked -p moq-wasm --target wasm32-unknown-unknown --profile wasm-release - wasm-bindgen --target web --out-name moq \ - --out-dir js/wasm/dist "${CARGO_TARGET_DIR:-target}/wasm32-unknown-unknown/wasm-release/moq_wasm.wasm" - -# Only this checkout by default. Agent worktrees each carry their own artifacts -# now that the shared target dir is gone, and another agent is usually building -# in one right now: `cargo clean` under a running build fails it, and there is no -# way to tell a finished worktree from a busy one from out here. `just clean all` -# is the explicit opt-in for a machine the caller knows is idle. -# -# Source is never touched either way, dirty or untracked: this deletes build -# output, not work. Nothing here reaches a machine-wide store -- no Nix garbage -# collection, no cargo/bun/uv home cache -- because those are shared with every -# other checkout and rebuilding them costs far more than the space they hold. + just py build + just js wasm # Delete this checkout's build artifacts and caches; `all` includes agent worktrees. clean SCOPE="here": - #!/usr/bin/env bash - set -euo pipefail - - case "{{ SCOPE }}" in - here | all) ;; - *) - echo "usage: just clean [here|all]" >&2 - exit 2 - ;; - esac - - just rs clean - just js clean - just py clean - just kt clean - just swift clean - just go clean - just dart clean - - # Caches not owned by any one language: nix build result, direnv, wrangler. - rm -rf result .direnv - find . -name .claude -prune -o -type d -name .wrangler -prune -exec rm -rf {} + - - # Worktrees don't nest, so this recurses exactly one level. Tolerate stale - # worktrees on branches that predate this recipe. - if [[ "{{ SCOPE }}" == all ]]; then - for wt in .claude/worktrees/*/; do - [ -f "${wt}justfile" ] || continue - echo "==> cleaning ${wt}" - (cd "$wt" && just clean) || echo " (skipped: just clean failed in ${wt})" - done - fi - -# Upgrade any tooling -update: - just js update - just rs update - nix flake update - -# Serve the documentation locally. -doc: - cd doc && bun run dev + sh/clean.sh {{ SCOPE }} diff --git a/kt/.gitignore b/kt/.gitignore index f9ad4114a9..2a7da565b6 100644 --- a/kt/.gitignore +++ b/kt/.gitignore @@ -4,7 +4,7 @@ build/ local.properties .kotlin/ -# Generated by kt/scripts/{check,package}.sh. Never check these in. +# Generated by sh/kt/{check,package}.sh. Never check these in. moq-ffi/src/jvmAndAndroidMain/kotlin/uniffi/ moq-ffi/src/jvmMain/resources/ moq-ffi/src/androidMain/jniLibs/ diff --git a/kt/README.md b/kt/README.md index b25426f977..f8a55837c6 100644 --- a/kt/README.md +++ b/kt/README.md @@ -89,7 +89,6 @@ kt/ src/ jvmAndAndroidMain/kotlin/dev/moq/ Wrapper sources (Moq, Server, Aliases, Flows, Durations, Errors) jvmAndAndroidTest/ Facade smoke test - scripts/ check.sh, package.sh ``` The native/UniFFI layer stays in a single `dev.moq:moq-ffi` artifact because uniffi-linked libraries can't be split across separately packaged artifacts (the Python `moq-rs` wheel is one umbrella for the same reason). That constraint is about the *native* layer; the pure-Kotlin `dev.moq:moq` wrapper sits cleanly on top of it as its own artifact because it ships no native code. diff --git a/kt/justfile b/kt/justfile index d88d307c07..af419fb33e 100644 --- a/kt/justfile +++ b/kt/justfile @@ -9,42 +9,20 @@ set working-directory := '.' default: just check -# Missing tools are errors so wrapper drift cannot slip past a green check. -# # This runs the tests too, so there is no separate `just kt test`: the script -# builds, generates, and tests as one gradle invocation. Takes an optional -# newline-separated list of changed files and skips when none are Kotlin-scoped; -# `just kt check` (no FILES) always runs. +# builds, generates, and tests as one gradle invocation. # Build moq-ffi, regenerate the bindings, and run :moq-ffi:jvmTest + :moq:jvmTest. -check $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '^(kt/|rs/moq-ffi/)' <<< "$FILES"; then - echo "kt: no Kotlin changes; skipping." - exit 0 - fi - bash scripts/check.sh +check: + bash ../sh/kt/check.sh # Regenerate the checked-in uniffi bindings and host native library without # compiling or testing. For environments that intentionally lack Gradle. generate: - bash scripts/generate.sh + bash ../sh/kt/generate.sh # Assemble the KMP module from per-target moq-ffi binaries + bindings. # Used by .github/workflows/release-kt-ffi.yml; see kt/README.md for the - # expected --lib-dir layout. package *args: - bash scripts/package.sh {{ args }} - -# Remove gradle output plus the moq-ffi bindings and native libs that -# scripts/check.sh generates into the source tree (see kt/.gitignore). -clean: - #!/usr/bin/env bash - set -euo pipefail - find . -name .claude -prune -o -type d \( -name build -o -name .gradle -o -name .kotlin \) -prune -exec rm -rf {} + - rm -rf local.properties \ - moq/src/jvmAndAndroidMain/kotlin/uniffi \ - moq/src/jvmMain/resources \ - moq/src/androidMain/jniLibs + bash ../sh/kt/package.sh {{ args }} diff --git a/kt/moq-ffi/build.gradle.kts b/kt/moq-ffi/build.gradle.kts index c5e76a9f0b..4aa387e97f 100644 --- a/kt/moq-ffi/build.gradle.kts +++ b/kt/moq-ffi/build.gradle.kts @@ -11,7 +11,7 @@ // ├─ jvmMain JVM-specific: native libs as JAR resources // └─ androidMain Android-specific: native libs in jniLibs // -// Native libraries + bindings are populated by `kt/scripts/package.sh`: +// Native libraries + bindings are populated by `sh/kt/package.sh`: // src/jvmMain/resources/-/ (JNA classpath layout) // src/androidMain/jniLibs//libmoq_ffi.so (Android packaging layout) // src/jvmAndAndroidMain/kotlin/uniffi/moq/moq.kt (uniffi-bindgen output) diff --git a/py/justfile b/py/justfile index 2af24340e6..3f28e9029d 100644 --- a/py/justfile +++ b/py/justfile @@ -10,15 +10,6 @@ set working-directory := '.' -# Paths a Python check depends on. `rs/moq-ffi` is in scope because maturin -# bundles it into the moq-ffi wheel. `check`, `fix`, and `test` all skip -# themselves when a diff misses this; no FILES at all means force-run. -# -# They also skip when uv is absent, matching the kt/swift/go check scripts: an -# incomplete local toolchain should check less rather than fail. The root -# `_tools` preflight makes uv a hard requirement under MOQ_STRICT. -scope := '^(py/|pyproject\.toml$|uv\.lock$|rs/moq-ffi/)' - default: just check @@ -39,17 +30,7 @@ _develop: # the Docs after the fact. It mocks the native module, so it costs no Rust build. # Lint, type-check, editable-build, and render the docs. -check $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '{{ scope }}' <<< "$FILES"; then - echo "py: no Python changes; skipping." - exit 0 - fi - if ! command -v uv >/dev/null 2>&1; then - echo "py: uv not on PATH, skipping" >&2 - exit 0 - fi +check: uv sync --locked --no-install-workspace uv run --no-sync ruff check . uv run --no-sync ruff format --check . @@ -57,34 +38,14 @@ check $FILES="": uv run --no-sync pyright just docs -# Auto-fix lint and formatting issues. Skips on the same terms as `check`. -fix $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '{{ scope }}' <<< "$FILES"; then - echo "py: no Python changes; skipping." - exit 0 - fi - if ! command -v uv >/dev/null 2>&1; then - echo "py: uv not on PATH, skipping" >&2 - exit 0 - fi +# Auto-fix lint and formatting issues. +fix: uv sync --no-install-workspace uv run --no-sync ruff check --fix . uv run --no-sync ruff format . -# Run the unit tests. Skips on the same terms as `check`. -test $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '{{ scope }}' <<< "$FILES"; then - echo "py: no Python changes; skipping tests." - exit 0 - fi - if ! command -v uv >/dev/null 2>&1; then - echo "py: uv not on PATH, skipping" >&2 - exit 0 - fi +# Run the unit tests. +test: uv sync --locked --no-install-workspace just _develop uv run --no-sync pytest moq-rs/tests/ moq-ffi/tests/ @@ -96,18 +57,6 @@ build: uv sync --locked --no-install-workspace just _develop -# Remove the virtualenv, release dist, bytecode caches, and the uniffi -# bindings maturin drops in during editable installs. The uv workspace venv - -# lives at the repo root, so reach up for it. -clean: - #!/usr/bin/env bash - set -euo pipefail - rm -rf dist moq-ffi/moq_ffi/_uniffi - rm -rf ../.venv .venv - find . -name .claude -prune -o -type d -name __pycache__ -prune -exec rm -rf {} + - find . -name .claude -prune -o -type f -name '*.pyc' -exec rm -f {} + - # Build the pure-python moq-rs wrapper sdist + wheel into py/dist (for release). # moq-ffi is built separately by maturin (see release-py.yml); the wrapper is diff --git a/quest/m1/browser-benchmarks.md b/quest/m1/browser-benchmarks.md index 60c5dcb212..06043fac8d 100644 --- a/quest/m1/browser-benchmarks.md +++ b/quest/m1/browser-benchmarks.md @@ -7,7 +7,7 @@ Criterion targets and native relay load generator do not exercise. ## Plan -`rs/scripts/bench.sh::criterion_targets` discovers Cargo targets only. Existing JS unit +`bench/run.sh::criterion_targets` discovers Cargo targets only. Existing JS unit tests validate behavior, and `test/wasm` validates browser interop, but neither provides a repeatable JS performance comparison. Reuse the existing relay/browser harness pieces and add a focused recipe with artifacts under the benchmark diff --git a/quest/m1/dart-leak.md b/quest/m1/dart-leak.md index 9feb833b86..8a70080748 100644 --- a/quest/m1/dart-leak.md +++ b/quest/m1/dart-leak.md @@ -18,7 +18,7 @@ bound. path. The fix belongs upstream, not here: `dart/moq_ffi/lib/src/uniffi_runtime.dart` -is generated, and `dart/scripts/check.sh` diffs it against a fresh +is generated, and `sh/dart/check.sh` diffs it against a fresh `generate.sh` run, so an in-tree patch fails the staleness check by design. That upstream fix is merged as diff --git a/quest/m1/performance-comparisons.md b/quest/m1/performance-comparisons.md index 0169acbf11..3e53d674fe 100644 --- a/quest/m1/performance-comparisons.md +++ b/quest/m1/performance-comparisons.md @@ -7,7 +7,7 @@ estimate and preserved evidence, so a small reported speedup can be evaluated. ## Plan -`rs/scripts/bench.sh` runs each relay workload once as base then current, +`bench/run.sh` runs each relay workload once as base then current, without repeated rounds or alternating execution order. `cleanup` deletes the run directory, including Criterion estimates, load/host JSONL, relay logs, and summaries. Preserve the existing default command @@ -24,7 +24,7 @@ while extending this harness rather than creating another benchmark runner. hardware/kernel, allocator, affinity, workload, and execution order. Preserve partial evidence on failure while still cleaning up owned processes/worktrees. - Distinguish throughput-window counters from cumulative latency/loss. Today - `rs/scripts/bench.sh::summarize_load` differences bytes over the last five seconds + `bench/relay.sh::summarize_load` differences bytes over the last five seconds but reads final lifetime latency and group-loss counters. Label that explicitly; consume windowed data when the existing latency quest supplies it. Never subtract percentiles or call cumulative loss a steady-state sample. @@ -36,10 +36,6 @@ while extending this harness rather than creating another benchmark runner. samples. Validate an unchanged-revision A/A run and a deliberately degraded fixture; keep normal machine timing informational rather than a flaky CI gate. -## Required - -- [Thin justfiles](/quest/m1/tooling/justfiles.md) - finish benchmark script relocation before changing its lifecycle - ## Related - [Windowed latency](/quest/m1/3126-moq-bench-every-readme-example-fails-to-parse-and.md) - owns histogram/window semantics diff --git a/quest/m1/performance-profiles.md b/quest/m1/performance-profiles.md index 8fe119b4ee..eb52ef23b1 100644 --- a/quest/m1/performance-profiles.md +++ b/quest/m1/performance-profiles.md @@ -8,7 +8,7 @@ cost. Profiling is opt-in and has no production overhead when disabled. ## Plan -`rs/scripts/bench.sh` already owns the builds, relay PID, workload, and host +`bench/run.sh` already owns the builds, relay PID, workload, and host samples, but has no profiler integration. Reuse that lifecycle instead of adding a second launcher. `Cargo.toml` already has a `profiling` profile and `rs/moq-native/src/jemalloc.rs` already supports on-demand heap dumps. diff --git a/quest/m1/tooling/README.md b/quest/m1/tooling/README.md index 9f2ce96ac4..a0c524e019 100644 --- a/quest/m1/tooling/README.md +++ b/quest/m1/tooling/README.md @@ -19,7 +19,6 @@ requires the one before it, so they land as one line of pull requests. ## Quests -- [Thin justfiles](/quest/m1/tooling/justfiles.md) - recipe bodies move to `sh/`, one impact map scopes check/fix/test, self-tests and guards are deleted - [Workflows call just](/quest/m1/tooling/workflows-call-just.md) - no workflow `run:` step names a `.sh`; every script a workflow needs has a recipe - [Binary release workflow](/quest/m1/tooling/release-binary.md) - moq-cli and moq-relay share one reusable workflow behind two thin callers - [FFI release workflow](/quest/m1/tooling/release-ffi.md) - the five `release-*-ffi.yml` share the moq-ffi target matrix and artifact staging diff --git a/quest/m1/tooling/justfiles.md b/quest/m1/tooling/justfiles.md deleted file mode 100644 index de77164f40..0000000000 --- a/quest/m1/tooling/justfiles.md +++ /dev/null @@ -1,93 +0,0 @@ -# [L] Thin justfiles with one impact map - -## Goal - -Every justfile is a menu: each recipe is a single line that runs a tool or a -script under `sh/`, `just --list` shows each recipe once, and no recipe body -is bash. The scoped `check`, `fix`, and `test` resolve the branch diff once, -in one impact map, instead of a scope regex per language justfile plus a -mirror of them in the strict-tools preflight. Nothing that tests the tooling -itself runs on a pull request. A recipe that no workflow, doc, skill, or other -recipe invokes is gone, except the hand-run demo and infra ops recipes, which -stay as they are. - -## Plan - -Layout: - -- New root `sh/`. Root helpers sit at the top (`sh/dispatch.sh`, - `sh/markdown.sh`, `sh/shell.sh`, `sh/clean.sh`); per-area tooling moves - under `sh//`: `rs/scripts` to `sh/rs`, `.github/scripts` to `sh/gh`, - `go/scripts` to `sh/go`, `kt/scripts` to `sh/kt`, `swift/scripts` to - `sh/swift`, `dart/scripts` to `sh/dart`, and the OBS recipe bodies to - `sh/obs`. Workflow references to moved paths are updated mechanically here; - the next quest replaces them with recipes. -- Package build scripts that nix, CMake, gradle, and the dart build hook - point at stay put: `rs/*/build.sh`, `rs/moq-gst/{package,scrub,smoke}.sh`, - `cpp/obs/build.sh`, `infra/*/publish.sh`. The harness under `test/` is - harness code, not tooling; it stays too. -- `shfmt -f .` already enumerates `sh/`, so the shell lint covers everything - the justfiles stop containing. - -Dispatch: - -- `just check [BASE]`, `just fix [BASE]`, and `just test [BASE]` are one line - each: `sh/dispatch.sh check|fix|test "$BASE"`. The script resolves BASE - (arg, `GITHUB_BASE_REF`, upstream, `origin/main`), lists changed files into - a temp file, and applies one impact map: which language modules run, which - tools `MOQ_STRICT` demands, and whether the root orchestration changed and - everything runs. The map lives in one place; the `scope` variables in the - js, py, kt, swift, go, and dart justfiles and the regex table in `_tools` - are deleted with it. -- Per-language `check`, `fix`, and `test` lose their `$FILES` parameter and - skip logic. Only rs still receives the list: `just rs check-changed - ` (and `fix-changed`, `test-changed`) hand the path to - `sh/rs/select.sh`, which does the crate selection today spread over - `_select`, `_names`, and `_wants-wasm`. -- Passing a path instead of the list kills the argv budget: - `changed_max`, `_changed-cap`, `_changed-test`, `_echo`, and the E2BIG - commentary go. -- `check --all`, `fix --all`, and `test all` keep their names; cache.yml, - nightly.yml, and the docs call them. - -Delete: - -- Self-tests: `_changed-test`, `_markdown-test`, `_select-test`, - `_doc-names-test`, `_publish-test`, `sh/rs/package-nfpm.test.sh`, - `sh/gh/package-binary.test.sh`. -- Guards: `_doc-names` and `doc-names.jq`, and the publish lower-bound - check. This is a deliberate trade: the guards were incidents' worth of - bash plus their self-tests on every pull request. Where each surfaces - instead: a doc-directory collision shows up as an intermittent `failed to - remove directory` in the `cargo doc` step of whichever pull request selects - both crates, not deterministically (resolve with `doc = false` as before); - a stale internal lower bound fails release-plz's publish. `_fuzz-lock` is - already gone: #3543 folded the fuzz harness into the workspace so the root - lockfile and `--locked` cover it. `alert.sh check-coverage` stays: it is a - lint of alert.yml, not of a recipe. -- The `worktree` recipe and its 170 lines, plus the Worktrees section of - `test/README.md`; keep `_base` folded into `sh/dispatch.sh`. -- `rs bump` and `rs semver`. `rs release` stays for release-rs.yml. -- The `mod` lines in `demo/justfile`: `just pub`, `just relay`, `just boy`, - `just sub`, `just web` are the only spelling, and `--list` stops showing - each twice. `just demo` remains the default recipe and `just dev` its - documented alias. - -Rename: - -- Root `wasm` becomes `js wasm`: it emits `js/wasm/dist`, so it is a JS - package build. `rs wasm` stays the compile gate and `test wasm` the browser - run. Update wasm.yml, root `build`, `test/wasm/run.sh` (which runs `just - wasm` to build the package under test), and the four doc references. - -Keep, moved into scripts unchanged in behavior: the remark mirror-and-diff in -`sh/markdown.sh` (remark-cli has no check mode and the lint presets are -wanted), `_shell`, `_flake` as its one-liner, `clean`, the rs `package` and -`fuzz` bodies, and the OBS `compile`, `_includes`, `_unit`, `test`, `check`, -and `preset` bodies. - -Docs: `doc/setup/dev.md`, `CONTRIBUTING.md`, `test/README.md`, and the -`CLAUDE.md` mentions of `just wasm` follow the survivors. Verify with `just -check`, `just test`, and `just check --all`, and confirm every recipe name -check.yml, cache.yml, nightly.yml, interop.yml, wasm.yml, obs.yml, swift.yml, -and release-*.yml invoke still resolves. diff --git a/quest/m1/tooling/workflows-call-just.md b/quest/m1/tooling/workflows-call-just.md index 4d6c79c4e8..a38fcc5217 100644 --- a/quest/m1/tooling/workflows-call-just.md +++ b/quest/m1/tooling/workflows-call-just.md @@ -39,7 +39,3 @@ the swift `package`, `package-ffi`, `publish`, `publish-ffi`, `verify`, `workflow_dispatch` of apt-repo.yml, rpm-repo.yml, and release-winget.yml where the inputs allow a dry run. The tag-triggered release workflows are verified by the next release; say so in the PR. - -## Required - -- [Thin justfiles](/quest/m1/tooling/justfiles.md) - establishes `sh/` and the module recipes the workflows will call diff --git a/rs/justfile b/rs/justfile index dba4cbbf3d..4a2a3f3ad6 100644 --- a/rs/justfile +++ b/rs/justfile @@ -11,120 +11,30 @@ set working-directory := '..' # `moq-net-fuzz` is a workspace member only so it shares the root lockfile; libFuzzer # needs a nightly toolchain and sanitizer flags, so `just rs fuzz` is the one recipe -# that may compile it. Every `--workspace` below therefore carries this, and a new -# one that forgets it pays a vendored C++ libFuzzer build for nothing. The root -# justfile's `check --all` and `fix --all` spell it out, since a module cannot read this. +# that may compile it. Every `--workspace` below therefore carries this, as does +# sh/rs/select.sh, since it cannot read this. no_fuzz := "--exclude moq-net-fuzz" default: just check -# Default features for the selected workspace packages, here and in CI. The full -# workspace permutations (`--all-features`, `--no-default-features`) each cost a -# separate compile that shares almost nothing with this one, so `features` below -# covers them nightly. `check-changed` makes one scoped exception for moq-tokio, -# whose feature-gated transport surface must compile in isolation to avoid -# workspace feature unification hiding a broken configuration. -# -# No `cargo check` pass: clippy is a superset of it, and the two use different -# rustc wrappers, so running both means compiling the workspace twice. +# Default features for the selected packages. The full workspace permutations +# each cost a separate compile, so `features` covers them nightly. No `cargo +# check` pass: clippy is a superset of it, and the two use different rustc +# wrappers, so running both compiles the workspace twice. # # A bare `just rs check` is default-members, which skips libmoq, moq-ffi, -# moq-gst, and moq-wasm (they need cbindgen, a uniffi build, GStreamer, and a -# wasm32 target respectively, so they stay out of a casual local check). Every -# recipe that means "all of it" therefore passes `--workspace` explicitly, since -# `check` is the only gate now and nothing else would compile them. +# moq-gst, and moq-wasm; pass `--workspace` for all of it. # Compile, lint, format-check, doc-check, and verify dependency hygiene. check *args: - just rs _select-test - just rs _publish-test - just rs _doc-names-test - # A new MoqConnectionStats field has to reach every binding's doc page. - python3 rs/scripts/stats-docs.py - rs/scripts/package-nfpm.test.sh + python3 sh/rs/stats-docs.py cargo clippy --locked --all-targets {{ args }} -- -D warnings cargo fmt --all --check - just rs _doc-names RUSTDOCFLAGS="-D warnings" cargo doc --locked --no-deps {{ args }} cargo shear cargo sort --workspace --check --no-format -# Two targets whose names differ only by `-` vs `_` render to the same -# `target/doc/`, and a `cargo doc` that selects both races to clean it: the -# collision warning escalates to `error: failed to remove directory` depending on -# which rustdoc process gets there first. That reads as a flake, and because the doc -# pass above is diff-scoped it surfaces on whichever unrelated PR happens to select -# both crates. Resolve a pair by setting `doc = false` on whichever target has no -# published API to render (usually the binary). -# -# Metadata only, so this costs no compilation and can sit in the PR gate. Takes an -# optional metadata file so `_doc-names-test` can feed it synthetic workspaces. -_doc-names metadata="": - #!/usr/bin/env bash - set -euo pipefail - - if [[ -n "{{ metadata }}" ]]; then - collisions=$(jq -r -f rs/scripts/doc-names.jq < "{{ metadata }}") - else - collisions=$(cargo metadata --locked --no-deps --format-version 1 | jq -r -f rs/scripts/doc-names.jq) - fi - - if [[ -n "$collisions" ]]; then - echo "rs: these targets render to the same doc directory:" >&2 - echo "$collisions" >&2 - echo "set doc = false on whichever has no published API." >&2 - exit 1 - fi - -# Both directions of `_doc-names`, since the interesting half is what it must NOT -# report: cargo documents only the library when a binary shares its own package's -# name, so flagging that would reject the conventional src/lib.rs + src/main.rs layout. -_doc-names-test: - #!/usr/bin/env bash - set -euo pipefail - - scratch=$(mktemp -d) - trap 'rm -rf "$scratch"' EXIT - - # Same package, same name: cargo suppresses the bin, so this must pass. - cat > "$scratch/ok.json" <<'JSON' - {"packages":[{"name":"solo","targets":[ - {"name":"solo","kind":["lib"],"doc":true}, - {"name":"solo","kind":["bin"],"doc":true}]}]} - JSON - just rs _doc-names "$scratch/ok.json" - - # Same package, differing only by `-` vs `_`: cargo folds before suppressing, so - # this must pass too. Comparing the names as spelled would report it. - cat > "$scratch/folded.json" <<'JSON' - {"packages":[{"name":"folded","targets":[ - {"name":"foo_bar","kind":["lib"],"doc":true}, - {"name":"foo-bar","kind":["bin"],"doc":true}]}]} - JSON - just rs _doc-names "$scratch/folded.json" - - # Different packages: a real collision, must be reported. - cat > "$scratch/bad.json" <<'JSON' - {"packages":[ - {"name":"a-lib","targets":[{"name":"shared_name","kind":["lib"],"doc":true}]}, - {"name":"b-cli","targets":[{"name":"shared-name","kind":["bin"],"doc":true}]}]} - JSON - if just rs _doc-names "$scratch/bad.json" 2>/dev/null; then - echo "rs: _doc-names missed a cross-package collision" >&2 - exit 1 - fi - - # `doc = false` is how a pair gets resolved, so it must stop being reported. - cat > "$scratch/fixed.json" <<'JSON' - {"packages":[ - {"name":"a-lib","targets":[{"name":"shared_name","kind":["lib"],"doc":true}]}, - {"name":"b-cli","targets":[{"name":"shared-name","kind":["bin"],"doc":false}]}]} - JSON - just rs _doc-names "$scratch/fixed.json" - - echo "rs: _doc-names ok" - # Auto-fix clippy/format/shear/sort. fix *args: cargo clippy --locked --fix --allow-staged --allow-dirty --all-targets {{ args }} @@ -132,391 +42,27 @@ fix *args: cargo shear --fix cargo sort --workspace --no-format -# Prints `ALL` when the diff hits something every crate depends on, and nothing -# when no crate is affected. -# -# FILES is an exported parameter, not `{{ FILES }}`: just interpolates the -# latter into the recipe source, where a filename like `$(...)` would run as a -# command. Git-derived paths are untrusted input, so they stay data. - -# Print `--package` flags for the crates a diff touches, plus their dependents. -[private] -_select $FILES: - #!/usr/bin/env bash - set -euo pipefail - - # Workspace-wide inputs affect every crate, so there is nothing to scope to. - # - # `rs/justfile` and the nextest config are in here because they define how - # every crate is checked and tested. They also match `^rs/` below while - # seeding no crate (the seed pattern needs `rs//...`), so without this - # a PR editing only the recipes would compile and test nothing at all, and a - # broken cargo flag would pass CI on a format check alone. - if [[ -z "$FILES" ]] || grep -qE '^(Cargo\.(toml|lock)|rust-toolchain\.toml|rs/justfile|\.config/nextest\.toml)$' <<< "$FILES"; then - echo ALL - exit 0 - fi - - grep -q '^rs/' <<< "$FILES" || exit 0 - - # `--no-deps` keeps this to workspace members and off the network. - metadata=$(cargo metadata --format-version 1 --no-deps) - - # A seed is the first path segment under `rs/`, so only a crate whose manifest - # sits directly at `rs//Cargo.toml` can ever be one. Drop the rest here: - # `moq-net-fuzz` lives at `rs/moq-net/fuzz`, so its seed is already `moq-net`, - # and leaving it in the graph would select it as a dependent of the crate it - # fuzzes and compile libFuzzer on every moq-net diff. - # - # `manifest_path` is a native path, so normalize the separator before splitting - # on it. Windows would otherwise match no segment, drop every package, and - # select nothing at all, where the assertion below fails safe to ALL. - metadata=$(jq '.packages |= map((.manifest_path |= gsub("\\\\"; "/")) | select(.manifest_path | split("/")[-3] == "rs"))' <<< "$metadata") - - # Every remaining crate directory is named after the crate it holds, which is - # what makes a changed path a seed below. Verify it rather than trust it: under- - # selecting silently checks nothing, so fall back to checking everything. - if jq -e '.packages[] | select((.manifest_path | split("/")[-2]) != .name)' <<< "$metadata" > /dev/null; then - echo "rs: a crate directory no longer matches its crate name; selecting everything." >&2 - echo ALL - exit 0 - fi - - seeds=$(sed -n 's|^rs/\([^/]*\)/.*|\1|p' <<< "$FILES" | sort -u) - - # Every workspace member, not just the default ones. `--no-deps` already - # limits this to the workspace. The non-default members (libmoq, moq-ffi, - # moq-gst, moq-wasm) have to participate as both seeds and dependents: they - # used to drop out here because `check` matched a bare `cargo check`, but - # `check` is the only gate now, and the `cargo clippy --workspace` in the - # deleted `just rs ci` is what used to cover them. Without them, a PR - # touching only rs/libmoq selects nothing and compiles nothing. - edges=$(jq -r ' - .packages[] - | .name as $name - | .dependencies[] - | "\($name) \(.name)" - ' <<< "$metadata") - - # A crate has to be rebuilt when anything it depends on changed, so walk the - # edges backwards until the selection stops growing. - selected=$(awk -v seeds="$seeds" ' - BEGIN { split(seeds, s, "\n"); for (i in s) want[s[i]] = 1 } - { pkg[NR] = $1; dep[NR] = $2 } - END { - do { - grew = 0 - for (i = 1; i <= NR; i++) - if (want[dep[i]] && !want[pkg[i]]) { want[pkg[i]] = 1; grew = 1 } - } while (grew) - # Every wanted crate, not just those that appear as an edge source: a - # crate with no dependencies of its own emits no edge, so keying the - # output on `pkg[i]` would drop it and select nothing for a direct - # change to it. Test the VALUE, not the key -- awk creates a key on - # every `want[dep[i]]` read above, so iterating keys alone would emit - # every crate the workspace mentions. The lookup below drops names that - # are not workspace crates, which is what keeps a seed like the - # non-crate `rs/scripts` from reaching cargo. - for (name in want) if (want[name]) print name - } - ' <<< "$edges" | sort -u) - - [[ -n "$selected" ]] || exit 0 - - # Emit cargo's own package ids, not bare names. A workspace crate we also - # publish collides with its crates.io copy as soon as any dependency pulls - # that copy in (the web-transport-* crates depend on a published `kio`), and - # `--package kio` is then ambiguous: cargo refuses during package selection, - # before compiling anything, so the whole gate fails. Ids from `cargo - # metadata` carry the source, so they are unambiguous by construction. - jq -r --arg selected "$selected" ' - ($selected | split("\n")) as $want - | .packages[] - | select(.name as $name | $want | index($name) != null) - | "--package \(.id)" - ' <<< "$metadata" | sort -u | tr '\n' ' ' - -# Bare crate names for the `--package ` flags `_select` emits, for log lines -# and for the wasm gate's alternation below. Matching names rather than the flag -# spelling keeps those call sites working whatever spec form `_select` uses; the -# previous alternation matched `--package ` literally and silently stopped -# firing the moment the spec form changed. Ids are `path+file://#` -# and `_select` only emits crates whose directory it has verified matches -# their crate name, so the last path segment is the name. - -# True when a selection includes a crate the wasm32 pass covers. One predicate -# rather than the same alternation in each gate, so the test below exercises what -# the gates actually run instead of a copy of it. -[private] -_wants-wasm $PACKAGES: - #!/usr/bin/env bash - set -euo pipefail - grep -qwE '(moq-wasm|moq-mux|moq-ffi)' <<< "$(just rs _names "$PACKAGES")" - -# True when the selection includes moq-tokio, whose zero-feature build is hidden -# by workspace feature unification whenever any dependent enables a transport. -[private] -_wants-tokio-features $PACKAGES: - #!/usr/bin/env bash - set -euo pipefail - grep -qw 'moq-tokio' <<< "$(just rs _names "$PACKAGES")" - -# True when the selection includes a crate with a `capture` feature. Selection -# pulls in dependents, so a diff to moq-mux or moq-net lands here too, which is -# how a rename in either reaches the device code that calls it. -[private] -_wants-capture $PACKAGES: - #!/usr/bin/env bash - set -euo pipefail - grep -qwE '(moq-video|moq-audio)' <<< "$(just rs _names "$PACKAGES")" - -# True when the selection includes moq-relay, whose io_uring listener is off the -# default feature set. Selection pulls in dependents, so a diff to any crate the -# relay is built from (moq-uring, moq-sock, moq-tokio) lands here too. -[private] -_wants-uring $PACKAGES: - #!/usr/bin/env bash - set -euo pipefail - grep -qw 'moq-relay' <<< "$(just rs _names "$PACKAGES")" - -# Print the crate names behind a list of `--package` flags. -[private] -_names $PACKAGES: - #!/usr/bin/env bash - set -euo pipefail - tr ' ' '\n' <<< "$PACKAGES" | sed -n 's|.*/\([^/#]*\)#.*|\1|p' | sort -u | tr '\n' ' ' - -# Guards the two things about selection that fail SILENTLY rather than loudly: -# a `--package` spec cargo cannot resolve takes the gate down before it compiles -# anything, and a wasm gate that stops matching skips the only pass that ever -# looks at moq-wasm's code (its crate root is `#![cfg(target_arch = "wasm32")]`, -# so the host pass sees an empty crate and reports success). Both have happened. - -# Check that selection emits resolvable specs and still drives the wasm gate. -[private] -_select-test: - #!/usr/bin/env bash - set -euo pipefail - - fail() { echo "rs: _select-test: $1" >&2; exit 1; } - - # `kio` is published and pulled from crates.io by the web-transport-* crates, - # so the workspace copy and the registry copy share a name: a bare name is - # ambiguous and cargo refuses. Every emitted spec must name its source, and - # cargo must accept it. - packages=$(just rs _select "rs/kio/src/waiter.rs") - for spec in $(sed 's/--package //g' <<< "$packages"); do - [[ "$spec" == *"file://"* ]] || fail "expected a source-qualified id, got: $spec" - cargo pkgid --offline "$spec" > /dev/null 2>&1 || fail "cargo cannot resolve: $spec" - done - - # The wasm gate keys on names, so it survives whatever spec form the ids use. - just rs _wants-wasm "$packages" \ - || fail "a kio diff must still drive the wasm gate: $(just rs _names "$packages")" - - # ...and it must still be able to say no, or it is not a gate. - relay=$(just rs _select "rs/moq-relay/src/web.rs") - if just rs _wants-wasm "$relay"; then - fail "a moq-relay diff must not drive it: $(just rs _names "$relay")" - fi - - # The break the capture gate was added for: a moq-mux API change that missed - # its callers behind `capture`. A moq-mux diff has to reach it. - mux=$(just rs _select "rs/moq-mux/src/clock.rs") - just rs _wants-capture "$mux" \ - || fail "a moq-mux diff must drive the capture gate: $(just rs _names "$mux")" - if just rs _wants-capture "$relay"; then - fail "a moq-relay diff must not drive the capture gate: $(just rs _names "$relay")" - fi - - # The break the uring gate was added for: a change that breaks moq-relay's - # io_uring listener, which the default pass above never compiles. A moq-relay - # diff has to reach it, and a diff that cannot affect the relay must not. - just rs _wants-uring "$relay" \ - || fail "a moq-relay diff must drive the uring gate: $(just rs _names "$relay")" - hls=$(just rs _select "rs/moq-hls/src/lib.rs") - if just rs _wants-uring "$hls"; then - fail "a moq-hls diff must not drive the uring gate: $(just rs _names "$hls")" - fi - - tokio=$(just rs _select "rs/moq-tokio/src/lib.rs") - just rs _wants-tokio-features "$tokio" \ - || fail "a moq-tokio diff must drive its feature matrix: $(just rs _names "$tokio")" - if just rs _wants-tokio-features "$relay"; then - fail "a moq-relay-only selection must not drive the moq-tokio feature matrix" - fi - - # A non-crate directory under rs/ seeds a name that is not a package. It must - # select nothing rather than reach cargo as a bogus spec. - [[ -z "$(just rs _select "rs/scripts/package-binary.sh")" ]] \ - || fail "a non-crate seed must select nothing" - - # A crate nested inside another crate's directory is covered by its parent's - # seed. Selecting it directly would compile libFuzzer on every moq-net diff, - # and its directory name is not its crate name, so it would also mis-report. - fuzz=$(just rs _names "$(just rs _select "rs/moq-net/fuzz/fuzz_targets/lite.rs")") - grep -qw moq-net <<< "$fuzz" || fail "a fuzz diff must select moq-net: $fuzz" - ! grep -qw fuzz <<< "$fuzz" || fail "a fuzz diff must not select the harness: $fuzz" - - echo "rs: selection ok" - -# Published crates need the current workspace version as the lower bound for -# internal dependencies. The workspace only tests that version set, and a looser -# bound lets package verification combine incompatible older releases. -# -# Internal dev-dependencies must be path-only. `cargo publish` keeps a versioned -# dev-dependency and requires it on crates.io, but release-plz orders publishes -# by normal dependencies only, so a bumped dev-dependency is often not there yet. -[private] -_publish-test: - #!/usr/bin/env bash - set -euo pipefail - - metadata=$(cargo metadata --locked --format-version 1 --no-deps) - stale=$(jq -r ' - .packages as $packages - | $packages[] as $package - | select($package.publish != []) - | $package.dependencies[] - | select(.source == null and .path != null and .kind != "dev") - | . as $dependency - | ($packages[] | select(.name == $dependency.name)) as $current - | select($dependency.req != ("^" + $current.version)) - | "\($package.name): \($dependency.name) requires \($dependency.req), current is ^\($current.version)" - ' <<< "$metadata") - - if [[ -n "$stale" ]]; then - echo "rs: published workspace dependency lower bounds are stale:" >&2 - echo "$stale" >&2 - exit 1 - fi - - versioned=$(jq -r ' - .packages[] - | select(.publish != []) - | .name as $package - | .dependencies[] - | select(.source == null and .path != null and .kind == "dev" and .req != "*") - | "\($package): \(.name) \(.req)" - ' <<< "$metadata") - - if [[ -n "$versioned" ]]; then - echo "rs: published crates have versioned workspace dev-dependencies:" >&2 - echo "$versioned" >&2 - echo "use { path = \"../\" } instead of { workspace = true }." >&2 - exit 1 - fi - - echo "rs: published workspace dependencies current" - -# Takes a newline-separated list of changed files; skips when no crate is -# affected. The `just rs ...` calls below go through the root justfile's -# `mod rs` because these recipes run from the repo root, where a bare `check` -# would resolve to the root recipe of that name. - -# Like `check`, but only the crates a diff touches plus their dependents. -check-changed $FILES: - #!/usr/bin/env bash - set -euo pipefail - packages=$(just rs _select "$FILES") - case "$packages" in - "") echo "rs: no crates affected; skipping." ;; - ALL) just rs check --workspace {{ no_fuzz }} ;; - *) echo "rs: checking $(just rs _names "$packages")"; just rs check $packages ;; - esac - - if [[ "$packages" == "ALL" ]] || just rs _wants-tokio-features "$packages"; then - just rs tokio-features - fi - - # moq-wasm's crate root is entirely `#![cfg(target_arch = "wasm32")]`, so - # however it got selected the host-target pass above compiled it to nothing - # and saw no errors in it. This wasm32 pass is what actually checks it. - # - # The alternation must list every package `wasm` compiles, not just moq-wasm: - # nothing in the workspace depends on moq-ffi, so an moq-ffi-only diff selects - # only itself and would never reach the gate that exists to cover it. moq-mux - # is selected by its dependents, none of which are moq-wasm. - if [[ "$packages" == "ALL" ]] || just rs _wants-wasm "$packages"; then - just rs wasm - fi - - # moq-video's and moq-audio's device code is behind `capture`, which is off - # by default, so the pass above never compiles it. Only nightly's - # `--all-features` did, and a break there lands on main. - if [[ "$packages" == "ALL" ]] || just rs _wants-capture "$packages"; then - just rs capture - fi - - # moq-relay's io_uring listener is behind `io-uring`, off by default, so the - # pass above never compiles it. `check --all` reaches it through `bench - # check`, but only for a diff under `bench/`; a relay change would otherwise - # land on the nightly. - if [[ "$packages" == "ALL" ]] || just rs _wants-uring "$packages"; then - just rs uring-check - fi +# Like `check`, but only the crates in a changed-file list plus their dependents. +check-changed $LIST: + sh/rs/select.sh check "$LIST" + +# Like `fix`, but only the crates in a changed-file list plus their dependents. +fix-changed $LIST: + sh/rs/select.sh fix "$LIST" + +# Like `test`, but only the crates in a changed-file list plus their dependents. +test-changed $LIST: + sh/rs/select.sh test "$LIST" # Compile moq-tokio by itself at the feature extremes and with each crypto provider. -# Its default-feature build is already part of the ordinary clippy pass above. tokio-features: - #!/usr/bin/env bash - set -euo pipefail - - cargo clippy --locked -p moq-tokio --no-default-features -- -D warnings - cargo clippy --locked -p moq-tokio --all-features -- -D warnings - cargo clippy --locked -p moq-tokio --no-default-features --features noq,aws-lc-rs -- -D warnings - cargo clippy --locked -p moq-tokio --no-default-features --features noq,ring -- -D warnings - - for feature in aws-lc-rs ring tcp uds websocket; do - cargo clippy --locked -p moq-tokio --no-default-features --features "$feature" -- -D warnings - done - - want='a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature' - if output=$(cargo check --locked -p moq-tokio --no-default-features --features noq 2>&1); then - echo "rs: moq-tokio with only noq should reject its missing crypto provider" >&2 - exit 1 - fi - if ! grep -qF "$want" <<< "$output"; then - echo "rs: moq-tokio with only noq should fail on: $want" >&2 - echo "$output" | grep -E '^error' >&2 || true - exit 1 - fi - -# Same as `fix`, but scoped the same way as `check-changed`. -fix-changed $FILES: - #!/usr/bin/env bash - set -euo pipefail - packages=$(just rs _select "$FILES") - case "$packages" in - "") echo "rs: no crates affected; skipping." ;; - ALL) just rs fix --workspace {{ no_fuzz }} ;; - *) echo "rs: fixing $(just rs _names "$packages")"; just rs fix $packages ;; - esac - - # Mirrors `check-changed`'s gate, alternation included: without this a wasm32 - # lint is never auto-fixed, and `check`'s wasm32 clippy pass fails on it later. - if [[ "$packages" == "ALL" ]] || just rs _wants-wasm "$packages"; then - just rs wasm-fix - fi - -# Runs the `#[cfg(target_os = "windows")]` code past the compiler: moq-video's -# Media Foundation capture/encode/decode and its D3D11 frames, which the Linux -# gate skips entirely. Cross-compiling can't stand in, because openh264-sys2 -# builds vendored C++ that needs an MSVC toolchain and openh264 is a -# non-optional dependency. -# -# Windows runners are throttled too hard for a per-PR gate, so nightly.yml runs -# this once a day instead. A break there lands on main rather than being caught -# in review, which is the same trade the other nightly gates make. -# -# Default features rather than `--all-features`: jemalloc doesn't build on -# MSVC, while moq-video's Linux-only `nvidia` default -# are already no-ops off Linux. moq-gst is excluded because it links -# GStreamer via pkg-config, which the Windows runner doesn't have. -# -# `moq-cli/play` and `moq-cli/capture` are named explicitly because they are -# off by default, and they are the only thing that compiles the cli's own -# device and render code. + sh/rs/tokio-features.sh + +# Runs the `#[cfg(target_os = "windows")]` code past the compiler, which the +# Linux gate skips; nightly.yml runs it on a Windows runner. Default features, +# since jemalloc doesn't build on MSVC, and moq-gst is excluded because the +# runner has no GStreamer. `moq-cli/play` and `moq-cli/capture` are the only +# thing that compiles the cli's device and render code. # Compile the whole workspace on a Windows host. Must run ON Windows. windows *args: @@ -524,111 +70,52 @@ windows *args: # Compile and test the io_uring relay path, which is off the default feature set. # Linux-only, and the tests below the 6.12 kernel floor skip themselves. -# -# `qlog` is a second axis, and the traces are what a debugging run depends on, -# so it gets a separate pass. uring *args: just rs uring-check {{ args }} cargo nextest run --locked -p moq-uring -p moq-relay --features moq-relay/io-uring --all-targets {{ args }} cargo clippy --locked -p moq-uring -p moq-relay --features moq-relay/io-uring,moq-relay/qlog --all-targets {{ args }} -- -D warnings cargo nextest run --locked -p moq-uring --features qlog --all-targets {{ args }} -# Compile the io_uring relay path, off the default feature set. `check-changed` -# runs this whenever moq-relay is selected, so a change that breaks the ring -# listener fails its own PR rather than landing on the nightly. +# Compile the io_uring relay path, off the default feature set. uring-check *args: cargo clippy --locked -p moq-uring -p moq-relay --features moq-relay/io-uring --all-targets {{ args }} -- -D warnings -# Runs the `#[cfg(target_os = "macos")]` code past the compiler: moq-video's -# VideoToolbox encode/decode and its ScreenCaptureKit / AVFoundation capture, -# plus moq-audio's ScreenCaptureKit system audio and TCC permission pre-check. -# -# Same trade as `windows`: nightly.yml runs this once a day rather than gating a -# merge on a Mac runner. The moq-video half also has a release-time backstop, -# since libmoq's `libmoq-v*` tag build compiles it on Apple Silicon. The -# moq-audio half has none: libmoq and moq-ffi take it codecs-only, so no release -# build compiles its capture backend and this recipe is the only thing that does. -# -# Scoped to those two crates instead of the workspace, because they hold all -# the Apple-gated code the Linux gate misses. moq-ffi (and through it the -# Swift/Kotlin/Go wrappers) already compiles on macOS in swift.yml. -# `--all-features` also compiles the off-by-default cpal hosts and PipeWire -# capture, which are Linux-only and no-ops here, so it costs nothing extra. +# moq-video's VideoToolbox/ScreenCaptureKit/AVFoundation code and moq-audio's +# ScreenCaptureKit capture, which nightly.yml compiles on a Mac runner. # Compile the Apple-only code paths. Must run ON macOS. macos *args: cargo check --locked -p moq-video -p moq-audio --all-targets --all-features {{ args }} -# The Linux device code: moq-video's V4L2 and X11 capture and moq-audio's cpal -# capture, all behind the off-by-default `capture` feature. `check-changed` runs -# this whenever either crate is selected. The per-crate `--features` spelling -# keeps it to those two instead of widening what their dependencies compile. -# `--all-features` would also build the bindgen codecs, which nightly covers. - # Compile and lint the capture feature, which the default check skips. capture *args: cargo clippy --locked -p moq-video -p moq-audio --all-targets --features moq-video/capture,moq-audio/capture {{ args }} -- -D warnings -# Same idea as `windows`/`macos`, for Android: moq-video's MediaCodec encoder and -# decoder, its `Surface::HardwareBuffer` variant, and the `frame::android` module -# are all `#[cfg(target_os = "android")]`, so a host-target check compiles none of -# them and every error in them stays invisible. -# -# Unlike `windows` and `macos` this needs no special host: the Android targets -# cross-compile from Linux in about half a minute. It needs an NDK, which -# `cargo-ndk` finds through `ANDROID_NDK_HOME` or the standard SDK layout. The -# dev shell provides neither, so android.yml runs this outside Nix on a GitHub -# runner, whose image ships an NDK, for every PR that touches moq-video or -# anything it is built from. -# -# arm64-v8a alone: it is the only ABI anything here ships, and the code is not -# ABI-specific, so the other two would compile the same source twice more. -# -# `check` rather than a `-D warnings` clippy, matching `windows` and `macos`: -# the capture module's plumbing is dead on a target with no capture backend, so -# a denying lint would fail on code this recipe is not here to police. +# moq-video's MediaCodec code is `#[cfg(target_os = "android")]`. Needs an NDK, +# which the dev shell lacks, so android.yml runs it outside Nix. `check` rather +# than a denying clippy: the capture plumbing is dead on a target without a +# capture backend. # Compile the Android backends, which the host-target check skips. android *args: cargo ndk -t arm64-v8a --platform 26 check --locked -p moq-video --all-targets {{ args }} -# Same idea as `windows`/`macos`, for the browser: moq-wasm's whole crate root is -# `#![cfg(target_arch = "wasm32")]`, so a host-target `cargo check --workspace` -# compiles an empty crate and every error in it stays invisible. Only a wasm32 -# build sees the code. -# -# Unlike those two this needs no special host, so `ci` runs it on every Rust PR. -# The wasm32 target and the `getrandom`/`web-sys` cfg flags come from the Nix dev -# shell and `.cargo/config.toml`. Not to be confused with the root `just wasm`, -# which builds the shippable `@moq/wasm` package (wasm-bindgen, release profile); -# this is the compile gate. - -# `moq-mux` and `moq-ffi` ride along in the same invocation on purpose: `moq-mux` -# needs `getrandom`'s `wasm_js` backend that only its siblings declare, and feature -# unification only happens within one cargo invocation. Splitting them fails on -# getrandom. `--lib` rather than `--all-targets` because moq-mux's tests use tokio's -# native timers and moq-ffi's drive a native relay; moq-wasm has no test targets, so -# it loses no coverage. +# moq-wasm's crate root is `#![cfg(target_arch = "wasm32")]`, so only a wasm32 +# build sees the code. Not to be confused with `just js wasm`, which builds the +# shippable `@moq/wasm` package. `moq-mux` and `moq-ffi` share the invocation +# because moq-mux needs the getrandom backend only its siblings declare, and +# features unify only within one cargo invocation. # Compile and lint the browser/WASM bindings, which the host-target check skips. wasm *args: cargo clippy --locked -p moq-wasm -p moq-mux -p moq-ffi --target wasm32-unknown-unknown --lib {{ args }} -- -D warnings -# The `fix` counterpart to `wasm`. `cargo fmt` is scoped rather than `--all` -# because this runs alongside `fix`, which already formats everything else. - # Auto-fix the browser/WASM bindings. wasm-fix *args: cargo clippy --locked --fix --allow-staged --allow-dirty -p moq-wasm -p moq-mux -p moq-ffi --target wasm32-unknown-unknown --lib {{ args }} cargo fmt -p moq-wasm -p moq-mux -p moq-ffi -# Dependency advisories plus the license/ban policy. Time-based rather than -# diff-based: an advisory lands without this repo changing, so a per-PR run both -# fires on PRs that cannot affect it and stays silent on the days nobody pushes. -# Runs nightly instead (.github/workflows/nightly.yml), which also keeps a fresh -# advisory from blocking an unrelated PR mid-review. - -# Audit dependencies against deny.toml. +# Audit dependencies against deny.toml. Nightly: advisories land without a diff. audit: cargo deny check --show-stats @@ -636,96 +123,16 @@ audit: relay-minimal: cargo nextest run --locked -p moq-relay --no-default-features --lib -E 'test(workers_require_a_quic_backend)' -# libmoq's C fixtures (`rs/libmoq/c-tests/*.c`) link `libmoq.a` the way an -# embedder does: an external `cc` against the generated `moq.h`, plus the native -# libraries from `rs/libmoq/native-libs` that cargo can't inject into a link it -# doesn't drive. Each fixture pins part of the C ABI at compile time and -# exercises it at runtime, so it has to run, not just compile. -# -# Outside nextest on purpose. The staticlib link alone takes ~2 minutes, past the -# hang budget in `.config/nextest.toml` once the rest of the workspace builds -# beside it, and `rs/CLAUDE.md` says to make a slow test fast, not to raise the -# ceiling. obs.yml runs this on every PR that touches rs/libmoq, right after it -# has built the same debug libmoq.a for the plugin, and nightly.yml covers the -# paths that filter misses. -# -# Unix only: linking a Rust staticlib from MSVC spells the flags and the -# archive (`moq.lib`) differently, and this recipe doesn't model that. +# Outside nextest because the staticlib link alone outlasts its hang budget. # Compile and run libmoq's C fixtures against libmoq.a. Must run on Unix. c-tests: - #!/usr/bin/env bash - set -euo pipefail - cc="${CC:-cc}" - if ! command -v "$cc" >/dev/null; then - echo "no C compiler at '$cc'" >&2 - exit 1 - fi - - cargo build --locked -p libmoq - - # Ask cargo where it put the staticlib rather than assuming target/debug: - # CARGO_TARGET_DIR and a configured build target (`target//debug`) - # both move it, and guessing wrong would link a stale copy or nothing. - # Replays the build above from cache, so it costs a process, not a compile. - lib=$(cargo build --locked -p libmoq --message-format=json | - jq -r 'select(.reason == "compiler-artifact") | .filenames[] | select(endswith("/libmoq.a"))' | - tail -1) - if [ -z "$lib" ]; then - echo "cargo reported no libmoq.a for libmoq" >&2 - exit 1 - fi - # The header sits beside the profile directory: rs/libmoq/build.rs writes it - # to `/include`, one level above `//libmoq.a`. - profile=$(dirname "$lib") - include="$(dirname "$profile")/include" - if [ ! -f "$include/moq.h" ]; then - echo "$include/moq.h missing after 'cargo build --locked -p libmoq'" >&2 - exit 1 - fi - - # Same list build.rs (moq.pc), CMakeLists.txt, and test/interop/interop.sh read: - # `framework:Foo` is a linker framework flag, anything else a plain library. - case "$(uname -s)" in - Darwin) native_libs=rs/libmoq/native-libs/apple.txt ;; - *) native_libs=rs/libmoq/native-libs/linux.txt ;; - esac - libs=() - while read -r entry; do - case "$entry" in - '' | '#'*) continue ;; - framework:*) libs+=(-framework "${entry#framework:}") ;; - *) libs+=("-l$entry") ;; - esac - done <"$native_libs" - - out=$(mktemp -d) - trap 'rm -rf "$out"' EXIT - for source in rs/libmoq/c-tests/*.c; do - bin="$out/$(basename "$source" .c)" - "$cc" "$source" -I"$include" -L"$profile" -lmoq "${libs[@]}" -o "$bin" - echo "running $source" - "$bin" - done - -# The feature permutations `check` leaves out. Each is a full workspace compile -# at a feature set that shares almost no artifacts with the default one (~6min -# for the all-features clippy alone), which is why this is nightly rather than -# part of the per-PR gate. -# -# `--all-features` is the only thing that compiles moq-cli's play/capture, the -# moq-video and moq-audio device features, the Linux bindgen codecs (vaapi, -# v4l2), PipeWire, and jemalloc. `--no-default-features` is -# the only thing that compiles the `#[cfg(not(feature = ...))]` arms. -# -# Cargo unifies features across a workspace selection. Check each package alone -# to catch missing dependencies in minimal builds. The nightly sweep adds each -# feature to aws-lc-rs; tokio-features checks provider-free transports, standalone -# providers, and the intentional refusal of rustls QUIC without a provider. -# -# Both carry `--all-targets`. A `#[cfg(test)]` module leans on whatever the -# default features happened to import, so a test that only compiles by accident -# is invisible to a lib-only check. + sh/rs/c-tests.sh + +# The feature permutations `check` leaves out, nightly. `--all-features` is the +# only thing that compiles moq-cli's play/capture, the device features, the +# Linux bindgen codecs, PipeWire, and jemalloc; `--no-default-features` the only +# thing that compiles the `#[cfg(not(feature = ...))]` arms. # Compile the workspace at the feature extremes. features: @@ -743,162 +150,47 @@ features: # Compile the media crates at supported feature boundaries and prove expensive # native/render dependencies are absent when their features are excluded. media-features: - rs/scripts/media-features.sh + sh/rs/media-features.sh -# nextest, not `cargo test`, so a wedged test is killed instead of pinning a core -# until someone notices (`.config/nextest.toml` sets the timeout). The trade is -# doctests, which nextest does not execute; `just rs doctest` covers those. -# -# Takes cargo arguments, so `just rs test -p moq-net` works. `test-changed` is -# the diff-scoped variant that `just test` dispatches to. -# -# clippy-driver as the compiler wrapper is rustc plus the lint passes, so this -# one compilation both builds the test binaries and reports clippy findings. -# `cargo clippy` can't be reused here: it asks cargo for metadata only, and a -# metadata build and a codegen build have different fingerprints, so running -# both compiles the dependency tree twice. Linting here is free, which makes -# `just test` a single-compile substitute for `check` + `test` on a laptop. -# -# RUSTC_WORKSPACE_WRAPPER, not RUSTC_WRAPPER, for two reasons. Cargo folds it -# into the fingerprint, so a crate already built by a plain `cargo test` is -# rebuilt through clippy rather than silently reported as fresh with no -# findings. And it applies to workspace crates only, so dependencies keep the -# artifacts they share with plain cargo instead of duplicating the whole tree. -# It is also what `cargo clippy` sets internally. -# -# Findings stay warnings rather than errors. `check` is the gate (it runs -# `clippy -D warnings`), and a hard failure here would block running the tests -# over an unused import while you are mid-edit. -# -# No RUSTFLAGS: it is part of cargo's fingerprint too, so setting it here would -# split the artifact cache against every other cargo invocation, including CI's -# warm cache. Skips the wrapper entirely when clippy-driver is absent. +# nextest kills a wedged test (`.config/nextest.toml` sets the timeout), but +# skips doctests; `just rs doctest` covers those. clippy-driver as the workspace +# wrapper reports lint findings from the same compile, as warnings: `check` is +# the gate. It is part of cargo's fingerprint, and RUSTFLAGS would split the +# artifact cache, which is why neither is set any other way. # Run the test suite, reporting clippy findings from the same compile. test *args: RUSTC_WORKSPACE_WRAPPER="$(command -v clippy-driver || true)" cargo nextest run --locked --all-targets {{ args }} -# Exercise the Vulkan producer -> CUDA image contract, the GPU color conversion -# and resize, and NVENC encoding of the result on Linux/NVIDIA hardware. The -# driver directory is added explicitly because the Nix shell's dynamic loader -# path does not include Ubuntu's host driver directory. +# Exercise Vulkan -> CUDA -> NVENC on Linux/NVIDIA hardware. vulkan-cuda: - #!/usr/bin/env bash - set -euo pipefail - driver=$(/usr/sbin/ldconfig -p | awk '/libcuda\.so\.1/{print $NF; exit}') - if [[ -z "$driver" ]]; then - echo "libcuda.so.1 is not installed" >&2 - exit 1 - fi - driver_dir=$(dirname "$driver") - export LD_LIBRARY_PATH="$driver_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - cargo nextest run --locked -p moq-video --run-ignored only -E 'test(/^frame::(vulkan|cuda)::tests::vulkan_cuda_/)' - -# Same selection as `check-changed`: compiling a crate's test binaries is the -# expensive half of a test run, so scoping matters more here than anywhere else. -# -# `--no-tests=pass` only on the scoped branch. A selection can legitimately hold -# nothing testable (moq-wasm has no host-target tests, since its crate root is -# `#![cfg(target_arch = "wasm32")]`), and nextest exits 4 on that by default. -# The unscoped branch keeps the default, where finding no tests really is wrong. - -# Like `test`, but only the crates a diff touches plus their dependents. -test-changed $FILES: - #!/usr/bin/env bash - set -euo pipefail - packages=$(just rs _select "$FILES") - case "$packages" in - "") echo "rs: no crates affected; skipping tests." ;; - ALL) just rs test --workspace {{ no_fuzz }} ;; - *) echo "rs: testing $(just rs _names "$packages")"; just rs test --no-tests=pass $packages ;; - esac - - if [[ "$packages" == "ALL" ]] || just rs _names "$packages" | grep -qw moq-relay; then - just rs relay-minimal - fi + sh/rs/vulkan-cuda.sh # Compile and run the `/// ```` examples, which nextest skips. doctest *args: cargo test --locked --doc {{ args }} +# `--cfg loom` swaps kio's primitives for loom's, so it rebuilds the world and +# stays out of `check`. `--release` because a model check runs the body once +# per interleaving (411s -> 51s); the assertions are plain `assert!`. kio's own +# unit tests are `cfg(not(loom))`. + # Permutation-test the concurrent handoffs with loom. -# -# `--cfg loom` swaps kio's Mutex/atomics for loom's instrumented ones, so it -# rebuilds the world and can't share artifacts with a normal `cargo test`, so -# it stays separate from `check`/`ci`. -# -# `--release` because a model check runs the body once per interleaving, so the -# optimizer pays for itself many times over: 411s -> 51s across the two suites. -# Our assertions are plain `assert!`, not `debug_assert!`, so nothing is -# compiled out (verified by re-running the mutation check in release). -# -# The search is deliberately unbounded (no `preemption_bound`), so it covers -# every interleaving rather than just the short ones. -# -# kio's own unit tests are `cfg(not(loom))`: they'd construct loom primitives -# outside `loom::model`, which panics. loom *args: RUSTFLAGS="--cfg loom" cargo test --locked --release -p kio --lib loom:: {{ args }} RUSTFLAGS="--cfg loom" cargo test --locked --release -p moq-net --test loom {{ args }} -# Coverage-guided fuzzing of the moq-net wire codecs (libFuzzer, via cargo-fuzz). -# -# The one recipe that steps off the pinned toolchain: libFuzzer needs `-Zsanitizer`, -# which is nightly-only, so this stays out of `check` and `test` the same way `loom` -# does. What CI gets instead is the replay: the target bodies live in moq-net's hidden -# `fuzz` module, and its own tests run the generated corpus plus everything committed -# under `fuzz/regressions/` on stable. A crash found here becomes a regression test -# nobody has to install cargo-fuzz to run. -# -# Seeds are regenerated first so the corpus follows the dispatch rather than a stale -# run, and libFuzzer writes what it discovers to the FIRST corpus directory, which is -# why the generated seeds are passed second and stay read-only. -# -# TARGET is lite, ietf, varint, or path. Arguments pass through, so -# `just rs fuzz lite -- -max_total_time=300` bounds a run. See rs/moq-net/fuzz/README.md. -fuzz target *args: - #!/usr/bin/env bash - set -euo pipefail - - if ! command -v cargo-fuzz > /dev/null 2>&1; then - echo "rs: cargo-fuzz is missing: cargo install --locked cargo-fuzz" >&2 - exit 1 - fi - - if ! rustup run nightly rustc --version > /dev/null 2>&1; then - echo "rs: the nightly toolchain is missing: rustup toolchain install nightly" >&2 - exit 1 - fi - - cargo run --locked -q -p moq-net --features fuzz --example fuzz-seeds -- rs/moq-net/fuzz/seeds - mkdir -p rs/moq-net/fuzz/corpus/{{ target }} +# Needs nightly and cargo-fuzz; CI replays the corpus on stable through +# moq-net's own tests instead. See rs/moq-net/fuzz/README.md. - # Nightly by PATH rather than by `cargo +nightly`: the dev shell puts a - # Nix-provided cargo ahead of the rustup shim, so the `+` form is not - # understood, and cargo-fuzz shells out to a bare `cargo` and `rustc` anyway. - # Both have to be the nightly ones or the sanitizer flags are rejected. - export PATH="$(rustup run nightly rustc --print sysroot)/bin:$PATH" - - cargo fuzz run --fuzz-dir rs/moq-net/fuzz {{ target }} \ - rs/moq-net/fuzz/corpus/{{ target }} rs/moq-net/fuzz/seeds/{{ target }} {{ args }} +# Fuzz the moq-net wire codecs: `just rs fuzz lite -- -max_total_time=300`. +fuzz target *args: + sh/rs/fuzz.sh {{ target }} {{ args }} +# Build the default workspace members. build: cargo build --locked -# Run a moq-bench preset against a relay, e.g. `just rs bench chat https://relay.example.com`. -# Extra args pass through, so `--connections 500 --output stats.jsonl` work as-is. -bench preset url *args: - cargo run --release -p moq-bench -- --file 'rs/moq-bench/config/{{ preset }}.toml' --connect '{{ url }}' {{ args }} - -# Measure the Linux Tokio/epoll UDP syscall path. Extra args pass to Criterion, -# e.g. `just rs bench-udp-tokio --quick`. -bench-udp-tokio *args: - cargo bench -p moq-uring --bench udp_tokio -- {{ args }} - -# Measure the Linux io_uring UDP completion path with multishot recvmsg. -bench-udp-uring *args: - cargo bench -p moq-uring --bench udp_uring -- {{ args }} - # Run the comparable epoll and io_uring UDP matrices. bench-udp *args: cargo bench -p moq-uring --bench udp_tokio -- {{ args }} @@ -915,71 +207,11 @@ bench-echo *args: bench-session *args: cargo bench -p moq-uring --bench session_lite -- {{ args }} -# Sample a process's CPU/memory/context switches on this host (run it where the -# relay runs; it only reads /proc). E.g. `just rs bench-host --name moq-relay`. -bench-host *args: - cargo run --release -p moq-bench --bin moq-bench-host -- {{ args }} - -# Remove the Rust target directory. -clean: - cargo clean - -# Check semver compatibility against crates.io (default-members only). -semver: - cargo semver-checks check-release - -# Update versions and changelogs via release-plz. -bump: - release-plz update - # Create release PRs and publish crates via release-plz. release: release-plz release-pr --git-token "$GITHUB_TOKEN" release-plz release --git-token "$GITHUB_TOKEN" -update: - cargo update - cargo upgrade --incompatible - -# Build a .deb or .rpm for one of the Rust binaries locally. nfpm comes -# from the flake's dev shell (`nix develop`). For .rpm produced this way, -# the linkage matches the host's glibc; CI uses an AlmaLinux 9 container -# to produce broadly compatible artifacts instead. -# -# Examples: -# just rs package moq-relay deb - -# just rs package moq-cli rpm +# Build a .deb or .rpm locally: `just rs package moq-relay deb`. package crate packager: - #!/usr/bin/env bash - set -euo pipefail - case "{{ crate }}" in - moq-relay) bin=moq-relay ;; - moq-cli) bin=moq ;; - *) echo "Unknown crate: {{ crate }} (use moq-relay or moq-cli)" >&2; exit 1 ;; - esac - case "{{ packager }}" in - deb) - if command -v dpkg >/dev/null 2>&1; then - arch=$(dpkg --print-architecture) - else - case "$(uname -m)" in - x86_64) arch=amd64 ;; - aarch64|arm64) arch=arm64 ;; - *) echo "Cannot infer deb arch from host $(uname -m)" >&2; exit 1 ;; - esac - fi - ;; - rpm) arch=$(uname -m) ;; - *) echo "Unknown packager: {{ packager }} (use deb or rpm)" >&2; exit 1 ;; - esac - version=$(grep -m1 '^version' rs/{{ crate }}/Cargo.toml | sed 's/.*"\(.*\)".*/\1/') - cargo build --locked --release -p {{ crate }} - mkdir -p dist - VERSION="$version" ARCH="$arch" BINARY_PATH="target/release/$bin" \ - rs/scripts/package-nfpm.sh packaging/{{ crate }}/nfpm.yaml {{ packager }} dist/ - if [[ "{{ packager }}" == deb && -f "packaging/{{ crate }}/transition.yaml" ]]; then - VERSION="$version" ARCH="$arch" \ - rs/scripts/package-nfpm.sh packaging/{{ crate }}/transition.yaml deb dist/ - fi - ls -1 dist/ + sh/rs/package.sh {{ crate }} {{ packager }} diff --git a/rs/moq-gst/package.sh b/rs/moq-gst/package.sh index b48f392eb2..85ff0731b6 100755 --- a/rs/moq-gst/package.sh +++ b/rs/moq-gst/package.sh @@ -145,7 +145,7 @@ mkdir -p "$OUTPUT_DIR" echo ">> Running nfpm ($PACKAGER)..." export VERSION ARCH="$PKG_ARCH" PKG_NAME PLUGIN_PATH="$BUILT_SO" PLUGIN_DIR -"$WORKSPACE_DIR/rs/scripts/package-nfpm.sh" \ +"$WORKSPACE_DIR/sh/rs/package-nfpm.sh" \ "$WORKSPACE_DIR/packaging/moq-gst/nfpm.yaml" "$PACKAGER" "$OUTPUT_DIR/" echo ">> Done. Artifacts in: $OUTPUT_DIR" diff --git a/rs/moq-gst/scrub.sh b/rs/moq-gst/scrub.sh index 5d4699deee..09ec56ad34 100755 --- a/rs/moq-gst/scrub.sh +++ b/rs/moq-gst/scrub.sh @@ -40,7 +40,7 @@ scrub_macos() { # Framework path covers the official .pkg installer, and /usr/lib lets # dyld resolve system libs (libiconv, libc++) via the dyld_shared_cache at # @rpath substitution time. - "$SCRIPT_DIR/../scripts/scrub-macho.sh" "$dylib" \ + "$SCRIPT_DIR/../../sh/rs/scrub-macho.sh" "$dylib" \ /opt/homebrew/lib \ /Library/Frameworks/GStreamer.framework/Libraries \ /usr/lib diff --git a/rs/moq-wasm/README.md b/rs/moq-wasm/README.md index 7a0609c710..34a90b2ccc 100644 --- a/rs/moq-wasm/README.md +++ b/rs/moq-wasm/README.md @@ -7,7 +7,7 @@ wire implementation in `@moq/net` (~10k LOC) with the canonical Rust one, so the protocol lives in exactly one place. This crate is the Rust half; the generated JS package is -[`@moq/wasm`](../../js/wasm) (`just wasm` builds it). It is **not** the same as +[`@moq/wasm`](../../js/wasm) (`just js wasm` builds it). It is **not** the same as `moq-ffi`: that crate uses UniFFI, which targets the C ABI (Kotlin/Swift/Python/ Go). Browsers need `wasm-bindgen`, so this is a separate sibling crate. (For *React Native* JS, `uniffi-bindgen-react-native` can reuse `moq-ffi` directly; @@ -66,7 +66,7 @@ dependency is commented out in `Cargo.toml` until then. ## Building -`just wasm` (from the repo root) does everything: builds for wasm and runs +`just js wasm` (from the repo root) does everything: builds for wasm and runs `wasm-bindgen` (web target) into `js/wasm/dist`. The wasm target, the cfg flags (`getrandom` wasm backend + web-sys unstable WebTransport APIs), and the `wasm-bindgen-cli` tool come from `.cargo/config.toml` and the Nix dev shell. diff --git a/rs/scripts/doc-names.jq b/rs/scripts/doc-names.jq deleted file mode 100644 index 5dfad17c57..0000000000 --- a/rs/scripts/doc-names.jq +++ /dev/null @@ -1,31 +0,0 @@ -# Find workspace targets that would render to the same `target/doc/`. -# -# `cargo doc` writes each documented target to a directory named after the target with -# dashes folded to underscores, so two targets that differ only there collide: both -# rustdoc processes race to clean it, and the warning escalates to a hard error -# depending on which wins. -# -# Reads `cargo metadata --no-deps` and prints one line per colliding group. -[ .packages[] - | . as $pkg - # Cargo skips a binary that would land on the same directory as a library in its own - # package, documenting only the library, so such a pair never actually collides. - # Compared after folding, matching cargo: a `foo_bar` lib beside a `foo-bar` bin is - # suppressed too (verified against cargo 1.95). Only cross-package pairs collide. - | [ $pkg.targets[] - | select([.kind[]] | any(. == "lib" or . == "rlib" or . == "dylib" - or . == "cdylib" or . == "staticlib" or . == "proc-macro")) - | (.name | gsub("-"; "_")) - ] as $libs - | $pkg.targets[] - # `doc = false` in the manifest, which is how a reported pair gets resolved. - | select(.doc) - | select([.kind[]] | any(. == "lib" or . == "rlib" or . == "dylib" - or . == "cdylib" or . == "staticlib" or . == "proc-macro" or . == "bin")) - | select((([.kind[]] | any(. == "bin")) and ((.name | gsub("-"; "_")) | IN($libs[]))) | not) - | { dir: (.name | gsub("-"; "_")), package: $pkg.name, target: .name, kind: .kind[0] } -] -| group_by(.dir) -| map(select(length > 1)) -| .[] -| " target/doc/\(.[0].dir) <- " + (map("\(.package) [\(.kind)] \(.target)") | join(" and ")) diff --git a/rs/scripts/package-nfpm.test.sh b/rs/scripts/package-nfpm.test.sh deleted file mode 100755 index 5582669426..0000000000 --- a/rs/scripts/package-nfpm.test.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if ! command -v envsubst >/dev/null 2>&1; then - exit 0 -fi - -scratch=$(mktemp -d) -trap 'rm -rf "$scratch"' EXIT -mkdir -p "$scratch/bin" "$scratch/capture" - -cat >"$scratch/bin/nfpm" <<'SH' -#!/usr/bin/env bash -set -euo pipefail - -while (($#)); do - case "$1" in - --config) - config=$2 - shift 2 - ;; - --packager) - printf '%s\n' "$2" >"$NFPM_CAPTURE/packager" - shift 2 - ;; - --target) - printf '%s\n' "$2" >"$NFPM_CAPTURE/target" - shift 2 - ;; - *) shift ;; - esac -done - -cp "$config" "$NFPM_CAPTURE/config.yaml" -SH -chmod +x "$scratch/bin/nfpm" - -cat >"$scratch/nfpm.yaml" <<'YAML' -name: test -version: ${VERSION} -arch: ${ARCH} -contents: - - src: ${BINARY_PATH} - dst: /usr/bin/test -YAML - -PATH="$scratch/bin:$PATH" \ - NFPM_CAPTURE="$scratch/capture" \ - VERSION=1.2.3 \ - ARCH=amd64 \ - BINARY_PATH=target/release/test \ - rs/scripts/package-nfpm.sh "$scratch/nfpm.yaml" deb "$scratch/dist" - -grep -qx 'version: 1.2.3' "$scratch/capture/config.yaml" -grep -qx 'arch: amd64' "$scratch/capture/config.yaml" -grep -qx ' - src: target/release/test' "$scratch/capture/config.yaml" -grep -qx 'deb' "$scratch/capture/packager" -grep -qx "$scratch/dist" "$scratch/capture/target" - -if PATH="$scratch/bin:$PATH" \ - NFPM_CAPTURE="$scratch/capture" \ - VERSION=1.2.3 \ - ARCH=amd64 \ - rs/scripts/package-nfpm.sh "$scratch/nfpm.yaml" deb "$scratch/dist" 2>"$scratch/missing.log"; then - echo "package-nfpm accepted a missing BINARY_PATH" >&2 - exit 1 -fi -grep -qx "Missing environment variable in $scratch/nfpm.yaml: BINARY_PATH" "$scratch/missing.log" diff --git a/sh/clean.sh b/sh/clean.sh new file mode 100755 index 0000000000..24f495cafe --- /dev/null +++ b/sh/clean.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Delete build artifacts and caches: sh/clean.sh [here|all] +# +# Only this checkout by default. Agent worktrees each carry their own +# artifacts, and another agent is usually building in one: `cargo clean` under +# a running build fails it, and there is no telling a finished worktree from a +# busy one from out here. `all` is the explicit opt-in for an idle machine. +# +# Source is never touched, dirty or untracked: this deletes build output, not +# work. Nothing here reaches a machine-wide store (Nix, cargo/bun/uv home +# caches), because those are shared with every other checkout. +set -euo pipefail + +scope=${1:-here} +case "$scope" in + here | all) ;; + *) + echo "usage: sh/clean.sh [here|all]" >&2 + exit 2 + ;; +esac + +cd "$(git rev-parse --show-toplevel)" + +# Rust. +cargo clean + +# JS: the bun workspace spans the repo, so sweep from the root. +find . -name .claude -prune -o \ + -type d \( -name node_modules -o -name dist -o -name out -o -name pkg \) \ + -prune -exec rm -rf {} + +find . -name .claude -prune -o -type f -name '*.tsbuildinfo' -exec rm -f {} + + +# Python: the venv, release dist, bytecode, and the bindings maturin drops in. +rm -rf .venv py/.venv py/dist py/moq-ffi/moq_ffi/_uniffi +find py -name .claude -prune -o -type d -name __pycache__ -prune -exec rm -rf {} + +find py -name .claude -prune -o -type f -name '*.pyc' -exec rm -f {} + + +# Kotlin: gradle output plus the bindings and native libs sh/kt generates. +find kt -type d \( -name build -o -name .gradle -o -name .kotlin \) -prune -exec rm -rf {} + +rm -rf kt/local.properties \ + kt/moq-ffi/src/jvmAndAndroidMain/kotlin/uniffi \ + kt/moq-ffi/src/jvmMain/resources \ + kt/moq-ffi/src/androidMain/jniLibs + +# Swift: SwiftPM output plus the XCFramework and bindings sh/swift lays down. +rm -rf swift/.build swift/.swiftpm swift/Package.resolved \ + swift/Sources/MoqFFI/Generated.swift swift/MoqFFI.xcframework + +# Go: the gitignored bindings, native libs, and vendoring in go/. +rm -rf go/ffi/moq/moq.go go/ffi/moq/moq.h go/ffi/moq/lib go/ffi/go.sum \ + go/ffi/vendor go/wrapper/go.sum go/wrapper/vendor + +# Dart: tool state and build output. +rm -rf dart/moq/.dart_tool dart/moq_ffi/.dart_tool dart/moq/build dart/moq_ffi/build + +# Rendered drafts and their reference cache. +rm -rf drafts/draft-*.xml drafts/draft-*.txt drafts/draft-*.html drafts/.refcache + +# Caches no one language owns: nix build result, direnv, wrangler. +rm -rf result .direnv +find . -name .claude -prune -o -type d -name .wrangler -prune -exec rm -rf {} + + +# Worktrees don't nest, so this recurses exactly one level. Tolerate stale +# worktrees on branches that predate this script. +if [[ "$scope" == all ]]; then + for wt in .claude/worktrees/*/; do + [[ -f "${wt}justfile" ]] || continue + echo "==> cleaning ${wt}" + (cd "$wt" && just clean) || echo " (skipped: just clean failed in ${wt})" + done +fi diff --git a/dart/scripts/check.sh b/sh/dart/check.sh similarity index 88% rename from dart/scripts/check.sh rename to sh/dart/check.sh index 5e658e9b18..eaa4f579be 100755 --- a/dart/scripts/check.sh +++ b/sh/dart/check.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../dart" && pwd) ACTION=check if [[ "${1:-}" == --fix ]]; then @@ -9,13 +9,6 @@ if [[ "${1:-}" == --fix ]]; then shift fi -# `just check` passes the whole changed-file list, so match the scope anywhere in -# it rather than only at the start. -files="${1:-}" -if [[ -n "$files" ]] && ! grep -qE '(^|[[:space:]])(dart/|rs/moq-ffi/)' <<<"$files"; then - exit 0 -fi - for tool in cargo dart uniffi_bindgen_dart; do if ! command -v "$tool" >/dev/null 2>&1; then echo "dart check: skipping, $tool is not installed" >&2 @@ -58,7 +51,7 @@ done generated=$(mktemp -d) trap 'rm -rf "$generated"' EXIT -"$DART_DIR/scripts/generate.sh" "$generated" +"$DART_DIR/../sh/dart/generate.sh" "$generated" for file in moq.dart uniffi_runtime.dart; do if ! diff -u -B "$DART_DIR/moq_ffi/lib/src/$file" "$generated/$file"; then diff --git a/dart/scripts/generate.sh b/sh/dart/generate.sh similarity index 96% rename from dart/scripts/generate.sh rename to sh/dart/generate.sh index bbbe65ad97..7a2c9c24fc 100755 --- a/dart/scripts/generate.sh +++ b/sh/dart/generate.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../dart" && pwd) WORKSPACE_DIR=$(cd "$DART_DIR/.." && pwd) OUTPUT_DIR="${1:-$DART_DIR/moq_ffi/lib/src}" diff --git a/dart/scripts/package.sh b/sh/dart/package.sh similarity index 97% rename from dart/scripts/package.sh rename to sh/dart/package.sh index 21c4d7b135..cb98477695 100755 --- a/dart/scripts/package.sh +++ b/sh/dart/package.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../dart" && pwd) PACKAGE="" VERSION="" OUTPUT_DIR="dist" diff --git a/dart/scripts/publish.sh b/sh/dart/publish.sh similarity index 84% rename from dart/scripts/publish.sh rename to sh/dart/publish.sh index 6a6caaa50e..1348e0ec64 100755 --- a/dart/scripts/publish.sh +++ b/sh/dart/publish.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash set -euo pipefail -DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../dart" && pwd) PACKAGE="${1:?usage: publish.sh }" VERSION="${2:?usage: publish.sh }" OUTPUT=$(mktemp -d) trap 'rm -rf "$OUTPUT"' EXIT package_dir=$( - "$DART_DIR/scripts/package.sh" \ + "$DART_DIR/../sh/dart/package.sh" \ --package "$PACKAGE" \ --version "$VERSION" \ --output "$OUTPUT" diff --git a/sh/dispatch.sh b/sh/dispatch.sh new file mode 100755 index 0000000000..7cfcc9d6a0 --- /dev/null +++ b/sh/dispatch.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +# Run `just check`, `just fix`, or `just test` over what the branch changed. +# +# Usage: sh/dispatch.sh check|fix|test [BASE|--all] +# +# The branch diff is resolved once and matched against the impact map below, +# the one place that says which paths put which module in scope. CI runs the +# same thing with MOQ_STRICT=1, so there is no second definition of "checked". +set -euo pipefail + +usage="usage: sh/dispatch.sh check|fix|test [BASE|--all]" +action=${1:?$usage} +base=${2:-} +case "$action" in + check | fix | test) ;; + *) + echo "$usage" >&2 + exit 2 + ;; +esac + +cd "$(git rev-parse --show-toplevel)" + +changed=$(mktemp) +trap 'rm -f "$changed"' EXIT + +all= +if [[ "$base" == --all ]]; then + all=1 +else + # BASE: the argument, then $GITHUB_BASE_REF (a PR checkout has no + # upstream), then the upstream, then origin/main. `git push -u` points the + # upstream at the branch's own remote copy, which would diff HEAD against + # itself, so that case falls through to origin/main. + if [[ -z "$base" && -n "${GITHUB_BASE_REF:-}" ]]; then + base="origin/$GITHUB_BASE_REF" + fi + if [[ -z "$base" ]]; then + base=$(git rev-parse --abbrev-ref '@{upstream}' 2>/dev/null || true) + if [[ -z "$base" || "$base" == */"$(git branch --show-current)" ]]; then + base=origin/main + fi + fi + merge_base=$(git merge-base "$base" HEAD) || { + echo "error: cannot resolve merge-base against $base (is full history fetched?)" >&2 + exit 1 + } + echo "$action: base $base" >&2 + + # Untracked files count too: a brand new crate or module is the whole change. + { + git diff --name-only "$merge_base" + git ls-files --others --exclude-standard + } | sort -u >"$changed" + + # These files hold the map and the recipes that call it, and match no + # module, so a change to them would otherwise validate none of it. + if grep -qE '^(justfile|test/justfile|sh/dispatch\.sh)$' "$changed"; then + echo "$action: root orchestration changed; running everything." >&2 + all=1 + fi +fi + +# The impact map: a module is in scope when a changed path matches its +# pattern. An empty pattern is a repository-wide lint that runs on every diff. +declare -A scope=( + [js]='^(js/|doc/|drafts/|demo/(boy|web)/|test/interop/clients/js|test/wasm/|sh/js/|sh/rs/stats-docs\.py$|package\.json$|bun\.lock(b)?$|biome\.jsonc$)' + # Workers with lockfiles outside the Bun workspace. + [workers]='^(infra/apt/|infra/rpm/|demo/pub/|sh/js/workers\.sh$)' + # sh/rs/select.sh widens to the whole workspace for inputs every crate shares. + [rs]='^(rs/|sh/rs/|Cargo\.(toml|lock)$|rust-toolchain\.toml$|\.config/nextest\.toml$)' + [bench]='^bench/' + [drafts]='^(drafts/|sh/drafts/|doc/\.vitepress/drafts\.ts$)' + # Quest documents form one graph, so any change validates the whole tree. + [quest]='^(quest/|rs/quest/)' + # maturin bundles rs/moq-ffi into the moq-ffi wheel, and the other + # bindings generate from it. + [py]='^(py/|pyproject\.toml$|uv\.lock$|rs/moq-ffi/)' + [kt]='^(kt/|sh/kt/|rs/moq-ffi/)' + [swift]='^(swift/|sh/swift/|rs/moq-ffi/)' + [go]='^(go/|sh/go/|rs/moq-ffi/)' + [dart]='^(dart/|sh/dart/|rs/moq-ffi/)' + # The plugin calls libmoq through its generated header, and flake.nix owns + # the libobs headers it compiles against. + [obs_compile]='^(cpp/obs/|sh/obs/|rs/libmoq/|flake\.nix$)' + # `obs check` compares the OBS pinned in buildspec.json, flake.nix, and + # nixpkgs, and the last moves on a flake.lock bump alone. + [obs]='^(cpp/obs/|sh/obs/|flake\.(nix|lock)$)' + [flake]='(^rs/|^Cargo\.(toml|lock)$|^flake\.lock$|\.nix$)' + [markdown]='' + [shell]='' + [toml]='' + [nix]='' + [justfile]='' + [gh]='' +) + +# The tools each module needs. A module missing one is skipped locally, so an +# incomplete toolchain checks less; under MOQ_STRICT (CI) it is an error, up +# front, because there a skip is indistinguishable from a pass. swift needs +# none: it skips off macOS by design, and swift.yml is its real gate. +declare -A tools=( + [js]='bun python3' + [workers]='bun' + [rs]='cargo jq' + [bench]='cargo' + [drafts]='bun kramdown-rfc xml2rfc' + [quest]='cargo' + [py]='uv' + [kt]='cargo gradle java' + [swift]='' + [go]='cargo go uniffi-bindgen-go' + [dart]='cargo dart uniffi_bindgen_dart' + [obs_compile]='cargo jq pkg-config' + [obs]='clang-format cmake gersemi jq' + [flake]='nix' + [markdown]='bun' + [shell]='shellcheck shfmt' + [toml]='taplo' + [nix]='nixfmt' + [justfile]='' + [gh]='actionlint bun' +) + +case "$action" in + check) modules=(js workers drafts rs bench quest py kt swift go dart obs_compile obs flake markdown shell toml nix justfile gh) ;; + fix) modules=(js rs py dart obs markdown shell toml nix justfile) ;; + test) modules=(js rs py) ;; +esac + +selected=() +missing=() +for module in "${modules[@]}"; do + pattern=${scope[$module]} + if [[ -z "$all" && -n "$pattern" ]] && ! grep -qE "$pattern" "$changed"; then + continue + fi + absent=() + for tool in ${tools[$module]}; do + command -v "$tool" >/dev/null 2>&1 || absent+=("$tool") + done + if ((${#absent[@]} == 0)); then + selected+=("$module") + elif [[ -n "${MOQ_STRICT:-}" ]]; then + missing+=("${absent[@]}") + else + echo "$action: skipping $module; missing ${absent[*]}" >&2 + fi +done + +if ((${#missing[@]})); then + echo "error: MOQ_STRICT is set but these tools are missing: $(printf '%s\n' "${missing[@]}" | sort -u | tr '\n' ' ')" >&2 + echo " run inside 'nix develop', or unset MOQ_STRICT to skip what isn't installed" >&2 + exit 1 +fi + +# rs selects crates from the same list, or the whole workspace. +rs_list=$changed +[[ -z "$all" ]] || rs_list=--all + +# Tracked and new files, skipping whatever .gitignore does (node_modules, target). +files() { + git ls-files -z --cached --others --exclude-standard -- "$@" +} + +for module in "${selected[@]}"; do + case "$action:$module" in + check:js) just js check ;; + check:workers) just js workers ;; + check:rs) just rs check-changed "$rs_list" ;; + check:bench) cargo check --locked --package moq-relay --package moq-bench --features moq-relay/io-uring ;; + check:quest) cargo run --quiet --locked --package quest -- check ;; + check:obs_compile) just obs compile ;; + check:flake) nix flake check ;; + check:markdown) + bun install --frozen-lockfile + sh/markdown.sh check + ;; + check:toml) RUST_LOG=error taplo format --check ;; + check:nix) files '*.nix' | xargs -0 nixfmt --check ;; + check:justfile) files justfile '*/justfile' | xargs -0 -n1 just --fmt --check --justfile ;; + fix:rs) just rs fix-changed "$rs_list" ;; + fix:markdown) + bun install + sh/markdown.sh fix + ;; + fix:toml) RUST_LOG=error taplo format ;; + fix:nix) files '*.nix' | xargs -0 nixfmt ;; + fix:justfile) files justfile '*/justfile' | xargs -0 -n1 just --fmt --justfile ;; + test:rs) just rs test-changed "$rs_list" ;; + *:shell) sh/shell.sh "$action" ;; + *) just "$module" "$action" ;; + esac +done diff --git a/sh/drafts/build.sh b/sh/drafts/build.sh new file mode 100755 index 0000000000..e050b63a3c --- /dev/null +++ b/sh/drafts/build.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Render drafts to .txt and .html: sh/drafts/build.sh [NAME...] +# +# The local editor's copy (gitignored), with the -latest docname as-is; use +# `just drafts publish` for a version. No NAME renders every draft. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)/drafts" + +names=("$@") +if ((${#names[@]} == 0)); then + for f in draft-*.md; do names+=("${f%.md}"); done +fi + +for name in "${names[@]}"; do + kramdown-rfc --v3 <"$name.md" >"$name.xml" + xml2rfc -q --allow-local-file-access --text "$name.xml" -o "$name.txt" + xml2rfc -q --allow-local-file-access --html "$name.xml" -o "$name.html" + echo "built $name.txt and $name.html" +done diff --git a/sh/drafts/check.sh b/sh/drafts/check.sh new file mode 100755 index 0000000000..15ad5bd6c0 --- /dev/null +++ b/sh/drafts/check.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Validate that every draft parses and that its generated XML passes validation. +# +# Both halves are needed. kramdown-rfc catches markdown and frontmatter errors, +# but happily emits XML for a cross-reference that points at no such section; +# xml2rfc is what rejects the dangling IDREF. Checking only the first half lets +# a broken `[text](#anchor)` pass here and then fail at `publish` time. +# +# --preptool stops after validation instead of rendering txt/html, and -N keeps +# it off the network (kramdown-rfc already inlined the references), which is +# what makes this a couple of seconds per draft rather than a minute. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" +bun install --frozen-lockfile +cd drafts + +for f in draft-*.md; do + echo "checking $f" + kramdown-rfc --v3 <"$f" >"${f%.md}.xml" + xml2rfc -q --allow-local-file-access --preptool -N "${f%.md}.xml" -o "${f%.md}.prepped.xml" || { + rm -f "${f%.md}.prepped.xml" + exit 1 + } + rm -f "${f%.md}.prepped.xml" +done + +for f in moq-e2ee-*.ts; do + bun tsc --noEmit --skipLibCheck --target esnext --module esnext --moduleResolution bundler --types bun "$f" + bun "$f" +done diff --git a/sh/drafts/publish.sh b/sh/drafts/publish.sh new file mode 100755 index 0000000000..9ef5e8ad5c --- /dev/null +++ b/sh/drafts/publish.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Submit a new draft version to the IETF datatracker: sh/drafts/publish.sh NAME VERSION EMAIL +# +# The datatracker emails the submitter a confirmation link; the submission is +# not final until that link is clicked. For a brand-new draft (-00) set +# "Replaces" on the confirmation page. +set -euo pipefail + +usage="usage: sh/drafts/publish.sh NAME VERSION EMAIL" +name=${1:?$usage} +version=${2:?$usage} +email=${3:?$usage} + +cd "$(git rev-parse --show-toplevel)/drafts" +case "$version" in + [0-9][0-9]) ;; + *) + echo "version must be two digits, e.g. 05" >&2 + exit 1 + ;; +esac +doc="$name-$version" +if [ ! -f "$name.md" ]; then + echo "no such draft: $name.md" >&2 + exit 1 +fi +echo "Building $doc.xml" +sed "s/$name-latest/$doc/g" "$name.md" | kramdown-rfc --v3 >"$doc.xml" +echo "Submitting $doc.xml to the datatracker as $email" +resp="$(mktemp)" +code="$(curl -sS -o "$resp" -w '%{http_code}' \ + -F "user=$email" -F "xml=@$doc.xml" \ + https://datatracker.ietf.org/api/submission)" +echo "HTTP $code" +cat "$resp" +echo +rm -f "$resp" +case "$code" in + 200 | 201) echo "Submitted. Check $email for the confirmation link." ;; + *) + echo "Submission failed." >&2 + exit 1 + ;; +esac diff --git a/.github/scripts/alert.sh b/sh/gh/alert.sh similarity index 99% rename from .github/scripts/alert.sh rename to sh/gh/alert.sh index 6a795dc47a..cfdec039c4 100755 --- a/.github/scripts/alert.sh +++ b/sh/gh/alert.sh @@ -18,7 +18,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -WORKFLOWS_DIR="$(cd "$SCRIPT_DIR/../workflows" && pwd)" +WORKFLOWS_DIR="$(cd "$SCRIPT_DIR/../../.github/workflows" && pwd)" # Post a red embed naming the workflow, what it was reacting to, and a link. # jq builds the body so a commit subject with quotes or newlines can't produce diff --git a/.github/scripts/install-formula.py b/sh/gh/install-formula.py similarity index 100% rename from .github/scripts/install-formula.py rename to sh/gh/install-formula.py diff --git a/.github/scripts/install-formula.test.py b/sh/gh/install-formula.test.py similarity index 100% rename from .github/scripts/install-formula.test.py rename to sh/gh/install-formula.test.py diff --git a/.github/scripts/release.sh b/sh/gh/release.sh similarity index 100% rename from .github/scripts/release.sh rename to sh/gh/release.sh diff --git a/.github/scripts/release.test.sh b/sh/gh/release.test.sh similarity index 100% rename from .github/scripts/release.test.sh rename to sh/gh/release.test.sh diff --git a/.github/scripts/render-formula.sh b/sh/gh/render-formula.sh similarity index 100% rename from .github/scripts/render-formula.sh rename to sh/gh/render-formula.sh diff --git a/.github/scripts/trigger-repo-publish.sh b/sh/gh/trigger-repo-publish.sh similarity index 100% rename from .github/scripts/trigger-repo-publish.sh rename to sh/gh/trigger-repo-publish.sh diff --git a/go/scripts/check-errors.sh b/sh/go/check-errors.sh similarity index 100% rename from go/scripts/check-errors.sh rename to sh/go/check-errors.sh diff --git a/go/scripts/check.sh b/sh/go/check.sh similarity index 84% rename from go/scripts/check.sh rename to sh/go/check.sh index 5f5901c0a7..86f45396d9 100755 --- a/go/scripts/check.sh +++ b/sh/go/check.sh @@ -3,7 +3,7 @@ set -euo pipefail # Local smoke check for the Go module. # -# Stages the ffi + wrapper modules from this checkout (scripts/stage.sh) and +# Stages the ffi + wrapper modules from this checkout (sh/go/stage.sh) and # runs `go build`/`go vet`/`go test` against them. Intended for `just go check`. # # The main repo stays binary-free: no `.a` or generated `.go` files land @@ -13,17 +13,13 @@ set -euo pipefail # Skipped cleanly on hosts without `go`, `cargo`, or `uniffi-bindgen-go`. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +GO_DIR="$(cd "$SCRIPT_DIR/../../go" && pwd)" WORKSPACE_DIR="$(cd "$GO_DIR/.." && pwd)" if ! command -v go >/dev/null 2>&1; then echo "go check: no go on PATH, skipping" >&2 exit 0 fi -# The publish state machine needs neither cargo nor generated bindings, so it -# runs ahead of the gates that skip a partial toolchain. -bash "$SCRIPT_DIR/publish-wrapper.test.sh" - if ! command -v cargo >/dev/null 2>&1; then echo "go check: no cargo on PATH, skipping" >&2 exit 0 diff --git a/go/scripts/package-ffi.sh b/sh/go/package-ffi.sh similarity index 98% rename from go/scripts/package-ffi.sh rename to sh/go/package-ffi.sh index 04450fd28f..b2d89cd51d 100755 --- a/go/scripts/package-ffi.sh +++ b/sh/go/package-ffi.sh @@ -10,7 +10,7 @@ set -euo pipefail # output at $BINDINGS_DIR/moq/moq.go. # # Usage: -# go/scripts/package-ffi.sh --version 0.0.0-dev --lib-dir libs --bindings-dir bindings --output dist +# sh/go/package-ffi.sh --version 0.0.0-dev --lib-dir libs --bindings-dir bindings --output dist # # Optional: # --source-dir DIR in-tree ffi module skeleton (default: go/ffi) @@ -24,7 +24,7 @@ set -euo pipefail # $LIB_DIR/x86_64-pc-windows-msvc/moq_ffi.lib SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +GO_ROOT="$(cd "$SCRIPT_DIR/../../go" && pwd)" WORKSPACE_DIR="$(cd "$GO_ROOT/.." && pwd)" VERSION="" diff --git a/go/scripts/package-wrapper.sh b/sh/go/package-wrapper.sh similarity index 97% rename from go/scripts/package-wrapper.sh rename to sh/go/package-wrapper.sh index 210d0db854..52159e7932 100755 --- a/go/scripts/package-wrapper.sh +++ b/sh/go/package-wrapper.sh @@ -13,7 +13,7 @@ set -euo pipefail # produces an identical tree and no release. # # Usage: -# go/scripts/package-wrapper.sh --line 0.3 --ffi-version 0.2.18 --output dist +# sh/go/package-wrapper.sh --line 0.3 --ffi-version 0.2.18 --output dist # # Optional: # --source-dir DIR in-tree wrapper skeleton (default: go/wrapper) @@ -22,7 +22,7 @@ set -euo pipefail # --no-archive SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +GO_ROOT="$(cd "$SCRIPT_DIR/../../go" && pwd)" WORKSPACE_DIR="$(cd "$GO_ROOT/.." && pwd)" LINE="" diff --git a/go/scripts/publish-ffi.sh b/sh/go/publish-ffi.sh similarity index 100% rename from go/scripts/publish-ffi.sh rename to sh/go/publish-ffi.sh diff --git a/go/scripts/publish-wrapper.sh b/sh/go/publish-wrapper.sh similarity index 100% rename from go/scripts/publish-wrapper.sh rename to sh/go/publish-wrapper.sh diff --git a/go/scripts/publish-wrapper.test.sh b/sh/go/publish-wrapper.test.sh similarity index 97% rename from go/scripts/publish-wrapper.test.sh rename to sh/go/publish-wrapper.test.sh index 423778218b..39e57fe794 100755 --- a/go/scripts/publish-wrapper.test.sh +++ b/sh/go/publish-wrapper.test.sh @@ -12,7 +12,7 @@ set -euo pipefail # missing. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +GO_DIR="$(cd "$SCRIPT_DIR/../../go" && pwd)" if ! command -v go >/dev/null 2>&1; then echo "publish-wrapper test: no go on PATH, skipping" >&2 @@ -75,7 +75,7 @@ LINE=$(tr -d '[:space:]' <"$GO_DIR/wrapper/VERSION") # Stage a real wrapper tarball. --skip-tidy keeps it offline (no go.sum), which # is fine: publish-wrapper.sh only reads VERSION out of the staged tree. -bash "$GO_DIR/scripts/package-wrapper.sh" \ +bash "$SCRIPT_DIR/package-wrapper.sh" \ --line "$LINE" \ --ffi-version "0.0.0-test" \ --source-dir "$GO_DIR/wrapper" \ @@ -93,7 +93,7 @@ publish() { ( cd "$WORK/run" GO_MIRROR_URL="$MIRROR" GO_MIRROR_TOKEN=unused \ - bash "$GO_DIR/scripts/publish-wrapper.sh" + bash "$SCRIPT_DIR/publish-wrapper.sh" ) } diff --git a/go/scripts/stage.sh b/sh/go/stage.sh similarity index 98% rename from go/scripts/stage.sh rename to sh/go/stage.sh index ca72d92c67..5992049805 100755 --- a/go/scripts/stage.sh +++ b/sh/go/stage.sh @@ -10,7 +10,7 @@ set -euo pipefail # its own module against them and build Go code against these bindings. # # Usage: -# go/scripts/stage.sh [--output DIR] +# sh/go/stage.sh [--output DIR] # # Optional: # --output DIR stage parent (default: /dist, gitignored) @@ -19,7 +19,7 @@ set -euo pipefail # for an optimized cdylib. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +GO_DIR="$(cd "$SCRIPT_DIR/../../go" && pwd)" WORKSPACE_DIR="$(cd "$GO_DIR/.." && pwd)" STAGE_PARENT="$WORKSPACE_DIR/dist" diff --git a/sh/js/workers.sh b/sh/js/workers.sh new file mode 100755 index 0000000000..4fb2178c2a --- /dev/null +++ b/sh/js/workers.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Validate each Worker's deploy bundle without publishing or credentials. +# +# These Workers have lockfiles outside the Bun workspace, so `just js check` +# never installs or builds them. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +for project in infra/apt infra/rpm demo/pub; do + ( + cd "$project" + bun install --frozen-lockfile + bun run deploy --dry-run + ) +done diff --git a/kt/scripts/check.sh b/sh/kt/check.sh similarity index 89% rename from kt/scripts/check.sh rename to sh/kt/check.sh index ee43ebe5cd..78e960dd3b 100644 --- a/kt/scripts/check.sh +++ b/sh/kt/check.sh @@ -2,7 +2,7 @@ set -euo pipefail # Full check for the Kotlin packages: regenerate the bindings + native lib -# (scripts/generate.sh), then run the raw-bindings and wrapper JVM tests. +# (sh/kt/generate.sh), then run the raw-bindings and wrapper JVM tests. # # Unlike generation, this needs a JDK and Gradle. Both ship in the `nix # develop` dev shell (see flake.nix ktDeps), so a missing one is an error @@ -11,7 +11,7 @@ set -euo pipefail # should run `just kt generate` instead. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +KT_DIR="$(cd "$SCRIPT_DIR/../../kt" && pwd)" bash "$SCRIPT_DIR/generate.sh" diff --git a/kt/scripts/generate.sh b/sh/kt/generate.sh similarity index 96% rename from kt/scripts/generate.sh rename to sh/kt/generate.sh index c98dddd32d..fcae7f8080 100755 --- a/kt/scripts/generate.sh +++ b/sh/kt/generate.sh @@ -6,11 +6,11 @@ set -euo pipefail # and regenerate the uniffi bindings. No Gradle, no JDK required. # # Intended for environments that intentionally lack Gradle (e.g. regenerating -# checked-in bindings after a moq-ffi change). `scripts/check.sh` calls this +# checked-in bindings after a moq-ffi change). `sh/kt/check.sh` calls this # first and then compiles + tests the result. Requires cargo. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +KT_DIR="$(cd "$SCRIPT_DIR/../../kt" && pwd)" WORKSPACE_DIR="$(cd "$KT_DIR/.." && pwd)" if ! command -v cargo >/dev/null 2>&1; then diff --git a/kt/scripts/package.sh b/sh/kt/package.sh similarity index 97% rename from kt/scripts/package.sh rename to sh/kt/package.sh index 73e9011d70..03c9cc1338 100755 --- a/kt/scripts/package.sh +++ b/sh/kt/package.sh @@ -10,7 +10,7 @@ set -euo pipefail # uniffi-bindgen kotlin output at $BINDINGS_DIR/uniffi/moq/moq.kt. # # Usage: -# kt/scripts/package.sh --version 0.0.0-dev --lib-dir libs --bindings-dir bindings --output dist +# sh/kt/package.sh --version 0.0.0-dev --lib-dir libs --bindings-dir bindings --output dist # # Expected $LIB_DIR layout (per target, populated by the build matrix): # $LIB_DIR/aarch64-linux-android/libmoq_ffi.so @@ -20,7 +20,7 @@ set -euo pipefail # ... etc. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +KT_DIR="$(cd "$SCRIPT_DIR/../../kt" && pwd)" VERSION="" LIB_DIR="" diff --git a/sh/markdown.sh b/sh/markdown.sh new file mode 100755 index 0000000000..cd811fa911 --- /dev/null +++ b/sh/markdown.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Lint or format every Markdown file: sh/markdown.sh check|fix +# +# remark-cli has no check mode: `--frail` raises the exit code on lint +# messages, and only `--output` formats, so a file that is merely misformatted +# passes both ways. `check` formats a scratch mirror and diffs it, so it stays +# read-only while `fix` writes in place. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +case "${1:-}" in + check) ;; + fix) + bun remark . --quiet --output + exit 0 + ;; + *) + echo "usage: sh/markdown.sh check|fix" >&2 + exit 2 + ;; +esac + +mirror=$(mktemp -d) +trap 'rm -rf "$mirror"' EXIT + +extension_list=$(bun -e \ + 'import extensions from "markdown-extensions"; console.log(extensions.join("\n"))') +patterns=() +while IFS= read -r extension; do + [[ -n "$extension" ]] && patterns+=("*.$extension") +done <<<"$extension_list" +((${#patterns[@]})) || { + echo "error: remark reported no Markdown extensions" >&2 + exit 1 +} + +# Untracked files are in scope so a new doc is linted before it is staged; +# --exclude-standard keeps build output out. +files=() +while IFS= read -r -d '' file; do + [[ -f "$file" ]] || continue + files+=("$file") + mkdir -p "$mirror/$(dirname "$file")" + cp "$file" "$mirror/$file" +done < <(git ls-files -z --cached --others --exclude-standard -- "${patterns[@]}") +((${#files[@]})) || exit 0 + +# The config rides along so every .remarkignore pattern resolves against the +# mirror root exactly as it does here, and node_modules is where the plugins +# named by .remarkrc.mjs come from. +cp .remarkrc.mjs .remarkignore "$mirror/" +ln -s "$PWD/node_modules" "$mirror/node_modules" + +status=0 +(cd "$mirror" && bun remark . --quiet --frail --output) || status=$? + +stale=() +for file in "${files[@]}"; do + cmp -s "$file" "$mirror/$file" || stale+=("$file") +done + +if ((${#stale[@]})); then + echo "error: these files are not formatted, run 'just fix':" >&2 + printf ' %s\n' "${stale[@]}" >&2 + status=1 +fi + +exit "$status" diff --git a/sh/obs/check.sh b/sh/obs/check.sh new file mode 100755 index 0000000000..b8b651b41b --- /dev/null +++ b/sh/obs/check.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Lint formatting. Skips a tool silently if it isn't on $PATH (matches repo convention). +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/../../cpp/obs" + +if grep -Fq '"MOQ_VERSION"' CMakePresets.json; then + echo "CMakePresets.json must not pin MOQ_VERSION" >&2 + exit 1 +fi +# The headers `just obs compile` type-checks against are a second copy of +# the OBS version this spec downloads, so a bump here has to reach flake.nix +# or the two gates check different libobs. +spec_obs=$(jq -r '.dependencies["obs-studio"].version // ""' buildspec.json) +flake_obs=$(sed -n '/pname = "libobs-headers"/,/}/p' ../../flake.nix | sed -n 's/.*version = "\([^"]*\)".*/\1/p') +# Both are pattern matches against files this recipe doesn't own the shape +# of. An empty parse compares equal to an empty parse, so the guard would +# pass by reporting nothing rather than by finding the versions in sync. +if [ -z "$spec_obs" ] || [ -z "$flake_obs" ]; then + echo "couldn't read the obs-studio version: '$spec_obs' from buildspec.json, '$flake_obs' from flake.nix" >&2 + exit 1 +fi +if [ "$spec_obs" != "$flake_obs" ]; then + echo "obs-studio is $spec_obs in buildspec.json but $flake_obs in flake.nix (libobs-headers)" >&2 + exit 1 +fi +# And a third: `just obs ci` links against nixpkgs' obs-studio, the one OBS +# this repo doesn't pick. Without this the released macOS/Windows binaries +# could be a whole libobs API apart from what CI ever compiles. Patch +# releases carry no API change, so only the major.minor has to agree; that +# also keeps the guard quiet until flake.lock actually moves OBS, which is +# the change that opens the gap and so the one that should fail. +# OBS_LINKED_VERSION comes from the dev shell (see flake.nix), so this leg +# is skipped outside it, where there is no nixpkgs obs-studio to compare. +# MOQ_STRICT turns that skip into an error for the same reason it turns a +# missing tool into one: in CI a leg that checks nothing still reports green. +if [ -z "${OBS_LINKED_VERSION:-}" ]; then + if [ -n "${MOQ_STRICT:-}" ]; then + echo "MOQ_STRICT is set but OBS_LINKED_VERSION is unset; run inside 'nix develop', which exports it" >&2 + exit 1 + fi +else + spec_api=$(echo "$spec_obs" | cut -d. -f1,2) + linked_api=$(echo "$OBS_LINKED_VERSION" | cut -d. -f1,2) + if [ "$spec_api" != "$linked_api" ]; then + echo "obs-studio is $spec_obs in buildspec.json but $OBS_LINKED_VERSION in nixpkgs, which is what 'just obs ci' links" >&2 + echo "bump buildspec.json and flake.nix's libobs-headers to $OBS_LINKED_VERSION, or pin nixpkgs back" >&2 + exit 1 + fi +fi +if command -v cmake >/dev/null; then + build_dir=$(mktemp -d) + trap 'rm -rf "$build_dir"' EXIT + if cmake -S . -B "$build_dir" -DMOQ_LOCAL= >"$build_dir/configure.log" 2>&1; then + echo "CMake accepted a release build without MOQ_VERSION" >&2 + exit 1 + fi + if ! grep -Fq "MOQ_VERSION is required when MOQ_LOCAL does not contain rs/libmoq" "$build_dir/configure.log"; then + cat "$build_dir/configure.log" >&2 + exit 1 + fi +fi +if command -v clang-format >/dev/null; then + git ls-files 'src/*.cpp' 'src/*.h' 'test/*.cpp' | xargs clang-format --dry-run --Werror +fi +if command -v gersemi >/dev/null; then + gersemi --check CMakeLists.txt cmake +fi diff --git a/sh/obs/compile.sh b/sh/obs/compile.sh new file mode 100755 index 0000000000..6253cc1586 --- /dev/null +++ b/sh/obs/compile.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Type-check every plugin source and unit test, without linking or an obs-deps +# download. +# +# This is the gate to run in a worktree. `just obs build` needs the +# multi-hundred-MB obs-deps bundle (macOS/Windows) and a full cargo build, per +# worktree; this needs neither, because the dev shell carries all three header +# sets: libobs pinned to the same OBS release buildspec.json downloads (see +# `obs-headers` in flake.nix), Qt6, and ffmpeg. CI links the real thing on Linux +# (.github/workflows/obs.yml). +# +# It compiles the Qt sources too, which the CMake build only does when +# ENABLE_QT and ENABLE_FRONTEND_API are on, so the definitions they gate on are +# set here to match. +# +# test/ is in scope because each test file defines the libmoq entry points the +# plugin calls, so a signature that drifts from the generated moq.h is a +# conflicting C declaration. Catching that needs only headers, which is why it +# belongs here: `just obs ci` finds the same drift, but only where obs.yml's +# path filter reaches, and `just obs test` is manual. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +cd "$here/../../cpp/obs" + +# Never skips, for the reason `test` doesn't: a compile gate that reports +# success without compiling is worse than no gate. Everything it needs is in +# the dev shell, so the fix is always to enter it. +cxx="${CXX:-c++}" +if ! command -v "$cxx" >/dev/null; then + echo "no C++ compiler at '$cxx'" >&2 + exit 1 +fi +for pkgs in "Qt6Widgets Qt6Gui Qt6Core" "libavcodec libavutil libswscale libswresample"; do + # shellcheck disable=SC2086 + if ! pkg-config --exists $pkgs; then + echo "missing headers for: $pkgs" >&2 + echo "run inside 'nix develop', which supplies Qt6 and ffmpeg on every platform" >&2 + exit 1 + fi +done + +# Assign first so a failure in `_includes` still aborts under `set -e`, +# then split on newlines only: an include path may contain spaces. +includes_raw=$("$here/includes.sh") +includes=() +while IFS= read -r flag; do includes+=("$flag"); done <<<"$includes_raw" + +qt=$(pkg-config --cflags Qt6Widgets Qt6Gui Qt6Core) +ffmpeg=$(pkg-config --cflags libavcodec libavutil libswscale libswresample) + +# MOQ_VERSION_STRING only reaches a label in the dock, so any value +# type-checks the same; CMake stamps the real libmoq version. +status=0 +for source in src/*.cpp test/*.cpp; do + # shellcheck disable=SC2086 + # -Wno-unused-command-line-argument: the nix compiler wrapper injects + # link flags, which a syntax-only run reports as unused, once per file. + # $qt and $ffmpeg stay unquoted on purpose: pkg-config hands back one + # space-separated string, and splitting it is the intended reading. + if ! "$cxx" -std=c++17 -fsyntax-only -Wno-unused-command-line-argument \ + "${includes[@]}" $qt $ffmpeg \ + -DMOQ_FRONTEND_ENABLED -DMOQ_VERSION_STRING='"0.0.0"' "$source"; then + status=1 + fi +done +exit $status diff --git a/sh/obs/fix.sh b/sh/obs/fix.sh new file mode 100755 index 0000000000..46cf8d27f8 --- /dev/null +++ b/sh/obs/fix.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Auto-fix formatting. +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/../../cpp/obs" + +if command -v clang-format >/dev/null; then + git ls-files 'src/*.cpp' 'src/*.h' 'test/*.cpp' | xargs clang-format -i +fi +if command -v gersemi >/dev/null; then + gersemi --in-place CMakeLists.txt cmake +fi diff --git a/sh/obs/includes.sh b/sh/obs/includes.sh new file mode 100755 index 0000000000..4456acfeca --- /dev/null +++ b/sh/obs/includes.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Print the -I flags for libobs and libmoq, shared by compile.sh and unit.sh. +# +# moq.h is generated by cbindgen from rs/libmoq/src, so regenerate rather than +# just locating it: a stale header is how a plugin call to a since-changed +# libmoq function type-checks anyway. +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/../../cpp/obs" + +# The dev shell's pinned headers first, so every platform checks against the +# same libobs. Then the obs-deps framework (macOS/Windows), the OBS sources +# CMake unpacks beside it, and a system install (Linux without nix). +obs_include="" +for candidate in \ + "${OBS_INCLUDE_DIR:-}" \ + .deps/Frameworks/libobs.framework/Versions/A/Headers \ + .deps/obs-studio-*/libobs; do + if [ -n "$candidate" ] && [ -f "$candidate/obs.h" ]; then + obs_include="$candidate" + break + fi +done +if [ -z "$obs_include" ] && command -v pkg-config >/dev/null && pkg-config --exists libobs; then + obs_include=$(pkg-config --variable=includedir libobs)/obs +fi +if [ -z "$obs_include" ]; then + echo "libobs headers not found; run inside 'nix develop' (or set OBS_INCLUDE_DIR)" >&2 + exit 1 +fi + +# obs.h reaches as of OBS 32, which used to be vendored +# under libobs/util/ and so came along with the headers above. The dev shell +# carries it (see flake.nix); the obs-deps bundle ships its own copy beside +# libobs, which is how the macOS and Windows builds get it. +simde_include="" +if command -v pkg-config >/dev/null && pkg-config --exists simde; then + simde_include=$(pkg-config --variable=includedir simde) +else + for candidate in .deps/obs-deps-*/include; do + if [ -f "$candidate/simde/x86/sse2.h" ]; then + simde_include="$candidate" + break + fi + done +fi +if [ -z "$simde_include" ]; then + echo "simde headers not found; run inside 'nix develop', which supplies them" >&2 + exit 1 +fi + +(cd ../.. && cargo check --locked --quiet -p libmoq) + +# Ask cargo where its build script ran rather than reconstructing the path. +# CARGO_TARGET_DIR, a `--target` triple and a custom profile each move the +# generated header, and guessing wrong doesn't fail: it type-checks against +# whatever stale copy the default directory still holds. Replays the check +# above from cache, so it costs a process rather than a compile. +out_dir=$(cd ../.. && cargo check --locked -p libmoq --message-format=json | + jq -r 'select(.reason == "build-script-executed") | select(.package_id | tostring | test("libmoq")) | .out_dir' | + tail -1) +if [ -z "$out_dir" ]; then + echo "cargo reported no build script output for libmoq" >&2 + exit 1 +fi +# //build/libmoq-/out, and the header sits beside the +# profile directory. rs/libmoq/build.rs derives it from OUT_DIR the same way. +moq_include="$(dirname "$(dirname "$(dirname "$(dirname "$out_dir")")")")/include" +if [ ! -f "$moq_include/moq.h" ]; then + echo "$moq_include/moq.h missing after 'cargo check --locked -p libmoq'" >&2 + exit 1 +fi + +# One flag per line, path glued to the -I, so a caller can rebuild the +# arguments without splitting on the spaces in a path. +printf -- '-I%s\n' src "$obs_include" "$simde_include" "$moq_include" diff --git a/sh/obs/preset.sh b/sh/obs/preset.sh new file mode 100755 index 0000000000..c6bd950dc6 --- /dev/null +++ b/sh/obs/preset.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Detect the CMake preset for the current platform (or use the override). +# Pass `ci` as the second argument for the warnings-as-errors variant, whose +# name upstream spells differently on each platform. +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/../../cpp/obs" +override=${1:-} +variant=${2:-} + +if [[ -n "$override" ]]; then + echo "$override" +elif [[ "$OSTYPE" == "darwin"* ]]; then + [[ "$variant" == "ci" ]] && echo "macos-ci" || echo "macos" +elif [[ "$OSTYPE" == "linux-gnu"* ]]; then + [[ "$variant" == "ci" ]] && echo "ubuntu-ci-x86_64" || echo "ubuntu-x86_64" +elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then + [[ "$variant" == "ci" ]] && echo "windows-ci-x64" || echo "windows-x64" +else + echo "Unknown platform: $OSTYPE" >&2 + exit 1 +fi diff --git a/sh/obs/test.sh b/sh/obs/test.sh new file mode 100755 index 0000000000..45033fbcff --- /dev/null +++ b/sh/obs/test.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Unit-test the plugin sources against stubbed libobs/libmoq/ffmpeg, under +# ThreadSanitizer. +# +# Manual, like `just rs macos` and `just rs windows`, because ThreadSanitizer +# needs its own build. `just obs ci` runs the same tests without it, so a +# regression these assertions catch still turns PR CI red; the sanitizer is what +# adds the races on top. Run this whenever you touch src/, especially the session +# callback plumbing, whose orderings the build can't check. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +cd "$here/../../cpp/obs" + +# Unlike the lint recipes, this one never skips: a test gate that reports +# success without running anything is worse than no gate. +cxx="${CXX:-c++}" +unsupported="'$cxx' cannot build and run -fsanitize=thread binaries. These tests need a +Clang or GCC whose ThreadSanitizer runtime works on this host: set CXX, or on Windows run +them from WSL, since neither MSVC nor Clang on Windows implements ThreadSanitizer. +They also run without the sanitizer as part of 'just obs ci'." +if ! command -v "$cxx" >/dev/null; then + echo "no C++ compiler at '$cxx'." >&2 + echo "$unsupported" >&2 + exit 1 +fi + +probe=$(mktemp -d) +trap 'rm -rf "$probe"' EXIT + +# Run the probe, don't just link it: a mismatch between the compiler's TSan +# runtime and the host kernel (macOS is where this shows up) links fine and +# then segfaults before main, which is an unreadable way for the real test +# to fail. +# The subshell keeps bash's own "Segmentation fault" job message out of the +# output, so the explanation below is all the reader gets. +if ! printf 'int main(){}\n' | "$cxx" -x c++ -fsanitize=thread -o "$probe/probe" - >/dev/null 2>&1 || + ! ("$probe/probe" >/dev/null 2>&1) 2>/dev/null; then + echo "$unsupported" >&2 + exit 1 +fi + +"$here/unit.sh" tsan diff --git a/sh/obs/unit.sh b/sh/obs/unit.sh new file mode 100755 index 0000000000..54b775baac --- /dev/null +++ b/sh/obs/unit.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Compile and run the unit tests: sh/obs/unit.sh [plain|tsan] +# +# Shared by `just obs test` (ThreadSanitizer) and `just obs ci` (plain), so +# both run the same assertions. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +cd "$here/../../cpp/obs" +mode=${1:-plain} + +cxx="${CXX:-c++}" +if ! command -v "$cxx" >/dev/null; then + echo "no C++ compiler at '$cxx'" >&2 + exit 1 +fi +# moq-source.cpp decodes with ffmpeg, so its test needs the headers even +# though it stubs every function out of them. +if ! pkg-config --exists libavcodec libavutil libswscale; then + echo "missing ffmpeg headers" >&2 + echo "run inside 'nix develop', which supplies ffmpeg on every platform" >&2 + exit 1 +fi + +includes_raw=$("$here/includes.sh") +includes=() +while IFS= read -r flag; do includes+=("$flag"); done <<<"$includes_raw" + +ffmpeg=$(pkg-config --cflags libavcodec libavutil libswscale) + +flags=(-std=c++17 -g -O0 -pthread) +if [ "$mode" = "tsan" ]; then + flags+=(-fsanitize=thread) +fi + +out=$(mktemp -d) +trap 'rm -rf "$out"' EXIT + +# One binary per source under test: each test file defines its own libobs and +# libmoq stubs, so two of them can't share a link. Header-only helpers +# (quality defaults) build as a third binary with no plugin source. +for name in moq-output moq-source; do + # shellcheck disable=SC2086 + # $ffmpeg stays unquoted on purpose: pkg-config hands back one + # space-separated string, and splitting it is the intended reading. + "$cxx" "${flags[@]}" "${includes[@]}" $ffmpeg \ + -o "$out/$name-test" "test/$name-test.cpp" "src/$name.cpp" + TSAN_OPTIONS="halt_on_error=1" "$out/$name-test" +done +for name in moq-quality-defaults moq-dock-stop moq-error moq-encoder-latency moq-dial moq-spark; do + "$cxx" "${flags[@]}" "${includes[@]}" \ + -o "$out/$name-test" "test/$name-test.cpp" + TSAN_OPTIONS="halt_on_error=1" "$out/$name-test" +done diff --git a/sh/rs/c-tests.sh b/sh/rs/c-tests.sh new file mode 100755 index 0000000000..249ee6c581 --- /dev/null +++ b/sh/rs/c-tests.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Compile and run libmoq's C fixtures (`rs/libmoq/c-tests/*.c`) against +# libmoq.a, linked the way an embedder does: an external `cc` against the +# generated `moq.h`, plus the native libraries from `rs/libmoq/native-libs` +# that cargo can't inject into a link it doesn't drive. Unix only. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +cc="${CC:-cc}" +if ! command -v "$cc" >/dev/null; then + echo "no C compiler at '$cc'" >&2 + exit 1 +fi + +cargo build --locked -p libmoq + +# Ask cargo where it put the staticlib rather than assuming target/debug: +# CARGO_TARGET_DIR and a configured build target (`target//debug`) +# both move it, and guessing wrong would link a stale copy or nothing. +# Replays the build above from cache, so it costs a process, not a compile. +lib=$(cargo build --locked -p libmoq --message-format=json | + jq -r 'select(.reason == "compiler-artifact") | .filenames[] | select(endswith("/libmoq.a"))' | + tail -1) +if [ -z "$lib" ]; then + echo "cargo reported no libmoq.a for libmoq" >&2 + exit 1 +fi +# The header sits beside the profile directory: rs/libmoq/build.rs writes it +# to `/include`, one level above `//libmoq.a`. +profile=$(dirname "$lib") +include="$(dirname "$profile")/include" +if [ ! -f "$include/moq.h" ]; then + echo "$include/moq.h missing after 'cargo build --locked -p libmoq'" >&2 + exit 1 +fi + +# Same list build.rs (moq.pc), CMakeLists.txt, and test/interop/interop.sh read: +# `framework:Foo` is a linker framework flag, anything else a plain library. +case "$(uname -s)" in + Darwin) native_libs=rs/libmoq/native-libs/apple.txt ;; + *) native_libs=rs/libmoq/native-libs/linux.txt ;; +esac +libs=() +while read -r entry; do + case "$entry" in + '' | '#'*) continue ;; + framework:*) libs+=(-framework "${entry#framework:}") ;; + *) libs+=("-l$entry") ;; + esac +done <"$native_libs" + +out=$(mktemp -d) +trap 'rm -rf "$out"' EXIT +for source in rs/libmoq/c-tests/*.c; do + bin="$out/$(basename "$source" .c)" + "$cc" "$source" -I"$include" -L"$profile" -lmoq "${libs[@]}" -o "$bin" + echo "running $source" + "$bin" +done diff --git a/sh/rs/fuzz.sh b/sh/rs/fuzz.sh new file mode 100755 index 0000000000..5c77f75d07 --- /dev/null +++ b/sh/rs/fuzz.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Coverage-guided fuzzing of the moq-net wire codecs: sh/rs/fuzz.sh TARGET [ARGS...] +# +# TARGET is lite, ietf, varint, or path; ARGS pass to `cargo fuzz run`, so +# `just rs fuzz lite -- -max_total_time=300` bounds a run. See +# rs/moq-net/fuzz/README.md. +# +# Seeds are regenerated first so the corpus follows the dispatch rather than a +# stale run, and libFuzzer writes what it discovers to the FIRST corpus +# directory, which is why the generated seeds are passed second and stay +# read-only. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +target=${1:?usage: sh/rs/fuzz.sh TARGET [ARGS...]} +shift + +if ! command -v cargo-fuzz >/dev/null 2>&1; then + echo "rs: cargo-fuzz is missing: cargo install --locked cargo-fuzz" >&2 + exit 1 +fi + +if ! rustup run nightly rustc --version >/dev/null 2>&1; then + echo "rs: the nightly toolchain is missing: rustup toolchain install nightly" >&2 + exit 1 +fi + +cargo run --locked -q -p moq-net --features fuzz --example fuzz-seeds -- rs/moq-net/fuzz/seeds +mkdir -p rs/moq-net/fuzz/corpus/"$target" + +# Nightly by PATH rather than by `cargo +nightly`: the dev shell puts a +# Nix-provided cargo ahead of the rustup shim, so the `+` form is not +# understood, and cargo-fuzz shells out to a bare `cargo` and `rustc` anyway. +# Both have to be the nightly ones or the sanitizer flags are rejected. +sysroot=$(rustup run nightly rustc --print sysroot) +export PATH="$sysroot/bin:$PATH" + +cargo fuzz run --fuzz-dir rs/moq-net/fuzz "$target" \ + rs/moq-net/fuzz/corpus/"$target" rs/moq-net/fuzz/seeds/"$target" "$@" diff --git a/rs/scripts/media-features.sh b/sh/rs/media-features.sh similarity index 100% rename from rs/scripts/media-features.sh rename to sh/rs/media-features.sh diff --git a/rs/scripts/package-binary.sh b/sh/rs/package-binary.sh similarity index 99% rename from rs/scripts/package-binary.sh rename to sh/rs/package-binary.sh index 114b9e269f..23fcde18c4 100755 --- a/rs/scripts/package-binary.sh +++ b/sh/rs/package-binary.sh @@ -16,7 +16,7 @@ set -euo pipefail # produces an extensionless executable beside the archive. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -RS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +RS_DIR="$(cd "$SCRIPT_DIR/../../rs" && pwd)" WORKSPACE_DIR="$(cd "$RS_DIR/.." && pwd)" CRATE="" diff --git a/rs/scripts/package-nfpm.sh b/sh/rs/package-nfpm.sh similarity index 100% rename from rs/scripts/package-nfpm.sh rename to sh/rs/package-nfpm.sh diff --git a/rs/scripts/package-rename.test.py b/sh/rs/package-rename.test.py similarity index 98% rename from rs/scripts/package-rename.test.py rename to sh/rs/package-rename.test.py index d4222ce00f..484dda146a 100644 --- a/rs/scripts/package-rename.test.py +++ b/sh/rs/package-rename.test.py @@ -27,7 +27,7 @@ def test_packages(self): for config, packager in (("nfpm", "deb"), ("transition", "deb"), ("nfpm", "rpm")): output = scratch / f"{name}-{config}.{packager}" subprocess.run([ - "bash", "rs/scripts/package-nfpm.sh", + "bash", "sh/rs/package-nfpm.sh", f"packaging/{old}/{config}.yaml", packager, str(output), ], env=env, cwd=ROOT, check=True) outputs[config, packager] = output diff --git a/rs/scripts/package-windows.sh b/sh/rs/package-windows.sh similarity index 98% rename from rs/scripts/package-windows.sh rename to sh/rs/package-windows.sh index 33658a71d5..19aee9b6a9 100755 --- a/rs/scripts/package-windows.sh +++ b/sh/rs/package-windows.sh @@ -15,7 +15,7 @@ set -euo pipefail # releases, so wingetcreate can carry it forward unchanged. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -RS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +RS_DIR="$(cd "$SCRIPT_DIR/../../rs" && pwd)" WORKSPACE_DIR="$(cd "$RS_DIR/.." && pwd)" CRATE="" diff --git a/sh/rs/package.sh b/sh/rs/package.sh new file mode 100755 index 0000000000..970d7c8ef7 --- /dev/null +++ b/sh/rs/package.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Build a .deb or .rpm for moq-relay or moq-cli locally: sh/rs/package.sh CRATE deb|rpm +# +# nfpm comes from the dev shell. An .rpm built this way links against the +# host's glibc; CI builds in an AlmaLinux 9 container for broad compatibility. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +crate=${1:?usage: sh/rs/package.sh CRATE deb|rpm} +packager=${2:?usage: sh/rs/package.sh CRATE deb|rpm} + +case "$crate" in + moq-relay) bin=moq-relay ;; + moq-cli) bin=moq ;; + *) + echo "Unknown crate: $crate (use moq-relay or moq-cli)" >&2 + exit 1 + ;; +esac +case "$packager" in + deb) + if command -v dpkg >/dev/null 2>&1; then + arch=$(dpkg --print-architecture) + else + case "$(uname -m)" in + x86_64) arch=amd64 ;; + aarch64 | arm64) arch=arm64 ;; + *) + echo "Cannot infer deb arch from host $(uname -m)" >&2 + exit 1 + ;; + esac + fi + ;; + rpm) arch=$(uname -m) ;; + *) + echo "Unknown packager: $packager (use deb or rpm)" >&2 + exit 1 + ;; +esac +version=$(grep -m1 '^version' "rs/$crate/Cargo.toml" | sed 's/.*"\(.*\)".*/\1/') +cargo build --locked --release -p "$crate" +mkdir -p dist +VERSION="$version" ARCH="$arch" BINARY_PATH="target/release/$bin" \ + sh/rs/package-nfpm.sh "packaging/$crate/nfpm.yaml" "$packager" dist/ +if [[ "$packager" == deb && -f "packaging/$crate/transition.yaml" ]]; then + VERSION="$version" ARCH="$arch" \ + sh/rs/package-nfpm.sh "packaging/$crate/transition.yaml" deb dist/ +fi +ls -1 dist/ diff --git a/rs/scripts/scrub-macho.sh b/sh/rs/scrub-macho.sh similarity index 100% rename from rs/scripts/scrub-macho.sh rename to sh/rs/scrub-macho.sh diff --git a/sh/rs/select.sh b/sh/rs/select.sh new file mode 100755 index 0000000000..1724524b4d --- /dev/null +++ b/sh/rs/select.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# Run `just rs check|fix|test` on the crates a diff touches, plus their dependents. +# +# Usage: sh/rs/select.sh check|fix|test LISTFILE|--all +# +# LISTFILE holds the changed paths, one per line. `--all` is the whole +# workspace, as is a change to anything every crate is built or tested by. +set -euo pipefail + +usage="usage: sh/rs/select.sh check|fix|test LISTFILE|--all" +action=${1:?$usage} +list=${2:?$usage} + +cd "$(git rev-parse --show-toplevel)" + +# Print `ALL`, nothing (no crate affected), or cargo package ids. +select_packages() { + if [[ "$list" == --all ]] || grep -qE '^(Cargo\.(toml|lock)|rust-toolchain\.toml|rs/justfile|sh/rs/select\.sh|\.config/nextest\.toml)$' "$list"; then + echo ALL + return + fi + + grep -q '^rs/' "$list" || return 0 + + # `--no-deps` keeps this to workspace members and off the network. + local metadata + metadata=$(cargo metadata --format-version 1 --no-deps) + + # A seed is the first path segment under `rs/`, so only a crate whose + # manifest sits directly at `rs//Cargo.toml` can be one. That drops + # `moq-net-fuzz` (at `rs/moq-net/fuzz`), which a moq-net diff would + # otherwise select as a dependent and compile libFuzzer for. The separator + # is normalized first so Windows paths split too. + metadata=$(jq '.packages |= map((.manifest_path |= gsub("\\\\"; "/")) | select(.manifest_path | split("/")[-3] == "rs"))' <<<"$metadata") + + # A changed path is a seed only because every crate directory is named + # after its crate. Under-selecting checks nothing, so fall back to all. + if jq -e '.packages[] | select((.manifest_path | split("/")[-2]) != .name)' <<<"$metadata" >/dev/null; then + echo "rs: a crate directory no longer matches its crate name; selecting everything." >&2 + echo ALL + return + fi + + local seeds edges selected + seeds=$(sed -n 's|^rs/\([^/]*\)/.*|\1|p' "$list" | sort -u) + edges=$(jq -r '.packages[] | .name as $name | .dependencies[] | "\($name) \(.name)"' <<<"$metadata") + + # A crate is rebuilt when anything it depends on changed, so walk the edges + # backwards until the selection stops growing. Test the value, not the key: + # awk creates a key on every `want[dep[i]]` read. The id lookup below drops + # names that are not workspace crates, such as a seed from `rs/CLAUDE.md`. + selected=$(awk -v seeds="$seeds" ' + BEGIN { split(seeds, s, "\n"); for (i in s) want[s[i]] = 1 } + { pkg[NR] = $1; dep[NR] = $2 } + END { + do { + grew = 0 + for (i = 1; i <= NR; i++) + if (want[dep[i]] && !want[pkg[i]]) { want[pkg[i]] = 1; grew = 1 } + } while (grew) + for (name in want) if (want[name]) print name + } + ' <<<"$edges" | sort -u) + + [[ -n "$selected" ]] || return 0 + + # Cargo's own package ids, not bare names: a workspace crate that is also + # pulled from crates.io (kio) makes `--package kio` ambiguous, and cargo + # refuses before compiling anything. + jq -r --arg selected "$selected" ' + ($selected | split("\n")) as $want + | .packages[] + | select(.name as $name | $want | index($name) != null) + | "--package \(.id)" + ' <<<"$metadata" | sort -u | tr '\n' ' ' +} + +packages=$(select_packages) + +# Ids are `path+file://#` for a directory named after its crate. +names=$(tr ' ' '\n' <<<"$packages" | sed -n 's|.*/\([^/#]*\)#.*|\1|p' | sort -u | tr '\n' ' ') + +# True when the selection includes a crate matching the pattern. +wants() { + [[ "$packages" == ALL ]] || grep -qwE "$1" <<<"$names" +} + +case "$packages" in + "") + echo "rs: no crates affected; skipping." + exit 0 + ;; + ALL) echo "rs: selecting the workspace" ;; + *) echo "rs: selecting $names" ;; +esac + +# `moq-net-fuzz` needs a nightly toolchain; only `just rs fuzz` compiles it. +if [[ "$packages" == ALL ]]; then + flags=(--workspace --exclude moq-net-fuzz) +else + # Space-separated `--package ` pairs; ids contain no spaces. + # shellcheck disable=SC2206 + flags=($packages) +fi + +case "$action" in + check) + just rs check "${flags[@]}" + # Workspace feature unification hides a broken moq-tokio feature set + # whenever any dependent enables a transport. + if wants moq-tokio; then just rs tokio-features; fi + # moq-wasm's crate root is `#![cfg(target_arch = "wasm32")]`, so only a + # wasm32 pass checks it. moq-mux and moq-ffi ride along in that pass, and + # nothing depends on moq-ffi, so each has to reach it on its own. + if wants '(moq-wasm|moq-mux|moq-ffi)'; then just rs wasm; fi + # Device code behind the off-by-default `capture` feature. + if wants '(moq-video|moq-audio)'; then just rs capture; fi + # The relay's io_uring listener, off the default feature set. + if wants moq-relay; then just rs uring-check; fi + ;; + fix) + just rs fix "${flags[@]}" + if wants '(moq-wasm|moq-mux|moq-ffi)'; then just rs wasm-fix; fi + ;; + test) + # A selection can hold nothing testable (moq-wasm has no host tests), and + # nextest exits 4 on that; the whole workspace finding none really is wrong. + [[ "$packages" == ALL ]] || flags+=(--no-tests=pass) + just rs test "${flags[@]}" + if wants moq-relay; then just rs relay-minimal; fi + ;; + *) + echo "$usage" >&2 + exit 2 + ;; +esac diff --git a/rs/scripts/stats-docs.py b/sh/rs/stats-docs.py similarity index 100% rename from rs/scripts/stats-docs.py rename to sh/rs/stats-docs.py diff --git a/sh/rs/tokio-features.sh b/sh/rs/tokio-features.sh new file mode 100755 index 0000000000..b80de4f8e8 --- /dev/null +++ b/sh/rs/tokio-features.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Compile moq-tokio by itself at the feature extremes and with each crypto +# provider. Its default-feature build is already part of the ordinary clippy +# pass; this catches what workspace feature unification hides. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +cargo clippy --locked -p moq-tokio --no-default-features -- -D warnings +cargo clippy --locked -p moq-tokio --all-features -- -D warnings +cargo clippy --locked -p moq-tokio --no-default-features --features noq,aws-lc-rs -- -D warnings +cargo clippy --locked -p moq-tokio --no-default-features --features noq,ring -- -D warnings + +for feature in aws-lc-rs ring tcp uds websocket; do + cargo clippy --locked -p moq-tokio --no-default-features --features "$feature" -- -D warnings +done + +# shellcheck disable=SC2016 # the backticks are literal, as rustc prints them +want='a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature' +if output=$(cargo check --locked -p moq-tokio --no-default-features --features noq 2>&1); then + echo "rs: moq-tokio with only noq should reject its missing crypto provider" >&2 + exit 1 +fi +if ! grep -qF "$want" <<<"$output"; then + echo "rs: moq-tokio with only noq should fail on: $want" >&2 + echo "$output" | grep -E '^error' >&2 || true + exit 1 +fi diff --git a/sh/rs/vulkan-cuda.sh b/sh/rs/vulkan-cuda.sh new file mode 100755 index 0000000000..bcdc12944c --- /dev/null +++ b/sh/rs/vulkan-cuda.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Exercise the Vulkan producer -> CUDA image contract, the GPU color conversion +# and resize, and NVENC encoding of the result on Linux/NVIDIA hardware. The +# driver directory is added explicitly because the Nix shell's dynamic loader +# path does not include Ubuntu's host driver directory. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +driver=$(/usr/sbin/ldconfig -p | awk '/libcuda\.so\.1/{print $NF; exit}') +if [[ -z "$driver" ]]; then + echo "libcuda.so.1 is not installed" >&2 + exit 1 +fi +driver_dir=$(dirname "$driver") +export LD_LIBRARY_PATH="$driver_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +cargo nextest run --locked -p moq-video --run-ignored only -E 'test(/^frame::(vulkan|cuda)::tests::vulkan_cuda_/)' diff --git a/sh/shell.sh b/sh/shell.sh new file mode 100755 index 0000000000..fb6ae022c8 --- /dev/null +++ b/sh/shell.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Lint or format every tracked shell script: sh/shell.sh check|fix +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +scripts=() +while IFS= read -r -d '' file; do + if git --literal-pathspecs ls-files --error-unmatch -- "$file" >/dev/null 2>&1; then + scripts+=("$file") + fi +done < <(shfmt -f=0 .) +((${#scripts[@]})) || exit 0 + +case "${1:-}" in + check) + shfmt --diff "${scripts[@]}" + shellcheck "${scripts[@]}" + ;; + fix) + shfmt --write "${scripts[@]}" + ;; + *) + echo "usage: sh/shell.sh check|fix" >&2 + exit 2 + ;; +esac diff --git a/swift/scripts/check.sh b/sh/swift/check.sh similarity index 98% rename from swift/scripts/check.sh rename to sh/swift/check.sh index afe9644464..bcacf80211 100755 --- a/swift/scripts/check.sh +++ b/sh/swift/check.sh @@ -8,7 +8,7 @@ set -euo pipefail # `cargo`. Intended for `just swift check`. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SWIFT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +SWIFT_DIR="$(cd "$SCRIPT_DIR/../../swift" && pwd)" WORKSPACE_DIR="$(cd "$SWIFT_DIR/.." && pwd)" if ! command -v swift >/dev/null 2>&1; then diff --git a/swift/scripts/package-ffi.sh b/sh/swift/package-ffi.sh similarity index 98% rename from swift/scripts/package-ffi.sh rename to sh/swift/package-ffi.sh index 5958c213ae..8166429c73 100755 --- a/swift/scripts/package-ffi.sh +++ b/sh/swift/package-ffi.sh @@ -12,7 +12,7 @@ set -euo pipefail # Only macOS hosts can run this (xcodebuild is required). # # Usage: -# swift/scripts/package-ffi.sh --version 0.0.0-dev --lib-dir dist --output dist +# sh/swift/package-ffi.sh --version 0.0.0-dev --lib-dir dist --output dist # # --version Version baked into Package.swift (the moq-ffi crate version). # --lib-dir Directory containing per-target moq-ffi outputs. @@ -24,7 +24,7 @@ set -euo pipefail # override when publishing from a fork. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SWIFT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +SWIFT_DIR="$(cd "$SCRIPT_DIR/../../swift" && pwd)" WORKSPACE_DIR="$(cd "$SWIFT_DIR/.." && pwd)" VERSION="" diff --git a/swift/scripts/package.sh b/sh/swift/package.sh similarity index 98% rename from swift/scripts/package.sh rename to sh/swift/package.sh index e1d4052719..4d281925ee 100755 --- a/swift/scripts/package.sh +++ b/sh/swift/package.sh @@ -12,7 +12,7 @@ set -euo pipefail # package depends on at .upToNextMinor(from: ). # # Usage: -# swift/scripts/package.sh --version 0.3.0 --output dist +# sh/swift/package.sh --version 0.3.0 --output dist # # --version Wrapper version (from swift/VERSION). Names the tarball only; # the SPM version comes from the mirror's git tag. @@ -21,7 +21,7 @@ set -euo pipefail # --output Destination directory for the .tar.gz. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SWIFT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +SWIFT_DIR="$(cd "$SCRIPT_DIR/../../swift" && pwd)" WORKSPACE_DIR="$(cd "$SWIFT_DIR/.." && pwd)" VERSION="" diff --git a/swift/scripts/publish-ffi.sh b/sh/swift/publish-ffi.sh similarity index 100% rename from swift/scripts/publish-ffi.sh rename to sh/swift/publish-ffi.sh diff --git a/swift/scripts/publish.sh b/sh/swift/publish.sh similarity index 100% rename from swift/scripts/publish.sh rename to sh/swift/publish.sh diff --git a/swift/scripts/verify-ffi.sh b/sh/swift/verify-ffi.sh similarity index 96% rename from swift/scripts/verify-ffi.sh rename to sh/swift/verify-ffi.sh index 65c8f60d85..886fb75c6d 100755 --- a/swift/scripts/verify-ffi.sh +++ b/sh/swift/verify-ffi.sh @@ -13,8 +13,8 @@ set -euo pipefail # never reaches consumers. # # Usage: -# swift/scripts/verify-ffi.sh --staged-dir -# swift/scripts/verify-ffi.sh --tarball +# sh/swift/verify-ffi.sh --staged-dir +# sh/swift/verify-ffi.sh --tarball # # Exactly one of --staged-dir / --tarball must be passed. diff --git a/swift/scripts/verify.sh b/sh/swift/verify.sh similarity index 96% rename from swift/scripts/verify.sh rename to sh/swift/verify.sh index 865495798d..6df5a561b5 100755 --- a/swift/scripts/verify.sh +++ b/sh/swift/verify.sh @@ -13,8 +13,8 @@ set -euo pipefail # already published (true on a normal release; a correct failure otherwise). # # Usage: -# swift/scripts/verify.sh --staged-dir -# swift/scripts/verify.sh --tarball +# sh/swift/verify.sh --staged-dir +# sh/swift/verify.sh --tarball # # Exactly one of --staged-dir / --tarball must be passed. diff --git a/swift/.gitignore b/swift/.gitignore index 17554c75de..2a28e6b5cc 100644 --- a/swift/.gitignore +++ b/swift/.gitignore @@ -3,7 +3,7 @@ .swiftpm/ Package.resolved -# Generated by swift/scripts/{check,package}.sh — never check these in. +# Generated by sh/swift/{check,package}.sh — never check these in. Sources/MoqFFI/Generated.swift MoqFFI.xcframework/ diff --git a/swift/.spi.yml b/swift/.spi.yml index 0259deae9c..9fb2ec6db2 100644 --- a/swift/.spi.yml +++ b/swift/.spi.yml @@ -3,7 +3,7 @@ # served at https://swiftpackageindex.com/moq-dev/moq-swift/documentation. # # This file is copied into the published moq-dev/moq-swift mirror by -# swift/scripts/package.sh, since SPI reads it from the package root. +# sh/swift/package.sh, since SPI reads it from the package root. version: 1 builder: configs: diff --git a/swift/Package.swift b/swift/Package.swift index 97256a74c9..e06be8154a 100644 --- a/swift/Package.swift +++ b/swift/Package.swift @@ -1,9 +1,9 @@ // swift-tools-version:5.9 -// Local-development manifest. swift/scripts/check.sh builds a single-slice +// Local-development manifest. sh/swift/check.sh builds a single-slice // MoqFFI.xcframework next to this file so the path-based binaryTarget below // resolves locally without a network fetch. // -// The released form (URL + checksum) is generated by swift/scripts/package.sh +// The released form (URL + checksum) is generated by sh/swift/package.sh // from Package.swift.template at release time. Edit the template, not this // file, when changing the released manifest. diff --git a/swift/Package.swift.template b/swift/Package.swift.template index 0cdfbf0606..f6339a07a8 100644 --- a/swift/Package.swift.template +++ b/swift/Package.swift.template @@ -1,7 +1,7 @@ // swift-tools-version:5.9 // Released manifest for the ergonomic wrapper at moq-dev/moq-swift. The // source-of-truth template lives at swift/Package.swift.template in -// moq-dev/moq; swift/scripts/package.sh substitutes the moq-ffi version pin +// moq-dev/moq; sh/swift/package.sh substitutes the moq-ffi version pin // (REPLACE_FFI_VERSION) at release time. // // The wrapper versions independently of the bindings (see swift/VERSION). The diff --git a/swift/README.md b/swift/README.md index d6c1779213..b22d5a8910 100644 --- a/swift/README.md +++ b/swift/README.md @@ -82,7 +82,7 @@ is an `AsyncSequence` of the decoded type. Pass matching `compression` on both s ## Local development -`swift/scripts/check.sh` builds `moq-ffi` for the host, regenerates the UniFFI Swift bindings, builds a single-slice `MoqFFI.xcframework`, and runs `swift test`. Requires macOS with `xcodebuild` and `swift` on `$PATH`. Run via `just swift check`; skips cleanly on non-macOS hosts. +`sh/swift/check.sh` builds `moq-ffi` for the host, regenerates the UniFFI Swift bindings, builds a single-slice `MoqFFI.xcframework`, and runs `swift test`. Requires macOS with `xcodebuild` and `swift` on `$PATH`. Run via `just swift check`; skips cleanly on non-macOS hosts. Local development uses one **monolithic** `Package.swift` containing both the `Moq` and `MoqFFI` targets plus the path-based XCFramework, so `swift test` and Xcode work against a single package. The split into two packages exists only in the released artifacts, assembled from the two templates below at release time. Because the FFI module is named `MoqFFI` in both layouts, the wrapper sources (`import MoqFFI`) compile identically either way. @@ -98,7 +98,6 @@ swift/ Moq/ Ergonomic wrapper (Client, Server, Origin, Broadcast, Track, Media, Audio, …) MoqFFI/ UniFFI-generated swift (populated by check.sh/package-ffi.sh, gitignored) Tests/MoqTests/ Smoke tests - scripts/ check.sh, package{,-ffi}.sh, verify{,-ffi}.sh, publish{,-ffi}.sh ``` Edit the templates when changing a released manifest; never copy the monolithic dev-mode form into the release path. @@ -117,8 +116,8 @@ To release a new wrapper version: bump `swift/VERSION` in a PR. On merge, `relea To dry-run a publish locally against a staged tarball: ```bash -BUILD_VERSION= ./swift/scripts/publish.sh --dry-run # wrapper -> moq-swift -BUILD_VERSION= ./swift/scripts/publish-ffi.sh --dry-run # bindings -> moq-swift-ffi +BUILD_VERSION= ./sh/swift/publish.sh --dry-run # wrapper -> moq-swift +BUILD_VERSION= ./sh/swift/publish-ffi.sh --dry-run # bindings -> moq-swift-ffi ``` No Apple Developer account or App Store Connect setup needed. diff --git a/swift/ffi/Package.swift.template b/swift/ffi/Package.swift.template index d66fb3d01b..55588ffc09 100644 --- a/swift/ffi/Package.swift.template +++ b/swift/ffi/Package.swift.template @@ -1,7 +1,7 @@ // swift-tools-version:5.9 // Released manifest for the raw UniFFI bindings package at moq-dev/moq-swift-ffi. // The source-of-truth template lives at swift/ffi/Package.swift.template in -// moq-dev/moq; swift/scripts/package-ffi.sh substitutes the xcframework URL and +// moq-dev/moq; sh/swift/package-ffi.sh substitutes the xcframework URL and // SHA-256 (REPLACE_URL / REPLACE_CHECKSUM) at release time. // // Lockstep with the moq-ffi Rust crate: each moq-ffi-v* tag publishes a matching diff --git a/swift/justfile b/swift/justfile index 83b585910d..acb2b15029 100644 --- a/swift/justfile +++ b/swift/justfile @@ -9,29 +9,13 @@ default: just check # Skips cleanly on non-macOS hosts, which is why swift.yml runs it on a Mac. -# -# This runs the tests too, so there is no separate `just swift test`. Takes an -# optional newline-separated list of changed files and skips when none are -# Swift-scoped; `just swift check` (no FILES) always runs. +# This runs the tests too, so there is no separate `just swift test`. # Build moq-ffi for the host, regenerate uniffi bindings, run swift test. -check $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '^(swift/|rs/moq-ffi/)' <<< "$FILES"; then - echo "swift: no Swift changes; skipping." - exit 0 - fi - bash scripts/check.sh +check: + bash ../sh/swift/check.sh # Assemble the released wrapper package from Package.swift.template. - # Used by .github/workflows/release-swift-lib.yml. package *args: - bash scripts/package.sh {{ args }} - -# Remove SwiftPM build output plus the XCFramework and generated bindings - -# that scripts/check.sh lays down (see swift/.gitignore). -clean: - rm -rf .build .swiftpm Package.resolved Sources/MoqFFI/Generated.swift MoqFFI.xcframework + bash ../sh/swift/package.sh {{ args }} diff --git a/test/README.md b/test/README.md index 5c4f994c43..94d044a773 100644 --- a/test/README.md +++ b/test/README.md @@ -84,25 +84,3 @@ with the `rm -rf` the run prints; nothing expires it for you. In CI the harness writes under `MOQ_TEST_RUNS`, and `interop.yml` and `wasm.yml` upload that directory as a short-lived artifact when the job fails. - -## Worktrees - -`just worktree` reports what a checkout can actually do before anything is built: -its base and how stale that base is, and whether the Git metadata is reachable for -fetch, branch creation, and rebase. A linked worktree keeps its shared metadata -under `--git-common-dir`, inside the main repository, so write access to the -source tree does not imply any of the three. - -```bash -just worktree # report, change nothing -just worktree setup # fetch, set the branch upstream, record the base SHA -``` - -`setup` never resets, rebases, or cleans, so it is safe to run against a checkout -with work in progress. - -The upstream is the only place the base survives, so `setup BASE` fails when it -cannot be written -- a detached HEAD has no branch to hang it on, and the shared -config may be read-only. Left as a warning, `just check` would go on scoping -against `origin/main` while setup reported success. Without a `BASE`, that -fallback is what would have been written anyway, so it warns instead. diff --git a/test/interop/README.md b/test/interop/README.md index e41a0d7179..e2641b7aa7 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -29,7 +29,7 @@ player survive the publication lifecycle. See [Media QA](#media-qa). |---|---|---|---| | Rust | `rs/moq-relay` + `rs/moq-cli` | `cargo build` | publish (video) + subscribe | | Python | `py/moq-rs` (+ `rs/moq-ffi`, import `moq`) | `just py build` (maturin editable into `.venv`) | publish (video + audio) + subscribe | -| Go | `go/wrapper` (+ `rs/moq-ffi`, import `moq-go/moq`) | `go/scripts/stage.sh` (uniffi-bindgen-go) + `go build` | publish (video + audio) + subscribe | +| Go | `go/wrapper` (+ `rs/moq-ffi`, import `moq-go/moq`) | `sh/go/stage.sh` (uniffi-bindgen-go) + `go build` | publish (video + audio) + subscribe | | Browser | `js/watch` + `js/publish` | `vite build` + headless Chromium (Playwright) | publish (video + audio) + rendered playback | | Native JS | `js/net` + `js/hang` + the npm `@moq/web-transport` polyfill | `node` (tsx) and `bun` | subscribe | | C | `rs/libmoq` | `cargo build -p libmoq` + `cc` | subscribe | @@ -40,7 +40,7 @@ The browser, native JS, C, and GStreamer clients subscribe only by choice intentionally minimal, and `moqsink` publishing needs request-pad muxing this client doesn't drive). Rust, Python, Go, and the browser publish. -The Go client builds against the modules `go/scripts/stage.sh` assembles from +The Go client builds against the modules `sh/go/stage.sh` assembles from this checkout: `moq-ffi` compiled for the host, bindings regenerated with `uniffi-bindgen-go`, and the `go/wrapper` module wired to them by a `replace`. That is the same staging `just go check` uses, so this cell covers the Go diff --git a/test/interop/clients/go/go.mod b/test/interop/clients/go/go.mod index a57ce62b53..bd1ae2e204 100644 --- a/test/interop/clients/go/go.mod +++ b/test/interop/clients/go/go.mod @@ -1,6 +1,6 @@ // Not a published module. test/interop/interop.sh builds this out of a scratch copy // and rewrites the require below into a `replace` pointing at the wrapper that -// go/scripts/stage.sh assembles from this checkout, so nothing resolves from the +// sh/go/stage.sh assembles from this checkout, so nothing resolves from the // module proxy. The placeholder version keeps the committed file honest: there // is no such tag, and there is no go.sum because every dependency is local. module moq.dev/interop diff --git a/test/interop/clients/js-native/subscribe.ts b/test/interop/clients/js-native/subscribe.ts index 61df7fdeac..1cdfcd8e67 100644 --- a/test/interop/clients/js-native/subscribe.ts +++ b/test/interop/clients/js-native/subscribe.ts @@ -40,6 +40,9 @@ if (role !== "subscribe" || !url || !broadcast || !Number.isFinite(timeoutMs) || process.exit(2); } +// Group resets that only mean content was skipped at the live edge. +const GAPS: Moq.StreamCode[] = [Moq.StreamCode.TooFarBehind, Moq.StreamCode.Old, Moq.StreamCode.Evicted]; + async function run(): Promise { const origin = new Moq.Origin.Producer(); const connection = await Moq.Connection.connect({ url: new URL(url as string), consume: origin }); @@ -72,7 +75,15 @@ async function run(): Promise { const group = await video.recvGroup(); if (!group) break; for (;;) { - const frame = await group.readFrame(); + let frame: Moq.Group.Frame | undefined; + try { + frame = await group.readFrame(); + } catch (err) { + // A live subscriber can be handed a group the relay then drops for a newer one; + // that is a gap, not a failure, so move on to the next group. + if (!(err instanceof Moq.Error.Stream && GAPS.includes(err.code))) throw err; + break; + } if (!frame) break; total += frame.payload.byteLength; if (total > 0) { diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 713d4e6865..13a50032c8 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -80,6 +80,14 @@ while [[ $# -gt 0 ]]; do MEDIA=1 shift ;; + # The full matrix. --timeout 30 gives headless Chromium cold-start + # headroom; flags after it still override. + --all) + PUBLISHERS="rust,python,go,js" + SUBSCRIBERS="rust,python,go,js,js-native-node,js-native-bun,c,gst" + TIMEOUT=30 + shift + ;; *) echo "unknown arg: $1" >&2 exit 2 @@ -240,7 +248,7 @@ prepare_js() { fi } -# Stage the Go modules from this checkout (go/scripts/stage.sh builds moq-ffi for +# Stage the Go modules from this checkout (sh/go/stage.sh builds moq-ffi for # the host, regenerates the bindings, and wires the wrapper to them by replace), # then build the interop client against that exact tree. The client is copied to a # scratch dir first so the committed go.mod keeps its placeholder require; every @@ -256,8 +264,8 @@ prepare_go() { } echo "building go client (workspace moq-go via uniffi-bindgen-go)..." local staged ffi_pkg wrapper_pkg src="$HARNESS_RUN/go-client" - if ! staged=$(bash "$WORKSPACE/go/scripts/stage.sh" 2>"$HARNESS_RUN/go-stage.log"); then - mark_broken go "go/scripts/stage.sh failed" + if ! staged=$(bash "$WORKSPACE/sh/go/stage.sh" 2>"$HARNESS_RUN/go-stage.log"); then + mark_broken go "sh/go/stage.sh failed" sed 's/^/ /' "$HARNESS_RUN/go-stage.log" >&2 || true return fi diff --git a/test/justfile b/test/justfile index fb0ecbf168..ed94847ff8 100644 --- a/test/justfile +++ b/test/justfile @@ -3,89 +3,37 @@ # Tests that span more than one language. Per-language unit tests live in each # language's own justfile; this module aggregates them. # -# Shebang recipes read their arguments as $1..$n. -set positional-arguments - # Two cross-language interop tests exist: # - https://github.com/moq-dev/smoke installs each client from its PUBLIC # package registry to catch packaging breakage in a release. # - `just test interop` (interop/, below) builds every client from THIS checkout # to catch interop regressions before anything is published. -# Fall back to the root justfile so `just js test` (etc.) resolve from here. +# Fall back to the root justfile so `just rs test` resolves from here. set fallback set working-directory := '.' -# `interop` reads its args as `$1`/`$@`; the `positional-arguments` setting above -# is what lets a shebang recipe see them. # The `check` sibling: same diff scoping, same BASE resolution, and what CI runs -# after `just check`. Only js/rs/py appear here. The kt, swift, and go wrappers -# build, regenerate bindings, and test in a single script, so their tests run as -# part of `just check` and there is nothing left to dispatch. -# -# `just _changed` resolves through `set fallback` to the root justfile, which is -# also where it runs, so the paths it prints stay repo-relative. -# -# BASE has to be passed as `just test default origin/dev`. This is a module, so -# a bare `just test origin/dev` makes just look for a recipe by that name rather -# than reaching the default recipe's parameter. +# after `just check`. Only js/rs/py have tests to scope; the kt, swift, and go +# wrappers test inside their `check`. BASE has to be passed as `just test +# default origin/dev`: a bare `just test origin/dev` names a recipe. # Run unit tests for what the branch changed since BASE, plus its dependents. default $BASE="": - #!/usr/bin/env bash - set -euo pipefail - - files=$(just _changed "$BASE") - - # Mirrors the root `check`: too long for argv means test everything instead. - if [[ "$files" == ALL ]]; then - just all - exit 0 - fi - - just _tools "$files" - - # Mirrors the root `check`: the dispatch below lives in these two files and - # matches no language scope, so a change to either has to run everything or - # it goes untested. - if grep -qE '^(justfile|test/justfile)$' <<< "$files"; then - echo "test: root orchestration changed; testing everything." >&2 - just all - exit 0 - fi - - # An empty list means "force-run" to the per-lang recipes, which is the - # wrong semantic here, so don't dispatch at all. - if [[ -n "$files" ]]; then - just js test "$files" - just rs test-changed "$files" - just py test "$files" - else - echo "test: nothing changed." - fi + ../sh/dispatch.sh test "$BASE" # Run every language's unit tests, unscoped. all: - just js test - just rs test --workspace --exclude moq-net-fuzz - just py test + ../sh/dispatch.sh test --all # Cross-language media interop test, built from this checkout. Stands up a # relay and runs the publisher x subscriber matrix. Default: rust only (fast # sanity check). `interop --all` runs the full matrix: rust + python + go + # browser publish; everyone subscribes (incl. native node/bun JS, the C libmoq -# client, and the GStreamer moqsrc plugin). --timeout 30 gives headless Chromium -# cold-start headroom. Other flags pass through, e.g. +# client, and the GStreamer moqsrc plugin). Other flags pass through, e.g. # `just test interop --publishers rust,python --subscribers rust,c`. interop *args: - #!/usr/bin/env bash - set -euo pipefail - if [[ "${1:-}" == --all ]]; then - shift - ./interop/interop.sh --publishers rust,python,go,js --subscribers rust,python,go,js,js-native-node,js-native-bun,c,gst --timeout 30 "$@" - else - ./interop/interop.sh "$@" - fi + ./interop/interop.sh {{ args }} # Negative control: no publisher, every subscriber must time out (proves the diff --git a/test/wasm/README.md b/test/wasm/README.md index a1eb599b49..e9ab1301de 100644 --- a/test/wasm/README.md +++ b/test/wasm/README.md @@ -21,7 +21,7 @@ just test wasm just test wasm --timeout 60 ``` -Everything is built from this checkout: `just wasm` for the bindings, `cargo +Everything is built from this checkout: `just js wasm` for the bindings, `cargo build -p moq-relay` for the relay. Each relay reserves its own port (see [the harness contract](../README.md)); `WASM_PORT` pins the first one instead. `WASM_PROFILE` picks the relay's cargo profile, and `RELAY_BIN` points at a diff --git a/test/wasm/run.sh b/test/wasm/run.sh index e2e649743e..71ed51ba53 100755 --- a/test/wasm/run.sh +++ b/test/wasm/run.sh @@ -92,7 +92,7 @@ command -v wasm-bindgen >/dev/null 2>&1 || { # ── build ─────────────────────────────────────────────────────────────────── echo "building @moq/wasm..." -(cd "$WORKSPACE" && just wasm) +(cd "$WORKSPACE" && just js wasm) echo "building moq-relay ($PROFILE)..." flag=() From c5000beb3c27218dd28d11aa1d9f3209949c883c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 24 Sep 2026 23:44:54 -0700 Subject: [PATCH 3/6] chore(ci): every workflow step runs a just recipe (#4098) Co-authored-by: Claude Opus 5.5 --- .github/actions/just/action.yml | 11 +++++++ .github/justfile | 29 ++++++++++++++++- .github/workflows/alert.yml | 4 ++- .github/workflows/android.yml | 7 +++-- .github/workflows/apt-repo.yml | 5 +-- .github/workflows/check.yml | 3 -- .github/workflows/docker.yml | 4 ++- .github/workflows/libmoq.yml | 26 ++++++++++------ .github/workflows/moq-cli.yml | 34 ++++++++++++-------- .github/workflows/moq-gst.yml | 26 ++++++++++------ .github/workflows/moq-relay.yml | 32 +++++++++++-------- .github/workflows/nightly.yml | 15 ++------- .github/workflows/release-brew.yml | 6 ++-- .github/workflows/release-dart-ffi.yml | 18 ++++++++--- .github/workflows/release-dart.yml | 6 ++-- .github/workflows/release-go-ffi.yml | 24 +++++++++++---- .github/workflows/release-go.yml | 10 ++++-- .github/workflows/release-kt-ffi.yml | 16 +++++++--- .github/workflows/release-kt-lib.yml | 14 ++++++--- .github/workflows/release-py-ffi.yml | 4 ++- .github/workflows/release-py.yml | 10 +++--- .github/workflows/release-swift-ffi.yml | 30 +++++++++++++----- .github/workflows/release-swift-lib.yml | 24 ++++++++++----- .github/workflows/rpm-repo.yml | 5 +-- .github/workflows/swift.yml | 4 ++- CONTRIBUTING.md | 4 +++ cpp/obs/justfile | 9 ++++++ dart/justfile | 9 ++++++ go/ffi/README.md | 2 +- go/justfile | 33 +++++++++++++++++--- js/justfile | 5 +++ kt/justfile | 11 ++++--- quest/m1/tooling/README.md | 1 - quest/m1/tooling/release-binary.md | 4 --- quest/m1/tooling/workflows-call-just.md | 41 ------------------------- rs/justfile | 35 +++++++++++++++++++++ sh/gh/check-runs.ts | 26 ++++++++++++++++ swift/justfile | 38 +++++++++++++++++++++-- 38 files changed, 409 insertions(+), 176 deletions(-) create mode 100644 .github/actions/just/action.yml delete mode 100644 quest/m1/tooling/workflows-call-just.md create mode 100644 sh/gh/check-runs.ts diff --git a/.github/actions/just/action.yml b/.github/actions/just/action.yml new file mode 100644 index 0000000000..f52ffe41e3 --- /dev/null +++ b/.github/actions/just/action.yml @@ -0,0 +1,11 @@ +name: Install just +description: Install the pinned just for jobs outside the nix dev shell, which already provides it. + +runs: + using: composite + steps: + # A checksum-verified prebuilt, so it costs seconds rather than a + # `cargo install`. Keep the version in step with nixpkgs' just in flake.lock. + - uses: taiki-e/install-action@9983c65e42da123ff25d1f78505eb6de315aa172 # v2.87.20 + with: + tool: just@1.58.0 diff --git a/.github/justfile b/.github/justfile index 8a3fb9ff67..1bccfc5338 100644 --- a/.github/justfile +++ b/.github/justfile @@ -1,9 +1,10 @@ set working-directory := '..' -# Lint GitHub Actions workflows and their alert coverage. +# Lint GitHub Actions workflows, their alert coverage, and that every step runs a recipe. check: actionlint sh/gh/alert.sh check-coverage + bun sh/gh/check-runs.ts # Tests of the release tooling itself; nightly, not on every pull request. test: @@ -11,3 +12,29 @@ test: sh/gh/release.test.sh python3 sh/rs/package-rename.test.py sh/go/publish-wrapper.test.sh + bash .claude/hooks/direnv.test.sh + +# Release workflow helpers, e.g. `just gh release parse-version moq-cli`. +[positional-arguments] +release *args: + sh/gh/release.sh "$@" + +# Post a failed run to Discord, or check that alert.yml covers every workflow. +[positional-arguments] +alert *args: + sh/gh/alert.sh "$@" + +# Render a Homebrew formula template for a release. +[positional-arguments] +formula *args: + sh/gh/render-formula.sh "$@" + +# Copy a rendered formula into a Homebrew tap checkout, migrating renamed formulae. +[positional-arguments] +install-formula *args: + python3 sh/gh/install-formula.py "$@" + +# Trigger the apt-repo and rpm-repo workflows for a release tag. +[positional-arguments] +trigger-repo-publish *args: + sh/gh/trigger-repo-publish.sh "$@" diff --git a/.github/workflows/alert.yml b/.github/workflows/alert.yml index 78d9c48125..006dfad7f7 100644 --- a/.github/workflows/alert.yml +++ b/.github/workflows/alert.yml @@ -78,6 +78,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Post to Discord env: DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} @@ -88,4 +90,4 @@ jobs: RUN_REF: ${{ github.event.workflow_run.head_branch }} RUN_TITLE: ${{ github.event.workflow_run.display_title }} RUN_ACTOR: ${{ github.event.workflow_run.triggering_actor.login }} - run: sh/gh/alert.sh discord + run: just gh alert discord diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 9b8e9b4d85..44c6e807a4 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -50,7 +50,6 @@ concurrency: env: CARGO_NDK_VERSION: 4.1.2 - JUST_VERSION: 1.52.0 jobs: android: @@ -74,8 +73,10 @@ jobs: # No Rust cache: this workflow only runs on pull requests, which may not # save one, so it would have nothing to restore. - - name: Install just and cargo-ndk - run: cargo install --locked "just@$JUST_VERSION" "cargo-ndk@$CARGO_NDK_VERSION" + - uses: ./.github/actions/just + + - name: Install cargo-ndk + run: cargo install --locked "cargo-ndk@$CARGO_NDK_VERSION" - name: Check run: just rs android diff --git a/.github/workflows/apt-repo.yml b/.github/workflows/apt-repo.yml index ff790e1583..5e8905ed21 100644 --- a/.github/workflows/apt-repo.yml +++ b/.github/workflows/apt-repo.yml @@ -34,6 +34,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Set up packaging tools uses: ./.github/actions/setup-packaging @@ -53,10 +55,9 @@ jobs: - name: Publish to R2 shell: bash env: - ARTIFACTS_DIR: artifacts R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} SIGNING_KEY: ${{ secrets.SIGNING_KEY }} SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} - run: ./infra/apt/publish.sh + run: just infra apt publish "$GITHUB_WORKSPACE/artifacts" diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 4b26c72456..5cc62c44b7 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -73,9 +73,6 @@ jobs: # Locally an absent formatter means less gets checked; here it would mean # a green run that checked nothing, which is the failure mode worth # spending a preflight on. - - name: Test direnv hook - run: bash .claude/hooks/direnv.test.sh - - name: Check run: nix develop --command just ci check env: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9f2ba00f14..5fcbc152e4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -39,6 +39,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - id: parse env: CRATE: ${{ inputs.package }} @@ -58,7 +60,7 @@ jobs: *) package="$crate" ;; esac echo "package=${package}" >> "$GITHUB_OUTPUT" - sh/gh/release.sh parse-version "$crate" + just gh release parse-version "$crate" build: name: Build ${{ matrix.platform }} diff --git a/.github/workflows/libmoq.yml b/.github/workflows/libmoq.yml index 3ad2daf514..b74274da81 100644 --- a/.github/workflows/libmoq.yml +++ b/.github/workflows/libmoq.yml @@ -40,6 +40,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Install Nix if: matrix.use_nix uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main @@ -55,12 +57,12 @@ jobs: - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version libmoq + run: just gh release parse-version libmoq - name: Build and package shell: bash run: | - ./rs/libmoq/build.sh \ + just rs package-libmoq \ --target ${{ matrix.target }} \ --version ${{ steps.parse.outputs.version }} \ --output dist @@ -85,13 +87,15 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version libmoq + run: just gh release parse-version libmoq - name: Find previous tag id: prev_tag - run: sh/gh/release.sh prev-tag libmoq + run: just gh release prev-tag libmoq - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -105,7 +109,7 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "libmoq v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: sh/gh/release.sh create artifacts + run: just gh release create artifacts # The OBS plugin (cpp/obs) statically links libmoq, so it ships with every # libmoq release: rebuild it against the release we just published (no @@ -137,10 +141,12 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version libmoq + run: just gh release parse-version libmoq # Native Xcode / Visual Studio; libobs/Qt6/ffmpeg come from the obs-deps # bundle and libmoq from the release we just published (--libmoq-release). @@ -150,7 +156,7 @@ jobs: TARGET: ${{ matrix.target }} VERSION: ${{ steps.parse.outputs.version }} run: | - ./cpp/obs/build.sh --target "$TARGET" --libmoq-release "$VERSION" --output dist + just obs package --target "$TARGET" --libmoq-release "$VERSION" --output dist - name: Upload artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -171,9 +177,11 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version libmoq + run: just gh release parse-version libmoq # Only the obs-moq archives: the libmoq-* artifacts from the build job # belong to the libmoq release, not this one. @@ -208,4 +216,4 @@ jobs: RELEASE_TAG: "obs-moq-v${{ steps.parse.outputs.version }}" RELEASE_TITLE: "obs-moq v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: sh/gh/release.sh create artifacts + run: just gh release create artifacts diff --git a/.github/workflows/moq-cli.yml b/.github/workflows/moq-cli.yml index 9e82930991..3975e32663 100644 --- a/.github/workflows/moq-cli.yml +++ b/.github/workflows/moq-cli.yml @@ -39,6 +39,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -47,7 +49,7 @@ jobs: - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version moq-cli + run: just gh release parse-version moq-cli - name: Set up packaging tools uses: ./.github/actions/setup-packaging @@ -64,7 +66,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} TARGET: ${{ matrix.target }} run: | - ./sh/rs/package-binary.sh \ + just rs package-binary \ --crate moq-cli \ --bin moq \ --binary "target/${TARGET}/release/moq" \ @@ -80,8 +82,8 @@ jobs: ARCH: ${{ matrix.deb-arch }} BINARY_PATH: target/${{ matrix.target }}/release/moq run: | - sh/rs/package-nfpm.sh packaging/moq-cli/nfpm.yaml deb dist/ - sh/rs/package-nfpm.sh packaging/moq-cli/transition.yaml deb dist/ + just rs package-nfpm packaging/moq-cli/nfpm.yaml deb dist/ + just rs package-nfpm packaging/moq-cli/transition.yaml deb dist/ - name: Package .rpm shell: bash @@ -89,7 +91,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} ARCH: ${{ matrix.rpm-arch }} BINARY_PATH: target/${{ matrix.target }}/release/moq - run: sh/rs/package-nfpm.sh packaging/moq-cli/nfpm.yaml rpm dist/ + run: just rs package-nfpm packaging/moq-cli/nfpm.yaml rpm dist/ - name: Upload artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -114,6 +116,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main with: determinate: false @@ -121,12 +125,12 @@ jobs: - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version moq-cli + run: just gh release parse-version moq-cli - name: Build and package tarball shell: bash run: | - ./sh/rs/package-binary.sh \ + just rs package-binary \ --crate moq-cli \ --bin moq \ --target ${{ matrix.target }} \ @@ -154,6 +158,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -169,12 +175,12 @@ jobs: - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version moq-cli + run: just gh release parse-version moq-cli - name: Build and package zip shell: bash run: | - ./sh/rs/package-windows.sh \ + just rs package-windows \ --crate moq-cli \ --bin moq \ --target ${{ matrix.target }} \ @@ -202,13 +208,15 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version moq-cli + run: just gh release parse-version moq-cli - name: Find previous tag id: prev_tag - run: sh/gh/release.sh prev-tag moq-cli + run: just gh release prev-tag moq-cli - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -222,10 +230,10 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "moq-cli v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: sh/gh/release.sh create artifacts + run: just gh release create artifacts - name: Trigger apt/rpm repo publish env: GH_TOKEN: ${{ github.token }} TAG: ${{ github.ref_name }} - run: sh/gh/trigger-repo-publish.sh + run: just gh trigger-repo-publish diff --git a/.github/workflows/moq-gst.yml b/.github/workflows/moq-gst.yml index 139d066c0f..dd0a3218e5 100644 --- a/.github/workflows/moq-gst.yml +++ b/.github/workflows/moq-gst.yml @@ -34,6 +34,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main with: determinate: false @@ -41,7 +43,7 @@ jobs: - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version moq-gst + run: just gh release parse-version moq-gst - name: Build and package (tarball) shell: bash @@ -49,7 +51,7 @@ jobs: TARGET: ${{ matrix.target }} VERSION: ${{ steps.parse.outputs.version }} run: | - ./rs/moq-gst/build.sh \ + just rs package-gst \ --target "$TARGET" \ --version "$VERSION" \ --output dist @@ -80,7 +82,7 @@ jobs: run: | set -euo pipefail tar -xzf "dist/moq-gst-v${VERSION}-${TARGET}.tar.gz" -C dist/ - ./rs/moq-gst/smoke.sh "dist/moq-gst-v${VERSION}-${TARGET}/lib/gstreamer-1.0" + just rs gst-smoke "dist/moq-gst-v${VERSION}-${TARGET}/lib/gstreamer-1.0" - name: Upload artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -112,6 +114,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -123,7 +127,7 @@ jobs: - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version moq-gst + run: just gh release parse-version moq-gst - name: Install GStreamer dev libraries shell: bash @@ -160,7 +164,7 @@ jobs: PLUGIN_DIR: /usr/lib/${{ matrix.deb-multiarch }}/gstreamer-1.0 run: | mkdir -p dist - sh/rs/package-nfpm.sh packaging/moq-gst/nfpm.yaml deb dist/ + just rs package-nfpm packaging/moq-gst/nfpm.yaml deb dist/ - name: Package .rpm shell: bash @@ -172,7 +176,7 @@ jobs: PLUGIN_DIR: /usr/lib64/gstreamer-1.0 run: | mkdir -p dist - sh/rs/package-nfpm.sh packaging/moq-gst/nfpm.yaml rpm dist/ + just rs package-nfpm packaging/moq-gst/nfpm.yaml rpm dist/ - name: Upload artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -195,13 +199,15 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version moq-gst + run: just gh release parse-version moq-gst - name: Find previous tag id: prev_tag - run: sh/gh/release.sh prev-tag moq-gst + run: just gh release prev-tag moq-gst - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -215,10 +221,10 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "moq-gst v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: sh/gh/release.sh create artifacts + run: just gh release create artifacts - name: Trigger apt/rpm repo publish env: GH_TOKEN: ${{ github.token }} TAG: ${{ github.ref_name }} - run: sh/gh/trigger-repo-publish.sh + run: just gh trigger-repo-publish diff --git a/.github/workflows/moq-relay.yml b/.github/workflows/moq-relay.yml index 96496d9731..b9ac806f96 100644 --- a/.github/workflows/moq-relay.yml +++ b/.github/workflows/moq-relay.yml @@ -42,6 +42,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -50,7 +52,7 @@ jobs: - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version moq-relay + run: just gh release parse-version moq-relay - name: Set up packaging tools uses: ./.github/actions/setup-packaging @@ -67,7 +69,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} TARGET: ${{ matrix.target }} run: | - ./sh/rs/package-binary.sh \ + just rs package-binary \ --crate moq-relay \ --bin moq-relay \ --binary "target/${TARGET}/release/moq-relay" \ @@ -82,7 +84,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} ARCH: ${{ matrix.deb-arch }} BINARY_PATH: target/${{ matrix.target }}/release/moq-relay - run: sh/rs/package-nfpm.sh packaging/moq-relay/nfpm.yaml deb dist/ + run: just rs package-nfpm packaging/moq-relay/nfpm.yaml deb dist/ - name: Package .rpm shell: bash @@ -90,7 +92,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} ARCH: ${{ matrix.rpm-arch }} BINARY_PATH: target/${{ matrix.target }}/release/moq-relay - run: sh/rs/package-nfpm.sh packaging/moq-relay/nfpm.yaml rpm dist/ + run: just rs package-nfpm packaging/moq-relay/nfpm.yaml rpm dist/ - name: Upload artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -115,6 +117,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main with: determinate: false @@ -122,12 +126,12 @@ jobs: - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version moq-relay + run: just gh release parse-version moq-relay - name: Build and package tarball shell: bash run: | - ./sh/rs/package-binary.sh \ + just rs package-binary \ --crate moq-relay \ --target ${{ matrix.target }} \ --version ${{ steps.parse.outputs.version }} \ @@ -154,6 +158,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -169,12 +175,12 @@ jobs: - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version moq-relay + run: just gh release parse-version moq-relay - name: Build and package zip shell: bash run: | - ./sh/rs/package-windows.sh \ + just rs package-windows \ --crate moq-relay \ --target ${{ matrix.target }} \ --version ${{ steps.parse.outputs.version }} \ @@ -201,13 +207,15 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version moq-relay + run: just gh release parse-version moq-relay - name: Find previous tag id: prev_tag - run: sh/gh/release.sh prev-tag moq-relay + run: just gh release prev-tag moq-relay - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -221,10 +229,10 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "moq-relay v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: sh/gh/release.sh create artifacts + run: just gh release create artifacts - name: Trigger apt/rpm repo publish env: GH_TOKEN: ${{ github.token }} TAG: ${{ github.ref_name }} - run: sh/gh/trigger-repo-publish.sh + run: just gh trigger-repo-publish diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 431d89002e..3341c6d040 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -73,9 +73,7 @@ jobs: # over independent observers of the new live broadcast map. - name: JS origin broadcast map benchmark if: ${{ !cancelled() }} - run: | - nix develop --command bun install --frozen-lockfile - nix develop --command bun js/net/bench/broadcasts.ts + run: nix develop --command just js bench - name: Audit if: ${{ !cancelled() }} @@ -286,11 +284,7 @@ jobs: choco install nasm -y --no-progress "C:\Program Files\NASM" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - # Pinned: `--locked` fixes just's own dependencies, not which version of - # just cargo selects. - - name: Install just - shell: bash - run: cargo install --locked just@1.52.0 + - uses: ./.github/actions/just - name: Check shell: bash @@ -323,10 +317,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - # Pinned: `--locked` fixes just's own dependencies, not which version of - # just cargo selects. - - name: Install just - run: cargo install --locked just@1.52.0 + - uses: ./.github/actions/just - name: Check run: just rs macos diff --git a/.github/workflows/release-brew.yml b/.github/workflows/release-brew.yml index fb06fe444f..5e8594de05 100644 --- a/.github/workflows/release-brew.yml +++ b/.github/workflows/release-brew.yml @@ -41,6 +41,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Parse crate and version id: parse env: @@ -98,7 +100,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} run: | mkdir -p rendered - ./sh/gh/render-formula.sh \ + just gh formula \ --template ".github/homebrew/Formula/${CRATE}.rb.tmpl" \ --version "$VERSION" \ --release-dir release-assets \ @@ -156,7 +158,7 @@ jobs: EOF fi - python3 ../sh/gh/install-formula.py "$CRATE" "../rendered/${CRATE}.rb" . + just gh install-formula "$CRATE" "rendered/${CRATE}.rb" homebrew-tap if git diff --quiet && [[ -z "$(git status --porcelain)" ]]; then echo "No changes to commit (formula already up to date)." diff --git a/.github/workflows/release-dart-ffi.yml b/.github/workflows/release-dart-ffi.yml index 59f6c7a61c..2744516380 100644 --- a/.github/workflows/release-dart-ffi.yml +++ b/.github/workflows/release-dart-ffi.yml @@ -46,9 +46,11 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version moq-ffi + run: just gh release parse-version moq-ffi build: name: Build Native Asset (${{ matrix.target }}) @@ -87,6 +89,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -114,7 +118,7 @@ jobs: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} IPHONEOS_DEPLOYMENT_TARGET: ${{ contains(matrix.target, 'apple-ios') && '16.0' || '' }} run: | - ./rs/moq-ffi/build.sh \ + just rs package-ffi \ --target "$BUILD_TARGET" \ --version "$BUILD_VERSION" \ --output dist \ @@ -151,13 +155,15 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - uses: dart-lang/setup-dart@6afc89df92d6eb3834022f73cd65adc8cdfcb92d # v1.8.1 - name: Stage and check package env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./sh/dart/package.sh \ + just dart package \ --package moq_ffi \ --version "$BUILD_VERSION" \ --output release-out @@ -186,9 +192,11 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Find previous tag id: prev_tag - run: sh/gh/release.sh prev-tag moq-ffi + run: just gh release prev-tag moq-ffi - name: Download Native Assets uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -204,7 +212,7 @@ jobs: RELEASE_TITLE: "moq-ffi v${{ needs.parse-version.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} RELEASE_CHECKSUMS: SHA256SUMS-dart - run: sh/gh/release.sh create artifacts + run: just gh release create artifacts publish: name: Publish moq_ffi to pub.dev diff --git a/.github/workflows/release-dart.yml b/.github/workflows/release-dart.yml index 4df8f205f4..3578ae386c 100644 --- a/.github/workflows/release-dart.yml +++ b/.github/workflows/release-dart.yml @@ -37,6 +37,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - uses: dart-lang/setup-dart@6afc89df92d6eb3834022f73cd65adc8cdfcb92d # v1.8.1 - name: Resolve version @@ -47,7 +49,7 @@ jobs: if [[ "$EVENT_NAME" == "pull_request" ]]; then VERSION=$(sed -n 's/^version: //p' dart/moq/pubspec.yaml | head -1) else - sh/gh/release.sh parse-version moq-dart + just gh release parse-version moq-dart exit 0 fi echo "version=$VERSION" >> "$GITHUB_OUTPUT" @@ -69,7 +71,7 @@ jobs: env: BUILD_VERSION: ${{ steps.version.outputs.version }} run: | - ./sh/dart/package.sh \ + just dart package \ --package moq \ --version "$BUILD_VERSION" \ --output release-out diff --git a/.github/workflows/release-go-ffi.yml b/.github/workflows/release-go-ffi.yml index d1d8477d38..2039f200b3 100644 --- a/.github/workflows/release-go-ffi.yml +++ b/.github/workflows/release-go-ffi.yml @@ -51,9 +51,11 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version moq-ffi + run: just gh release parse-version moq-ffi build: name: Build moq-ffi (${{ matrix.target }}) @@ -78,6 +80,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -103,7 +107,7 @@ jobs: BUILD_TARGET: ${{ matrix.target }} BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./rs/moq-ffi/build.sh \ + just rs package-ffi \ --target "$BUILD_TARGET" \ --version "$BUILD_VERSION" \ --output dist @@ -125,6 +129,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @@ -144,7 +150,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./rs/moq-ffi/build.sh \ + just rs package-ffi \ --bindings-only \ --version "$BUILD_VERSION" \ --output dist @@ -165,6 +171,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Download per-target libs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -208,7 +216,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./sh/go/package-ffi.sh \ + just go package-ffi \ --version "$BUILD_VERSION" \ --source-dir go/ffi \ --lib-dir libs \ @@ -232,6 +240,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Generate moq-bot token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 id: token @@ -256,7 +266,7 @@ jobs: GO_FFI_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./sh/go/publish-ffi.sh + run: just go publish-ffi publish-dry-run: name: Publish dry-run @@ -269,6 +279,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Download package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -278,4 +290,4 @@ jobs: - name: Dry-run publish to mirror env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: ./sh/go/publish-ffi.sh --dry-run + run: just go publish-ffi --dry-run diff --git a/.github/workflows/release-go.yml b/.github/workflows/release-go.yml index 92300e5f13..bb817841ac 100644 --- a/.github/workflows/release-go.yml +++ b/.github/workflows/release-go.yml @@ -66,6 +66,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: "1.23" @@ -105,7 +107,7 @@ jobs: if: github.event_name == 'push' env: FFI_VERSION: ${{ steps.ffi.outputs.version }} - run: sh/gh/release.sh ffi-unreleased "$FFI_VERSION" + run: just gh release ffi-unreleased "$FFI_VERSION" # Skip on unreleased ffi changes, and when the pinned ffi's mirror tag is # missing (release-plz merged but Release Go FFI hasn't published yet): @@ -160,7 +162,7 @@ jobs: LINE: ${{ steps.line.outputs.line }} FFI_VERSION: ${{ steps.ffi.outputs.version }} run: | - ./sh/go/package-wrapper.sh --line "$LINE" --ffi-version "$FFI_VERSION" --output release-out + just go package-wrapper --line "$LINE" --ffi-version "$FFI_VERSION" --output release-out - name: Upload wrapper module if: steps.gate.outputs.skip != 'true' @@ -180,6 +182,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Generate moq-bot token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 id: token @@ -203,4 +207,4 @@ jobs: GO_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./sh/go/publish-wrapper.sh + run: just go publish-wrapper diff --git a/.github/workflows/release-kt-ffi.yml b/.github/workflows/release-kt-ffi.yml index 31a6979587..207e70f35b 100644 --- a/.github/workflows/release-kt-ffi.yml +++ b/.github/workflows/release-kt-ffi.yml @@ -37,9 +37,11 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version moq-ffi + run: just gh release parse-version moq-ffi build: name: Build moq-ffi (${{ matrix.target }}) @@ -72,6 +74,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -98,7 +102,7 @@ jobs: BUILD_TARGET: ${{ matrix.target }} BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./rs/moq-ffi/build.sh \ + just rs package-ffi \ --target "$BUILD_TARGET" \ --version "$BUILD_VERSION" \ --output dist @@ -120,6 +124,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @@ -130,7 +136,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./rs/moq-ffi/build.sh \ + just rs package-ffi \ --bindings-only \ --version "$BUILD_VERSION" \ --output dist @@ -160,6 +166,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 with: distribution: temurin @@ -199,7 +207,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./sh/kt/package.sh \ + just kt package \ --version "$BUILD_VERSION" \ --lib-dir libs \ --bindings-dir bindings \ diff --git a/.github/workflows/release-kt-lib.yml b/.github/workflows/release-kt-lib.yml index 1bec21db75..ce0334622e 100644 --- a/.github/workflows/release-kt-lib.yml +++ b/.github/workflows/release-kt-lib.yml @@ -75,6 +75,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Read moq.version id: version run: | @@ -103,18 +105,18 @@ jobs: - name: Check for unreleased FFI changes id: ffi-diff if: github.event_name != 'workflow_run' - run: sh/gh/release.sh ffi-unreleased "${{ steps.ffi.outputs.version }}" + run: just gh release ffi-unreleased "${{ steps.ffi.outputs.version }}" - name: Check wrapper publication id: wrapper - run: sh/gh/release.sh maven-exists dev.moq moq "${{ steps.version.outputs.version }}" + run: just gh release maven-exists dev.moq moq "${{ steps.version.outputs.version }}" # The bindings this checkout generates must be on Maven Central, or the # wrapper's POM range resolves to an older moq-ffi without them. - name: Check bindings publication id: bindings if: github.event_name != 'workflow_run' && steps.wrapper.outputs.exists != 'true' && steps.ffi-diff.outputs.unreleased != 'true' - run: sh/gh/release.sh maven-exists dev.moq moq-ffi "${{ steps.ffi.outputs.version }}" + run: just gh release maven-exists dev.moq moq-ffi "${{ steps.ffi.outputs.version }}" - name: Decide publish id: decide @@ -167,6 +169,8 @@ jobs: ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }} persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @@ -178,7 +182,7 @@ jobs: env: BUILD_VERSION: ${{ needs.check-version.outputs.version }} run: | - ./rs/moq-ffi/build.sh --bindings-only --version "$BUILD_VERSION" --output dist + just rs package-ffi --bindings-only --version "$BUILD_VERSION" --output dist mkdir -p kt/moq-ffi/src/jvmAndAndroidMain/kotlin/uniffi/moq cp dist/bindings/kotlin/uniffi/moq/moq.kt \ kt/moq-ffi/src/jvmAndAndroidMain/kotlin/uniffi/moq/moq.kt @@ -202,7 +206,7 @@ jobs: - name: Recheck wrapper publication id: recheck if: github.event_name != 'pull_request' - run: sh/gh/release.sh maven-exists dev.moq moq "${{ needs.check-version.outputs.version }}" + run: just gh release maven-exists dev.moq moq "${{ needs.check-version.outputs.version }}" - name: Publish to Maven Central if: github.event_name != 'pull_request' && steps.recheck.outputs.exists != 'true' diff --git a/.github/workflows/release-py-ffi.yml b/.github/workflows/release-py-ffi.yml index 2e1fa1bb49..678abd2438 100644 --- a/.github/workflows/release-py-ffi.yml +++ b/.github/workflows/release-py-ffi.yml @@ -34,9 +34,11 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version moq-ffi + run: just gh release parse-version moq-ffi # Maturin reads the wheel version from rs/moq-ffi/Cargo.toml (via # `dynamic = ["version"]` in pyproject.toml). If a tag is pushed by diff --git a/.github/workflows/release-py.yml b/.github/workflows/release-py.yml index 67a2741e8b..fdb4717c7b 100644 --- a/.github/workflows/release-py.yml +++ b/.github/workflows/release-py.yml @@ -56,9 +56,11 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Read version id: version - run: sh/gh/release.sh read-version py/moq-rs/pyproject.toml + run: just gh release read-version py/moq-rs/pyproject.toml - name: Resolve FFI version id: ffi @@ -78,14 +80,14 @@ jobs: if: github.event_name == 'push' env: FFI_VERSION: ${{ steps.ffi.outputs.version }} - run: sh/gh/release.sh ffi-unreleased "$FFI_VERSION" + run: just gh release ffi-unreleased "$FFI_VERSION" - name: Check FFI on PyPI id: ffi-pypi if: github.event_name == 'push' && steps.ffi-gate.outputs.unreleased != 'true' env: FFI_VERSION: ${{ steps.ffi.outputs.version }} - run: sh/gh/release.sh pypi-exists moq-ffi "$FFI_VERSION" + run: just gh release pypi-exists moq-ffi "$FFI_VERSION" - name: Unpublished FFI notice if: github.event_name == 'push' && steps.ffi-gate.outputs.unreleased != 'true' && steps.ffi-pypi.outputs.exists != 'true' @@ -100,7 +102,7 @@ jobs: id: pypi env: VERSION: ${{ steps.version.outputs.version }} - run: sh/gh/release.sh pypi-exists moq-rs "$VERSION" + run: just gh release pypi-exists moq-rs "$VERSION" - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main with: diff --git a/.github/workflows/release-swift-ffi.yml b/.github/workflows/release-swift-ffi.yml index 966a1b86c3..0ab0eaa056 100644 --- a/.github/workflows/release-swift-ffi.yml +++ b/.github/workflows/release-swift-ffi.yml @@ -34,9 +34,11 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version moq-ffi + run: just gh release parse-version moq-ffi build: name: Build moq-ffi (${{ matrix.target }}) @@ -56,6 +58,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -72,7 +76,7 @@ jobs: # Set iOS deployment target to avoid ___chkstk_darwin linker errors. IPHONEOS_DEPLOYMENT_TARGET: ${{ contains(matrix.target, 'apple-ios') && '16.0' || '' }} run: | - ./rs/moq-ffi/build.sh \ + just rs package-ffi \ --target "$BUILD_TARGET" \ --version "$BUILD_VERSION" \ --output dist @@ -93,6 +97,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @@ -103,7 +109,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./rs/moq-ffi/build.sh \ + just rs package-ffi \ --bindings-only \ --version "$BUILD_VERSION" \ --output dist @@ -124,6 +130,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Download per-target libs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -149,7 +157,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./sh/swift/package-ffi.sh \ + just swift package-ffi \ --version "$BUILD_VERSION" \ --lib-dir libs \ --bindings-dir bindings \ @@ -181,9 +189,11 @@ jobs: # up GH_TOKEN directly, so don't persist a contents-write token. persist-credentials: false + - uses: ./.github/actions/just + - name: Find previous tag id: prev_tag - run: sh/gh/release.sh prev-tag moq-ffi + run: just gh release prev-tag moq-ffi - name: Download package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -197,7 +207,7 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "moq-ffi v${{ needs.parse-version.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: sh/gh/release.sh create artifacts + run: just gh release create artifacts verify: # Gate the mirror push on actually being able to resolve the staged package @@ -213,6 +223,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Download staged package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -223,7 +235,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./sh/swift/verify-ffi.sh --tarball "artifacts/moq-ffi-${BUILD_VERSION}-swift-ffi.tar.gz" + just swift verify-ffi --tarball "artifacts/moq-ffi-${BUILD_VERSION}-swift-ffi.tar.gz" publish: name: Publish to Swift Package mirror @@ -235,6 +247,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Generate moq-bot token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 id: token @@ -259,4 +273,4 @@ jobs: SWIFT_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./sh/swift/publish-ffi.sh + run: just swift publish-ffi diff --git a/.github/workflows/release-swift-lib.yml b/.github/workflows/release-swift-lib.yml index 6361566a57..2684aa4f59 100644 --- a/.github/workflows/release-swift-lib.yml +++ b/.github/workflows/release-swift-lib.yml @@ -79,6 +79,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Read versions id: version env: @@ -101,13 +103,13 @@ jobs: if: github.event_name != 'workflow_run' env: FFI_VERSION: ${{ steps.version.outputs.ffi_version }} - run: sh/gh/release.sh ffi-unreleased "$FFI_VERSION" + run: just gh release ffi-unreleased "$FFI_VERSION" - name: Check FFI mirror tag id: ffi-gate env: FFI_VERSION: ${{ steps.version.outputs.ffi_version }} - run: sh/gh/release.sh git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" + run: just gh release git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" - name: Unpublished FFI notice if: steps.ffi-diff.outputs.unreleased != 'true' && steps.ffi-gate.outputs.exists != 'true' && github.event_name != 'pull_request' @@ -121,12 +123,12 @@ jobs: id: gate env: VERSION: ${{ steps.version.outputs.version }} - run: sh/gh/release.sh git-tag-exists moq-dev/moq-swift "$VERSION" + run: just gh release git-tag-exists moq-dev/moq-swift "$VERSION" - name: Package env: BUILD_VERSION: ${{ steps.version.outputs.version }} - run: ./sh/swift/package.sh --version "$BUILD_VERSION" --output release-out + run: just swift package --version "$BUILD_VERSION" --output release-out - name: Upload Swift package uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -149,6 +151,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Check FFI mirror tag id: ffi if: needs.build.outputs.publish == 'true' || (github.event_name == 'pull_request' && needs.build.outputs.ffi_unreleased != 'true') @@ -158,7 +162,7 @@ jobs: # rather than a hard failure, so the first bootstrap (before # moq-swift-ffi exists) skips the cross-package resolve instead of # failing the job. publish.sh stays idempotent regardless. - run: sh/gh/release.sh git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" || echo "exists=false" >> "$GITHUB_OUTPUT" + run: just gh release git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" || echo "exists=false" >> "$GITHUB_OUTPUT" - name: Download staged package if: steps.ffi.outputs.exists == 'true' @@ -171,7 +175,7 @@ jobs: if: steps.ffi.outputs.exists == 'true' env: BUILD_VERSION: ${{ needs.build.outputs.version }} - run: ./sh/swift/verify.sh --tarball "artifacts/moq-${BUILD_VERSION}-swift.tar.gz" + run: just swift verify --tarball "artifacts/moq-${BUILD_VERSION}-swift.tar.gz" - name: Skip notice if: needs.build.outputs.ffi_unreleased == 'true' @@ -194,6 +198,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Generate moq-bot token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 id: token @@ -218,7 +224,7 @@ jobs: SWIFT_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./sh/swift/publish.sh + run: just swift publish publish-dry-run: name: Publish dry-run @@ -231,6 +237,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Download package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -240,4 +248,4 @@ jobs: - name: Dry-run publish to mirror env: BUILD_VERSION: ${{ needs.build.outputs.version }} - run: ./sh/swift/publish.sh --dry-run + run: just swift publish --dry-run diff --git a/.github/workflows/rpm-repo.yml b/.github/workflows/rpm-repo.yml index 4c7df9dd84..93dc8c4a94 100644 --- a/.github/workflows/rpm-repo.yml +++ b/.github/workflows/rpm-repo.yml @@ -32,6 +32,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Set up packaging tools uses: ./.github/actions/setup-packaging @@ -51,10 +53,9 @@ jobs: - name: Publish to R2 shell: bash env: - ARTIFACTS_DIR: artifacts R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} SIGNING_KEY: ${{ secrets.SIGNING_KEY }} SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} - run: ./infra/rpm/publish.sh + run: just infra rpm publish "$GITHUB_WORKSPACE/artifacts" diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index 23fd44fbf3..9ba45fe5cd 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -55,6 +55,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -83,4 +85,4 @@ jobs: done - name: Swift check - run: ./sh/swift/check.sh + run: just swift check diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 10c8135fdb..ccccac5757 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,6 +45,10 @@ For each finding: - If you don't agree with it, reply to the finding and move on. - If it's a relatively easy improvement, fix it and push. Update the summary if needed. +# CI + +Workflow steps run `just` recipes, never a script path; `just gh check` enforces it. + # Follow-ups If you encounter issues, or findings that are out of scope, create follow-up quests. diff --git a/cpp/obs/justfile b/cpp/obs/justfile index eed0d37344..8039fc603f 100644 --- a/cpp/obs/justfile +++ b/cpp/obs/justfile @@ -57,3 +57,12 @@ check: # Auto-fix formatting. fix: ../../sh/obs/fix.sh + +# Runs in the caller's directory, so a relative `--output` means what it would +# to the script itself. + +# Build and package the obs-moq plugin for release. +[no-cd] +[positional-arguments] +package *args: + {{ quote(justfile_directory() / "cpp/obs/build.sh") }} "$@" diff --git a/dart/justfile b/dart/justfile index 4387f113c8..5c2fc803ff 100644 --- a/dart/justfile +++ b/dart/justfile @@ -11,3 +11,12 @@ fix: # Regenerate the raw UniFFI Dart bindings. generate: ../sh/dart/generate.sh + +# `package` runs in the caller's directory, so a relative `--output` means +# what it would to the script itself. + +# Package a Dart library for pub.dev. +[no-cd] +[positional-arguments] +package *args: + {{ quote(justfile_directory() / "sh/dart/package.sh") }} "$@" diff --git a/go/ffi/README.md b/go/ffi/README.md index c45b1e7026..308836ae30 100644 --- a/go/ffi/README.md +++ b/go/ffi/README.md @@ -53,4 +53,4 @@ Compiled binaries never live in the source tree. `sh/go/check.sh` stages into th ## Release -The `release-go-ffi.yml` workflow fires on every `moq-ffi-v*` tag, builds per-target static libraries, runs `uniffi-bindgen-go`, calls `sh/go/package-ffi.sh` to assemble the module, and `sh/go/publish-ffi.sh` to push the result to `moq-dev/moq-go-ffi` with a bare-semver tag (e.g. `v0.2.18`) lockstep with the crate. Go's module proxy picks up the new tag automatically. +The `release-go-ffi.yml` workflow fires on every `moq-ffi-v*` tag, builds per-target static libraries, runs `uniffi-bindgen-go`, calls `just go package-ffi` to assemble the module, and `just go publish-ffi` to push the result to `moq-dev/moq-go-ffi` with a bare-semver tag (e.g. `v0.2.18`) lockstep with the crate. Go's module proxy picks up the new tag automatically. diff --git a/go/justfile b/go/justfile index ad26a9875d..b4fa2d0d07 100644 --- a/go/justfile +++ b/go/justfile @@ -16,12 +16,35 @@ default: check: bash ../sh/go/check.sh -# Stage the in-tree go/ffi source + per-target moq-ffi libs + generated -# bindings into the moq.dev/moq-ffi module ready for publish. +# Release recipes run in the caller's directory, so a relative `--output` or +# `--tarball` means what it would to the script itself. + +# Stage the in-tree go/ffi source, per-target moq-ffi libs, and generated +# bindings into the moq.dev/moq-ffi module. + +# Stage the moq.dev/moq-ffi module for publish. +[no-cd] +[positional-arguments] package-ffi *args: - bash ../sh/go/package-ffi.sh {{ args }} + {{ quote(justfile_directory() / "sh/go/package-ffi.sh") }} "$@" # Stage the in-tree go/wrapper source into the moq.dev/moq module, rewriting its -# moq.dev/moq-ffi require to the target ffi version, ready for publish. +# moq.dev/moq-ffi require to the target ffi version. + +# Stage the moq.dev/moq wrapper module for publish. +[no-cd] +[positional-arguments] package-wrapper *args: - bash ../sh/go/package-wrapper.sh {{ args }} + {{ quote(justfile_directory() / "sh/go/package-wrapper.sh") }} "$@" + +# Push the staged moq-go-ffi module to the moq-dev/moq-go-ffi mirror. +[no-cd] +[positional-arguments] +publish-ffi *args: + {{ quote(justfile_directory() / "sh/go/publish-ffi.sh") }} "$@" + +# Push the staged moq-go wrapper module to the moq-dev/moq-go mirror. +[no-cd] +[positional-arguments] +publish-wrapper *args: + {{ quote(justfile_directory() / "sh/go/publish-wrapper.sh") }} "$@" diff --git a/js/justfile b/js/justfile index 3cc52a9e22..812fa6d9c9 100644 --- a/js/justfile +++ b/js/justfile @@ -43,6 +43,11 @@ test: bun test common/deps.test.ts bun run --filter='*' --elide-lines=0 test +# Benchmark re-pricing the origin broadcast map as routes and observers grow. +bench: + bun install --frozen-lockfile + bun net/bench/broadcasts.ts + # Build all packages. build: bun run --filter='*' build diff --git a/kt/justfile b/kt/justfile index af419fb33e..2a3994710a 100644 --- a/kt/justfile +++ b/kt/justfile @@ -21,8 +21,11 @@ check: generate: bash ../sh/kt/generate.sh -# Assemble the KMP module from per-target moq-ffi binaries + bindings. -# Used by .github/workflows/release-kt-ffi.yml; see kt/README.md for the -# expected --lib-dir layout. +# Runs in the caller's directory, so a relative `--lib-dir` or `--output` means +# what it would to the script itself. See kt/README.md for the --lib-dir layout. + +# Assemble the KMP module from per-target moq-ffi binaries and bindings. +[no-cd] +[positional-arguments] package *args: - bash ../sh/kt/package.sh {{ args }} + {{ quote(justfile_directory() / "sh/kt/package.sh") }} "$@" diff --git a/quest/m1/tooling/README.md b/quest/m1/tooling/README.md index ae2b81172b..e01a88ebc7 100644 --- a/quest/m1/tooling/README.md +++ b/quest/m1/tooling/README.md @@ -19,6 +19,5 @@ requires the one before it, so they land as one line of pull requests. ## Quests -- [Workflows call just](/quest/m1/tooling/workflows-call-just.md) - no workflow `run:` step names a `.sh`; every script a workflow needs has a recipe - [Binary release workflow](/quest/m1/tooling/release-binary.md) - moq-cli and moq-relay share one reusable workflow behind two thin callers - [FFI release workflow](/quest/m1/tooling/release-ffi.md) - the five `release-*-ffi.yml` share the moq-ffi target matrix and artifact staging diff --git a/quest/m1/tooling/release-binary.md b/quest/m1/tooling/release-binary.md index 95eecf91f1..4a1dcdf0c5 100644 --- a/quest/m1/tooling/release-binary.md +++ b/quest/m1/tooling/release-binary.md @@ -25,7 +25,3 @@ and workflow name, so `alert.yml`, `release-brew.yml`, and - Verify with `just gh check` and by diffing the rendered job list of each caller against its predecessor with `gh workflow view`. The next tagged release is the end-to-end check; say so in the PR. - -## Required - -- [Workflows call just](/quest/m1/tooling/workflows-call-just.md) - the reusable workflow is written in the recipe style from the start diff --git a/quest/m1/tooling/workflows-call-just.md b/quest/m1/tooling/workflows-call-just.md deleted file mode 100644 index a38fcc5217..0000000000 --- a/quest/m1/tooling/workflows-call-just.md +++ /dev/null @@ -1,41 +0,0 @@ -# [M] Every workflow step runs a recipe - -## Goal - -No `run:` line in `.github/workflows` names a `.sh` file. Every script a -workflow needs has a `just` recipe, so the menu is the whole command surface -and a script can move or change its arguments without touching twenty -workflows. The same recipes work locally for a dry run. - -## Plan - -Today the workflows call scripts directly at roughly ninety sites: -`sh/gh/release.sh` (54 calls as a helper library: `parse-version`, -`prev-tag`, `create`, registry existence checks), `sh/rs/package-nfpm.sh`, -`sh/rs/package-binary.sh`, `sh/rs/package-windows.sh`, -`sh/gh/trigger-repo-publish.sh`, `sh/gh/render-formula.sh`, `sh/gh/alert.sh`, -the swift `package`, `package-ffi`, `publish`, `publish-ffi`, `verify`, -`verify-ffi`, and `check` scripts, the go `package-ffi`, `package-wrapper`, -`publish-ffi`, `publish-wrapper` scripts, `sh/dart/package.sh`, -`sh/kt/package.sh`, and `infra/*/publish.sh`. - -- Add pass-through recipes where none exist, named by role under the area - module: `just gh release ...`, `just gh alert ...`, - `just gh formula ...`, `just gh trigger-repo-publish ...`, - `just rs package-binary ...`, `just rs package-nfpm ...`, - `just rs package-windows ...`, `just swift publish|verify|package-ffi|...`, - `just go publish-ffi|publish-wrapper`, `just dart package`, - `just infra apt publish` and `just infra rpm publish` (the recipes exist). - Existing wrappers (`kt package`, `swift package`, `go package-ffi`, - `go package-wrapper`, `py package`) are used as they are. -- Rewrite every `run:` step to the recipe. Where a workflow runs outside the - nix dev shell (Windows and macOS runners, AlmaLinux containers), `just` is - already installed or installable the way nightly.yml does it; keep that - pattern rather than falling back to a path. -- Record the rule in the CI section of `CONTRIBUTING.md` in one line, and - have `just gh check` enforce it: fail on any workflow `run:` line matching - `\.sh\b`. -- Verify with `actionlint` via `just gh check`, and with a manual - `workflow_dispatch` of apt-repo.yml, rpm-repo.yml, and release-winget.yml - where the inputs allow a dry run. The tag-triggered release workflows are - verified by the next release; say so in the PR. diff --git a/rs/justfile b/rs/justfile index 7b5919eec1..71c272ae43 100644 --- a/rs/justfile +++ b/rs/justfile @@ -236,3 +236,38 @@ release: # Build a .deb or .rpm locally: `just rs package moq-relay deb`. package crate packager: sh/rs/package.sh {{ crate }} {{ packager }} + +# Build and package a workspace binary as a release tarball. +[positional-arguments] +package-binary *args: + sh/rs/package-binary.sh "$@" + +# Build and package a workspace binary as a Windows .zip. +[positional-arguments] +package-windows *args: + sh/rs/package-windows.sh "$@" + +# Render an nfpm config from the environment and build a .deb or .rpm: CONFIG PACKAGER OUTPUT. +[positional-arguments] +package-nfpm *args: + sh/rs/package-nfpm.sh "$@" + +# Build and package the moq-ffi native libraries, or only its bindings. +[positional-arguments] +package-ffi *args: + rs/moq-ffi/build.sh "$@" + +# Build and package libmoq. +[positional-arguments] +package-libmoq *args: + rs/libmoq/build.sh "$@" + +# Build and package the moq-gst GStreamer plugin. +[positional-arguments] +package-gst *args: + rs/moq-gst/build.sh "$@" + +# Load a packaged moq-gst plugin into the host's GStreamer and find its elements. +[positional-arguments] +gst-smoke *args: + rs/moq-gst/smoke.sh "$@" diff --git a/sh/gh/check-runs.ts b/sh/gh/check-runs.ts new file mode 100644 index 0000000000..c979f47b73 --- /dev/null +++ b/sh/gh/check-runs.ts @@ -0,0 +1,26 @@ +// Fail when a workflow `run:` step names a script file. Workflows call `just` +// recipes, so a script can move or change its arguments without touching them, +// and the same command works locally. +import { Glob } from "bun"; + +type Step = { run?: string }; +type Job = { steps?: Step[] }; +type Workflow = { jobs?: Record }; + +const violations: string[] = []; +for await (const path of new Glob(".github/workflows/*.{yml,yaml}").scan({ dot: true })) { + const workflow = Bun.YAML.parse(await Bun.file(path).text()) as Workflow; + for (const [name, job] of Object.entries(workflow.jobs ?? {})) { + for (const step of job.steps ?? []) { + for (const line of step.run?.split("\n") ?? []) { + if (/\.(sh|py|ts|js|mjs)\b/.test(line)) violations.push(`${path} (${name}): ${line.trim()}`); + } + } + } +} + +if (violations.length > 0) { + console.error("workflow steps must run a `just` recipe, not a script:"); + for (const violation of violations.sort()) console.error(` ${violation}`); + process.exit(1); +} diff --git a/swift/justfile b/swift/justfile index acb2b15029..075ae92314 100644 --- a/swift/justfile +++ b/swift/justfile @@ -15,7 +15,41 @@ default: check: bash ../sh/swift/check.sh +# Release recipes run in the caller's directory, so a relative `--output` or +# `--tarball` means what it would to the script itself. + # Assemble the released wrapper package from Package.swift.template. -# Used by .github/workflows/release-swift-lib.yml. +[no-cd] +[positional-arguments] package *args: - bash ../sh/swift/package.sh {{ args }} + {{ quote(justfile_directory() / "sh/swift/package.sh") }} "$@" + +# Assemble the MoqFFI package from per-target static libs and the generated bindings. +[no-cd] +[positional-arguments] +package-ffi *args: + {{ quote(justfile_directory() / "sh/swift/package-ffi.sh") }} "$@" + +# Build a throwaway consumer of a staged wrapper package. +[no-cd] +[positional-arguments] +verify *args: + {{ quote(justfile_directory() / "sh/swift/verify.sh") }} "$@" + +# Build a throwaway consumer of a staged MoqFFI package. +[no-cd] +[positional-arguments] +verify-ffi *args: + {{ quote(justfile_directory() / "sh/swift/verify-ffi.sh") }} "$@" + +# Push the staged wrapper package to the moq-dev/moq-swift mirror. +[no-cd] +[positional-arguments] +publish *args: + {{ quote(justfile_directory() / "sh/swift/publish.sh") }} "$@" + +# Push the staged MoqFFI package to the moq-dev/moq-swift-ffi mirror. +[no-cd] +[positional-arguments] +publish-ffi *args: + {{ quote(justfile_directory() / "sh/swift/publish-ffi.sh") }} "$@" From 39348893659958376d898f8074d71b40e1c52d67 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 25 Sep 2026 15:29:26 -0700 Subject: [PATCH 4/6] chore(ci): moq-cli and moq-relay share one reusable release workflow (#4174) Co-authored-by: Claude Opus 5.5 --- .github/workflows/moq-cli.yml | 220 +---------------------- .github/workflows/moq-relay.yml | 219 +---------------------- .github/workflows/release-binary.yml | 253 +++++++++++++++++++++++++++ quest/m1/moq-installer.md | 2 - quest/m1/tooling/README.md | 1 - quest/m1/tooling/release-binary.md | 27 --- quest/m1/tooling/release-ffi.md | 13 +- sh/gh/alert.sh | 11 +- 8 files changed, 273 insertions(+), 473 deletions(-) create mode 100644 .github/workflows/release-binary.yml delete mode 100644 quest/m1/tooling/release-binary.md diff --git a/.github/workflows/moq-cli.yml b/.github/workflows/moq-cli.yml index 3975e32663..0258e0ca07 100644 --- a/.github/workflows/moq-cli.yml +++ b/.github/workflows/moq-cli.yml @@ -17,223 +17,11 @@ permissions: contents: read jobs: - build: - name: Build (${{ matrix.target }}) - runs-on: ${{ matrix.runs-on }} - - strategy: - fail-fast: false - matrix: - include: - - target: x86_64-unknown-linux-gnu - runs-on: ubuntu-22.04 - deb-arch: amd64 - rpm-arch: x86_64 - - target: aarch64-unknown-linux-gnu - runs-on: ubuntu-22.04-arm - deb-arch: arm64 - rpm-arch: aarch64 - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - - name: Parse version - id: parse - shell: bash - run: just gh release parse-version moq-cli - - - name: Set up packaging tools - uses: ./.github/actions/setup-packaging - - - name: Build (release, glibc 2.34) - shell: bash - # Pin to glibc 2.34 so one binary covers Ubuntu 22.04+ (.deb) - # and AlmaLinux/RHEL/Rocky 9+ (.rpm). - run: cargo zigbuild --locked --release --target ${{ matrix.target }}.2.34 -p moq-cli - - - name: Package release assets - shell: bash - env: - VERSION: ${{ steps.parse.outputs.version }} - TARGET: ${{ matrix.target }} - run: | - just rs package-binary \ - --crate moq-cli \ - --bin moq \ - --binary "target/${TARGET}/release/moq" \ - --bare \ - --version "$VERSION" \ - --target "$TARGET" \ - --output dist - - - name: Package .deb (Ubuntu 22.04 toolchain) - shell: bash - env: - VERSION: ${{ steps.parse.outputs.version }} - ARCH: ${{ matrix.deb-arch }} - BINARY_PATH: target/${{ matrix.target }}/release/moq - run: | - just rs package-nfpm packaging/moq-cli/nfpm.yaml deb dist/ - just rs package-nfpm packaging/moq-cli/transition.yaml deb dist/ - - - name: Package .rpm - shell: bash - env: - VERSION: ${{ steps.parse.outputs.version }} - ARCH: ${{ matrix.rpm-arch }} - BINARY_PATH: target/${{ matrix.target }}/release/moq - run: just rs package-nfpm packaging/moq-cli/nfpm.yaml rpm dist/ - - - name: Upload artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: moq-cli-${{ matrix.target }} - path: dist/* - - build-tarball-macos: - name: Build macOS tarball (${{ matrix.target }}) - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - include: - - target: aarch64-apple-darwin - os: macos-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - persist-credentials: false - - - uses: ./.github/actions/just - - - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main - with: - determinate: false - - - name: Parse version - id: parse - shell: bash - run: just gh release parse-version moq-cli - - - name: Build and package tarball - shell: bash - run: | - just rs package-binary \ - --crate moq-cli \ - --bin moq \ - --target ${{ matrix.target }} \ - --version ${{ steps.parse.outputs.version }} \ - --output dist - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: moq-cli-${{ matrix.target }} - path: dist/* - - build-windows: - name: Build Windows zip (${{ matrix.target }}) - runs-on: windows-latest - - strategy: - fail-fast: false - matrix: - include: - - target: x86_64-pc-windows-msvc - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - # aws-lc-rs assembles its x86_64 crypto with NASM on Windows. - - name: Install NASM - shell: pwsh - run: | - choco install nasm -y --no-progress - "C:\Program Files\NASM" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - - name: Parse version - id: parse - shell: bash - run: just gh release parse-version moq-cli - - - name: Build and package zip - shell: bash - run: | - just rs package-windows \ - --crate moq-cli \ - --bin moq \ - --target ${{ matrix.target }} \ - --version ${{ steps.parse.outputs.version }} \ - --output dist - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: moq-cli-${{ matrix.target }} - path: dist/* - release: - name: Release - needs: [build, build-tarball-macos, build-windows] - if: github.ref_type == 'tag' - runs-on: ubuntu-latest + uses: ./.github/workflows/release-binary.yml + with: + crate: moq-cli + bin: moq permissions: contents: write actions: write - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Parse version - id: parse - run: just gh release parse-version moq-cli - - - name: Find previous tag - id: prev_tag - run: just gh release prev-tag moq-cli - - - name: Download artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - path: artifacts - merge-multiple: true - - - name: Create or update release - env: - GH_TOKEN: ${{ github.token }} - RELEASE_TAG: ${{ github.ref_name }} - RELEASE_TITLE: "moq-cli v${{ steps.parse.outputs.version }}" - RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: just gh release create artifacts - - - name: Trigger apt/rpm repo publish - env: - GH_TOKEN: ${{ github.token }} - TAG: ${{ github.ref_name }} - run: just gh trigger-repo-publish diff --git a/.github/workflows/moq-relay.yml b/.github/workflows/moq-relay.yml index b9ac806f96..cfb4767b53 100644 --- a/.github/workflows/moq-relay.yml +++ b/.github/workflows/moq-relay.yml @@ -17,222 +17,11 @@ permissions: contents: read jobs: - build: - name: Build (${{ matrix.target }}) - runs-on: ${{ matrix.runs-on }} - - strategy: - fail-fast: false - matrix: - include: - # Native runners. Each binary is built on the same arch it ships - # for, so the .deb/.rpm linkage matches the distro's glibc on the - # oldest LTS we support (Ubuntu 22.04 / AlmaLinux 9). - - target: x86_64-unknown-linux-gnu - runs-on: ubuntu-22.04 - deb-arch: amd64 - rpm-arch: x86_64 - - target: aarch64-unknown-linux-gnu - runs-on: ubuntu-22.04-arm - deb-arch: arm64 - rpm-arch: aarch64 - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - - name: Parse version - id: parse - shell: bash - run: just gh release parse-version moq-relay - - - name: Set up packaging tools - uses: ./.github/actions/setup-packaging - - - name: Build (release, glibc 2.34) - shell: bash - # Pin to glibc 2.34 so one binary covers Ubuntu 22.04+ (.deb) - # and AlmaLinux/RHEL/Rocky 9+ (.rpm). - run: cargo zigbuild --locked --release --target ${{ matrix.target }}.2.34 -p moq-relay - - - name: Package release assets - shell: bash - env: - VERSION: ${{ steps.parse.outputs.version }} - TARGET: ${{ matrix.target }} - run: | - just rs package-binary \ - --crate moq-relay \ - --bin moq-relay \ - --binary "target/${TARGET}/release/moq-relay" \ - --bare \ - --version "$VERSION" \ - --target "$TARGET" \ - --output dist - - - name: Package .deb (Ubuntu 22.04 toolchain, glibc 2.35) - shell: bash - env: - VERSION: ${{ steps.parse.outputs.version }} - ARCH: ${{ matrix.deb-arch }} - BINARY_PATH: target/${{ matrix.target }}/release/moq-relay - run: just rs package-nfpm packaging/moq-relay/nfpm.yaml deb dist/ - - - name: Package .rpm - shell: bash - env: - VERSION: ${{ steps.parse.outputs.version }} - ARCH: ${{ matrix.rpm-arch }} - BINARY_PATH: target/${{ matrix.target }}/release/moq-relay - run: just rs package-nfpm packaging/moq-relay/nfpm.yaml rpm dist/ - - - name: Upload artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: moq-relay-${{ matrix.target }} - path: dist/* - - build-tarball-macos: - name: Build macOS tarball (${{ matrix.target }}) - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - include: - - target: aarch64-apple-darwin - os: macos-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - persist-credentials: false - - - uses: ./.github/actions/just - - - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main - with: - determinate: false - - - name: Parse version - id: parse - shell: bash - run: just gh release parse-version moq-relay - - - name: Build and package tarball - shell: bash - run: | - just rs package-binary \ - --crate moq-relay \ - --target ${{ matrix.target }} \ - --version ${{ steps.parse.outputs.version }} \ - --output dist - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: moq-relay-${{ matrix.target }} - path: dist/* - - build-windows: - name: Build Windows zip (${{ matrix.target }}) - runs-on: windows-latest - - strategy: - fail-fast: false - matrix: - include: - - target: x86_64-pc-windows-msvc - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - # aws-lc-rs assembles its x86_64 crypto with NASM on Windows. - - name: Install NASM - shell: pwsh - run: | - choco install nasm -y --no-progress - "C:\Program Files\NASM" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - - name: Parse version - id: parse - shell: bash - run: just gh release parse-version moq-relay - - - name: Build and package zip - shell: bash - run: | - just rs package-windows \ - --crate moq-relay \ - --target ${{ matrix.target }} \ - --version ${{ steps.parse.outputs.version }} \ - --output dist - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: moq-relay-${{ matrix.target }} - path: dist/* - release: - name: Release - needs: [build, build-tarball-macos, build-windows] - if: github.ref_type == 'tag' - runs-on: ubuntu-latest + uses: ./.github/workflows/release-binary.yml + with: + crate: moq-relay + bin: moq-relay permissions: contents: write actions: write - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Parse version - id: parse - run: just gh release parse-version moq-relay - - - name: Find previous tag - id: prev_tag - run: just gh release prev-tag moq-relay - - - name: Download artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - path: artifacts - merge-multiple: true - - - name: Create or update release - env: - GH_TOKEN: ${{ github.token }} - RELEASE_TAG: ${{ github.ref_name }} - RELEASE_TITLE: "moq-relay v${{ steps.parse.outputs.version }}" - RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: just gh release create artifacts - - - name: Trigger apt/rpm repo publish - env: - GH_TOKEN: ${{ github.token }} - TAG: ${{ github.ref_name }} - run: just gh trigger-repo-publish diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml new file mode 100644 index 0000000000..56b518f5a2 --- /dev/null +++ b/.github/workflows/release-binary.yml @@ -0,0 +1,253 @@ +name: release-binary + +# Builds, packages, and (on a tag) releases one workspace binary. moq-cli.yml +# and moq-relay.yml call it; they keep the tag trigger and the workflow name +# that alert.yml, release-brew.yml, and release-winget.yml match on. + +on: + workflow_call: + inputs: + crate: + description: "Cargo package to build, e.g. moq-cli" + required: true + type: string + bin: + description: "Executable the package ships, e.g. moq" + required: true + type: string + +permissions: + contents: read + +env: + CRATE: ${{ inputs.crate }} + BIN: ${{ inputs.bin }} + +jobs: + build: + name: Build (${{ matrix.target }}) + runs-on: ${{ matrix.runs-on }} + + strategy: + fail-fast: false + matrix: + include: + # Native runners. Each binary is built on the same arch it ships + # for, so the .deb/.rpm linkage matches the distro's glibc on the + # oldest LTS we support (Ubuntu 22.04 / AlmaLinux 9). + - target: x86_64-unknown-linux-gnu + runs-on: ubuntu-22.04 + deb-arch: amd64 + rpm-arch: x86_64 + - target: aarch64-unknown-linux-gnu + runs-on: ubuntu-22.04-arm + deb-arch: arm64 + rpm-arch: aarch64 + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/just + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + with: + targets: ${{ matrix.target }} + + - name: Parse version + id: parse + shell: bash + run: just gh release parse-version "$CRATE" + + - name: Set up packaging tools + uses: ./.github/actions/setup-packaging + + - name: Build (release, glibc 2.34) + shell: bash + # Pin to glibc 2.34 so one binary covers Ubuntu 22.04+ (.deb) + # and AlmaLinux/RHEL/Rocky 9+ (.rpm). + run: cargo zigbuild --locked --release --target ${{ matrix.target }}.2.34 -p "$CRATE" + + - name: Package release assets + shell: bash + env: + VERSION: ${{ steps.parse.outputs.version }} + TARGET: ${{ matrix.target }} + run: | + just rs package-binary \ + --crate "$CRATE" \ + --bin "$BIN" \ + --binary "target/${TARGET}/release/${BIN}" \ + --bare \ + --version "$VERSION" \ + --target "$TARGET" \ + --output dist + + # A transition.yaml beside nfpm.yaml is a deb-only package that keeps + # apt upgrades following a renamed package. + - name: Package .deb (Ubuntu 22.04 toolchain) + shell: bash + env: + VERSION: ${{ steps.parse.outputs.version }} + ARCH: ${{ matrix.deb-arch }} + BINARY_PATH: target/${{ matrix.target }}/release/${{ inputs.bin }} + run: | + just rs package-nfpm "packaging/$CRATE/nfpm.yaml" deb dist/ + if [[ -f "packaging/$CRATE/transition.yaml" ]]; then + just rs package-nfpm "packaging/$CRATE/transition.yaml" deb dist/ + fi + + - name: Package .rpm + shell: bash + env: + VERSION: ${{ steps.parse.outputs.version }} + ARCH: ${{ matrix.rpm-arch }} + BINARY_PATH: target/${{ matrix.target }}/release/${{ inputs.bin }} + run: just rs package-nfpm "packaging/$CRATE/nfpm.yaml" rpm dist/ + + - name: Upload artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.crate }}-${{ matrix.target }} + path: dist/* + + build-tarball-macos: + name: Build macOS tarball (${{ matrix.target }}) + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + include: + - target: aarch64-apple-darwin + os: macos-latest + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: ./.github/actions/just + + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + + - name: Parse version + id: parse + shell: bash + run: just gh release parse-version "$CRATE" + + - name: Build and package tarball + shell: bash + run: | + just rs package-binary \ + --crate "$CRATE" \ + --bin "$BIN" \ + --target ${{ matrix.target }} \ + --version ${{ steps.parse.outputs.version }} \ + --output dist + + - name: Upload artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.crate }}-${{ matrix.target }} + path: dist/* + + build-windows: + name: Build Windows zip (${{ matrix.target }}) + runs-on: windows-latest + + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-pc-windows-msvc + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/just + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + with: + targets: ${{ matrix.target }} + + # aws-lc-rs assembles its x86_64 crypto with NASM on Windows. + - name: Install NASM + shell: pwsh + run: | + choco install nasm -y --no-progress + "C:\Program Files\NASM" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Parse version + id: parse + shell: bash + run: just gh release parse-version "$CRATE" + + - name: Build and package zip + shell: bash + run: | + just rs package-windows \ + --crate "$CRATE" \ + --bin "$BIN" \ + --target ${{ matrix.target }} \ + --version ${{ steps.parse.outputs.version }} \ + --output dist + + - name: Upload artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.crate }}-${{ matrix.target }} + path: dist/* + + release: + name: Release + needs: [build, build-tarball-macos, build-windows] + if: github.ref_type == 'tag' + runs-on: ubuntu-latest + permissions: + contents: write + actions: write + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: ./.github/actions/just + + - name: Parse version + id: parse + run: just gh release parse-version "$CRATE" + + - name: Find previous tag + id: prev_tag + run: just gh release prev-tag "$CRATE" + + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + path: artifacts + merge-multiple: true + + - name: Create or update release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.ref_name }} + RELEASE_TITLE: "${{ inputs.crate }} v${{ steps.parse.outputs.version }}" + RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} + run: just gh release create artifacts + + - name: Trigger apt/rpm repo publish + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.ref_name }} + run: just gh trigger-repo-publish diff --git a/quest/m1/moq-installer.md b/quest/m1/moq-installer.md index 78b023d514..6dfcd4072e 100644 --- a/quest/m1/moq-installer.md +++ b/quest/m1/moq-installer.md @@ -76,8 +76,6 @@ service setup, Windows support, or new release targets. ## Related -- [Binary release workflow](/quest/m1/tooling/release-binary.md) - reuse its - artifacts without requiring workflow consolidation - [`moq relay`](/quest/m1/moq-relay-subcommand.md) - relay functionality joins the same executable independently of its installation method - [Install URL](/quest/m1/moq-install-url.md) - exposes this installer through diff --git a/quest/m1/tooling/README.md b/quest/m1/tooling/README.md index e01a88ebc7..c210608dc1 100644 --- a/quest/m1/tooling/README.md +++ b/quest/m1/tooling/README.md @@ -19,5 +19,4 @@ requires the one before it, so they land as one line of pull requests. ## Quests -- [Binary release workflow](/quest/m1/tooling/release-binary.md) - moq-cli and moq-relay share one reusable workflow behind two thin callers - [FFI release workflow](/quest/m1/tooling/release-ffi.md) - the five `release-*-ffi.yml` share the moq-ffi target matrix and artifact staging diff --git a/quest/m1/tooling/release-binary.md b/quest/m1/tooling/release-binary.md deleted file mode 100644 index 4a1dcdf0c5..0000000000 --- a/quest/m1/tooling/release-binary.md +++ /dev/null @@ -1,27 +0,0 @@ -# [M] One reusable workflow for the binary releases - -## Goal - -`moq-cli.yml` and `moq-relay.yml` are two copies of the same workflow (226 -and 225 lines) with the binary name swapped. They become one reusable -`release-binary.yml` taking `crate` and `bin` inputs, and two callers of a -dozen lines each that keep their tag trigger -and workflow name, so `alert.yml`, `release-brew.yml`, and -`release-winget.yml` keep matching on the names they match today. - -## Plan - -- `release-binary.yml` on `workflow_call` with inputs `crate` (the cargo - package) and `bin` (the executable name; `moq-cli` ships `moq`). It holds the Linux native-runner matrix, - the macOS tarball job, the Windows job, the `.deb` and `.rpm` packaging, the - release creation, and the repo-publish trigger, all as `just` recipes per - the preceding quest. -- Each caller keeps `name:` and `on.push.tags` exactly as they are and does - nothing but `uses: ./.github/workflows/release-binary.yml` with the two - inputs and the secrets it forwards. `workflow_run` consumers trigger off - the caller's name, so nothing downstream changes. -- The one text difference between the three today (a comment and a step name - about the glibc floor) belongs in the reusable file. -- Verify with `just gh check` and by diffing the rendered job list of each - caller against its predecessor with `gh workflow view`. The next tagged - release is the end-to-end check; say so in the PR. diff --git a/quest/m1/tooling/release-ffi.md b/quest/m1/tooling/release-ffi.md index 1627f981c8..58ac9efe08 100644 --- a/quest/m1/tooling/release-ffi.md +++ b/quest/m1/tooling/release-ffi.md @@ -24,12 +24,11 @@ only its packaging and publish steps. read the version from `rs/moq-ffi/Cargo.toml` instead. The reusable workflow keeps that fallback (an input or a ref check), or those PR runs die before building. -- Callers keep their `name:` and triggers, as the binary quest did, so - `alert.yml` and the `workflow_run` chains (`release-go.yml`, - `release-py.yml`, `release-swift-lib.yml`) are untouched. +- Callers keep their `name:` and triggers, as `moq-cli.yml` and + `moq-relay.yml` do over `release-binary.yml`, so `alert.yml` and the + `workflow_run` chains (`release-go.yml`, `release-py.yml`, + `release-swift-lib.yml`) are untouched. A calling job must grant every + permission a called job raises to, and `alert.sh` already skips a + `workflow_call`-only file. - Verify with `just gh check`; the next tagged `moq-ffi-v*` release is the end-to-end check. - -## Required - -- [Binary release workflow](/quest/m1/tooling/release-binary.md) - proves the reusable-plus-callers pattern before it is applied to five files diff --git a/sh/gh/alert.sh b/sh/gh/alert.sh index cfdec039c4..ee95b2d7e3 100755 --- a/sh/gh/alert.sh +++ b/sh/gh/alert.sh @@ -88,7 +88,7 @@ check_coverage() { return 1 } -# Names of workflows carrying at least one non-pull_request trigger. +# Names of workflows with a trigger that starts a run outside a pull request. # # The YAML is parsed rather than pattern-matched. GitHub accepts `on:` as a # block map, block sequence, flow sequence or bare scalar, any of which may wrap @@ -113,9 +113,10 @@ non_pr_workflow_names() { printf "%s\n" "${files[@]}" | bun -e ' const files = (await Bun.stdin.text()).split("\n").filter(Boolean); -// Both report their failure as a check on the PR itself, so alert.yml skips -// them at runtime and a workflow triggered only by these needs no entry. -const PR_EVENTS = new Set(["pull_request", "pull_request_target"]); +// The PR events report their failure as a check on the PR itself, and +// workflow_call runs inside the run that called it, so a workflow triggered +// only by these needs no entry. +const UNWATCHED = new Set(["pull_request", "pull_request_target", "workflow_call"]); const names = []; for (const file of files) { const doc = Bun.YAML.parse(await Bun.file(file).text()); @@ -132,7 +133,7 @@ for (const file of files) { console.error("alert.sh: cannot read the on: value of " + file); process.exit(2); } - if (!triggers.some((t) => !PR_EVENTS.has(t))) continue; + if (!triggers.some((t) => !UNWATCHED.has(t))) continue; const name = doc.name; if (typeof name !== "string" || name.trim() === "") { From e1997df5866c9a578a65ff02644619fbaf85c953 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 25 Sep 2026 15:43:18 -0700 Subject: [PATCH 5/6] chore(ci): the moq-ffi releases share one reusable build workflow (#4185) Co-authored-by: Claude Opus 5.5 --- .github/workflows/release-dart-ffi.yml | 166 ++++++++-------------- .github/workflows/release-ffi.yml | 174 ++++++++++++++++++++++++ .github/workflows/release-go-ffi.yml | 157 +++------------------ .github/workflows/release-kt-ffi.yml | 148 +++----------------- .github/workflows/release-swift-ffi.yml | 118 ++-------------- quest/m1/tooling/README.md | 9 +- quest/m1/tooling/release-ffi.md | 34 ----- 7 files changed, 288 insertions(+), 518 deletions(-) create mode 100644 .github/workflows/release-ffi.yml delete mode 100644 quest/m1/tooling/release-ffi.md diff --git a/.github/workflows/release-dart-ffi.yml b/.github/workflows/release-dart-ffi.yml index 2744516380..3b887b22c8 100644 --- a/.github/workflows/release-dart-ffi.yml +++ b/.github/workflows/release-dart-ffi.yml @@ -15,6 +15,7 @@ on: pull_request: paths: - ".github/workflows/release-dart-ffi.yml" + - ".github/workflows/release-ffi.yml" - "dart/moq_ffi/**" - "sh/dart/**" - "rs/moq-ffi/**" @@ -31,123 +32,71 @@ concurrency: group: release-dart-ffi-${{ github.ref_type == 'tag' && 'publish' || github.ref }} cancel-in-progress: ${{ github.ref_type != 'tag' }} -env: - CARGO_NDK_VERSION: 4.1.2 - jobs: - parse-version: - name: Parse version + ffi: + name: moq-ffi + uses: ./.github/workflows/release-ffi.yml + with: + language: dart + # The raw bindings are checked in under dart/moq_ffi, so none here. + bindings: false + default-features: false + targets: >- + [ + "aarch64-linux-android", + "armv7-linux-androideabi", + "x86_64-linux-android", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "aarch64-apple-ios", + "aarch64-apple-ios-sim", + "x86_64-apple-ios", + "x86_64-pc-windows-msvc" + ] + + # hook/build.dart downloads moq-ffi--- and its + # .sha256 from the release, so each target's shared library is renamed flat. + assets: + name: Stage Native Assets + needs: [ffi] runs-on: ubuntu-latest - outputs: - version: ${{ steps.parse.outputs.version }} - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Parse version - id: parse - run: just gh release parse-version moq-ffi - - build: - name: Build Native Asset (${{ matrix.target }}) - needs: [parse-version] - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - include: - - target: aarch64-linux-android - os: ubuntu-latest - - target: armv7-linux-androideabi - os: ubuntu-latest - - target: x86_64-linux-android - os: ubuntu-latest - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-apple-darwin - os: macos-latest - - target: x86_64-apple-darwin - os: macos-latest - - target: aarch64-apple-ios - os: macos-latest - - target: aarch64-apple-ios-sim - os: macos-latest - - target: x86_64-apple-ios - os: macos-latest - - target: x86_64-pc-windows-msvc - os: windows-latest steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - - name: Rust cache - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 + - name: Download per-target libs + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - save-if: ${{ github.event_name != 'pull_request' }} + path: libs-raw + pattern: dart-lib-* - - name: Install cross-compilation tools (Linux ARM64) - if: matrix.target == 'aarch64-unknown-linux-gnu' - run: | - sudo apt-get update - sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu - - - name: Install cargo-ndk (Android) - if: contains(matrix.target, 'android') - run: cargo install --locked "cargo-ndk@$CARGO_NDK_VERSION" - - - name: Build and stage asset - shell: bash + - name: Stage env: - BUILD_TARGET: ${{ matrix.target }} - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - IPHONEOS_DEPLOYMENT_TARGET: ${{ contains(matrix.target, 'apple-ios') && '16.0' || '' }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: | - just rs package-ffi \ - --target "$BUILD_TARGET" \ - --version "$BUILD_VERSION" \ - --output dist \ - --no-default-features - - case "$BUILD_TARGET" in - *-apple-ios* | *-apple-darwin*) LIBRARY=libmoq_ffi.dylib ;; - *-windows-*) LIBRARY=moq_ffi.dll ;; - *) LIBRARY=libmoq_ffi.so ;; - esac - - NAME="moq-ffi-$BUILD_VERSION-$BUILD_TARGET-$LIBRARY" mkdir -p assets - cp "dist/moq-ffi-$BUILD_VERSION-$BUILD_TARGET/lib/$LIBRARY" "assets/$NAME" - if command -v sha256sum >/dev/null 2>&1; then - sha256sum "assets/$NAME" > "assets/$NAME.sha256" - else - shasum -a 256 "assets/$NAME" > "assets/$NAME.sha256" - fi - - - name: Upload Native Asset + for dir in libs-raw/dart-lib-*; do + target="${dir#libs-raw/dart-lib-}" + case "$target" in + *-apple-ios* | *-apple-darwin*) library=libmoq_ffi.dylib ;; + *-windows-*) library=moq_ffi.dll ;; + *) library=libmoq_ffi.so ;; + esac + name="moq-ffi-$BUILD_VERSION-$target-$library" + cp "$dir/$library" "assets/$name" + (cd assets && sha256sum "$name" > "$name.sha256") + done + ls -la assets + + - name: Upload Native Assets uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: dart-asset-${{ matrix.target }} + name: dart-assets path: assets/* package: name: Check pub package - needs: [parse-version] + needs: [ffi] runs-on: ubuntu-latest steps: @@ -161,7 +110,7 @@ jobs: - name: Stage and check package env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: | just dart package \ --package moq_ffi \ @@ -180,7 +129,7 @@ jobs: release: name: Attach Native Assets - needs: [parse-version, build] + needs: [ffi, assets] if: github.ref_type == 'tag' runs-on: ubuntu-latest permissions: @@ -201,22 +150,21 @@ jobs: - name: Download Native Assets uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: + name: dart-assets path: artifacts - pattern: dart-asset-* - merge-multiple: true - name: Create or update release env: GH_TOKEN: ${{ github.token }} RELEASE_TAG: ${{ github.ref_name }} - RELEASE_TITLE: "moq-ffi v${{ needs.parse-version.outputs.version }}" + RELEASE_TITLE: "moq-ffi v${{ needs.ffi.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} RELEASE_CHECKSUMS: SHA256SUMS-dart run: just gh release create artifacts publish: name: Publish moq_ffi to pub.dev - needs: [parse-version, package, release] + needs: [ffi, package, release] if: github.ref_type == 'tag' runs-on: ubuntu-latest environment: @@ -235,5 +183,5 @@ jobs: - name: Publish env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: cd "package/moq_ffi-$BUILD_VERSION" && dart pub publish --force diff --git a/.github/workflows/release-ffi.yml b/.github/workflows/release-ffi.yml new file mode 100644 index 0000000000..26eaf7fa90 --- /dev/null +++ b/.github/workflows/release-ffi.yml @@ -0,0 +1,174 @@ +name: release-ffi + +# Builds rs/moq-ffi for each requested target and generates its UniFFI +# bindings. The release-{dart,go,kt,swift}-ffi.yml callers package and publish; +# they keep the moq-ffi-v* tag trigger and the workflow name that alert.yml and +# the workflow_run chains match on. +# +# Uploads, all prefixed by `language` so callers sharing one nightly run never +# collide: +# -lib- the target's native libraries (the lib/ directory) +# -bindings dist/bindings//, when `bindings` is set + +on: + workflow_call: + inputs: + language: + description: "Binding language and artifact prefix, e.g. kotlin" + required: true + type: string + targets: + description: "JSON array of Rust targets to build" + required: true + type: string + bindings: + description: "Generate and upload the bindings for `language`" + type: boolean + default: true + default-features: + description: "Build moq-ffi with its default features (audio and video)" + type: boolean + default: true + outputs: + version: + description: "moq-ffi version, from the tag or (off a tag) rs/moq-ffi/Cargo.toml" + value: ${{ jobs.parse-version.outputs.version }} + +permissions: + contents: read + +# Pinned rather than "latest": `--locked` only fixes the tool's own +# dependencies, not which version of the tool cargo selects. +env: + CARGO_NDK_VERSION: 4.1.2 + +jobs: + parse-version: + name: Parse version + runs-on: ubuntu-latest + outputs: + version: ${{ steps.parse.outputs.version }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/just + + - name: Parse version + id: parse + run: just gh release parse-version moq-ffi + + build: + name: Build moq-ffi (${{ matrix.target }}) + needs: [parse-version] + runs-on: ${{ contains(matrix.target, '-apple-') && 'macos-latest' || contains(matrix.target, '-windows-') && 'windows-latest' || 'ubuntu-latest' }} + + strategy: + fail-fast: false + matrix: + target: ${{ fromJSON(inputs.targets) }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/just + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + with: + targets: ${{ matrix.target }} + + - name: Rust cache + uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 + with: + # Don't let untrusted PR builds populate caches reused by trusted runs. + save-if: ${{ github.event_name != 'pull_request' }} + + - name: Install cross-compilation tools (Linux ARM64) + if: matrix.target == 'aarch64-unknown-linux-gnu' + run: | + sudo apt-get update + # g++ as well as gcc: moq-video's openh264 fallback is C++, and + # cc-rs looks for aarch64-linux-gnu-g++ by name. + sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu + + - name: Install cargo-ndk (Android) + if: contains(matrix.target, 'android') + run: cargo install --locked "cargo-ndk@$CARGO_NDK_VERSION" + + - name: Build + shell: bash + env: + BUILD_TARGET: ${{ matrix.target }} + BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + DEFAULT_FEATURES: ${{ inputs.default-features }} + # Set iOS deployment target to avoid ___chkstk_darwin linker errors. + IPHONEOS_DEPLOYMENT_TARGET: ${{ contains(matrix.target, 'apple-ios') && '16.0' || '' }} + run: | + args=(--target "$BUILD_TARGET" --version "$BUILD_VERSION" --output dist) + if [[ "$DEFAULT_FEATURES" == false ]]; then + args+=(--no-default-features) + fi + just rs package-ffi "${args[@]}" + + - name: Upload lib + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.language }}-lib-${{ matrix.target }} + # Only the lib/ contents, so a caller's download is /. + path: dist/moq-ffi-${{ needs.parse-version.outputs.version }}-${{ matrix.target }}/lib/ + + bindings: + name: Generate bindings + needs: [parse-version] + if: inputs.bindings + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/just + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + + - name: Rust cache + uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 + with: + # Don't let untrusted PR builds populate caches reused by trusted runs. + save-if: ${{ github.event_name != 'pull_request' }} + + # Go uses a third-party generator; build.sh skips Go when it is absent. + - name: Install uniffi-bindgen-go + if: inputs.language == 'go' + env: + # A fork until NordSecurity ships a uniffi 0.32 generator; see flake.nix. + UNIFFI_BINDGEN_GO_REPO: https://github.com/kixelated/uniffi-bindgen-go + UNIFFI_BINDGEN_GO_REV: v0.9.0+v0.32.0 + run: | + cargo install --locked uniffi-bindgen-go \ + --git "$UNIFFI_BINDGEN_GO_REPO" \ + --rev "$UNIFFI_BINDGEN_GO_REV" + + - name: Generate bindings + env: + BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + run: | + just rs package-ffi \ + --bindings-only \ + --version "$BUILD_VERSION" \ + --output dist + + - name: Upload bindings + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.language }}-bindings + path: dist/bindings/${{ inputs.language }}/ + # A language build.sh does not generate would otherwise upload nothing. + if-no-files-found: error diff --git a/.github/workflows/release-go-ffi.yml b/.github/workflows/release-go-ffi.yml index 2039f200b3..e35e940046 100644 --- a/.github/workflows/release-go-ffi.yml +++ b/.github/workflows/release-go-ffi.yml @@ -16,6 +16,7 @@ on: pull_request: paths: - ".github/workflows/release-go-ffi.yml" + - ".github/workflows/release-ffi.yml" - "sh/gh/release.sh" - "go/ffi/**" - "sh/go/package-ffi.sh" @@ -33,141 +34,27 @@ concurrency: group: release-go-ffi-${{ github.ref_type == 'tag' && 'publish' || github.ref }} cancel-in-progress: ${{ github.ref_type != 'tag' }} -env: - # A fork until NordSecurity ships a uniffi 0.32 generator; see flake.nix. - # The revision is v0.9.0+v0.32.0. - UNIFFI_BINDGEN_GO_REPO: https://github.com/kixelated/uniffi-bindgen-go - UNIFFI_BINDGEN_GO_REV: v0.9.0+v0.32.0 - jobs: - parse-version: - name: Parse version - runs-on: ubuntu-latest - outputs: - version: ${{ steps.parse.outputs.version }} - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Parse version - id: parse - run: just gh release parse-version moq-ffi - - build: - name: Build moq-ffi (${{ matrix.target }}) - needs: [parse-version] - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - include: - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-apple-darwin - os: macos-latest - - target: x86_64-pc-windows-msvc - os: windows-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - - name: Rust cache - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 - with: - # Don't let untrusted PR builds populate caches reused by trusted runs. - save-if: ${{ github.event_name != 'pull_request' }} - - - name: Install cross-compilation tools (Linux ARM64) - if: matrix.target == 'aarch64-unknown-linux-gnu' - run: | - sudo apt-get update - # g++ as well as gcc: moq-video's openh264 fallback is C++, and - # cc-rs looks for aarch64-linux-gnu-g++ by name. - sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu - - - name: Build - shell: bash - env: - BUILD_TARGET: ${{ matrix.target }} - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: | - just rs package-ffi \ - --target "$BUILD_TARGET" \ - --version "$BUILD_VERSION" \ - --output dist - - - name: Upload lib - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: go-lib-${{ matrix.target }} - # package.sh expects /; upload just lib/ contents. - path: dist/moq-ffi-${{ needs.parse-version.outputs.version }}-${{ matrix.target }}/lib/ - - bindings: - name: Generate bindings - needs: [parse-version] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - - - name: Rust cache - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 - with: - # Don't let untrusted PR builds populate caches reused by trusted runs. - save-if: ${{ github.event_name != 'pull_request' }} - - - name: Install uniffi-bindgen-go - run: | - cargo install --locked uniffi-bindgen-go \ - --git "$UNIFFI_BINDGEN_GO_REPO" \ - --rev "$UNIFFI_BINDGEN_GO_REV" - - - name: Generate bindings - env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: | - just rs package-ffi \ - --bindings-only \ - --version "$BUILD_VERSION" \ - --output dist - - - name: Upload bindings - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: go-bindings - path: dist/bindings/go/ + ffi: + name: moq-ffi + uses: ./.github/workflows/release-ffi.yml + with: + language: go + targets: >- + [ + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc" + ] package: name: Package - needs: [parse-version, build, bindings] + needs: [ffi] runs-on: ubuntu-latest steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -214,7 +101,7 @@ jobs: - name: Package env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: | just go package-ffi \ --version "$BUILD_VERSION" \ @@ -231,12 +118,12 @@ jobs: publish: name: Publish to Go module mirror - needs: [package, parse-version] + needs: [ffi, package] runs-on: ubuntu-latest if: github.ref_type == 'tag' steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -261,7 +148,7 @@ jobs: - name: Publish to moq-dev/moq-go-ffi mirror env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} # Token is minted fresh per run, expires in 1 hour, never at rest. GO_FFI_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot @@ -270,12 +157,12 @@ jobs: publish-dry-run: name: Publish dry-run - needs: [package, parse-version] + needs: [ffi, package] runs-on: ubuntu-latest if: github.ref_type != 'tag' steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -289,5 +176,5 @@ jobs: - name: Dry-run publish to mirror env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: just go publish-ffi --dry-run diff --git a/.github/workflows/release-kt-ffi.yml b/.github/workflows/release-kt-ffi.yml index 207e70f35b..0ce2867daf 100644 --- a/.github/workflows/release-kt-ffi.yml +++ b/.github/workflows/release-kt-ffi.yml @@ -20,137 +20,27 @@ concurrency: group: release-kt-ffi cancel-in-progress: false -# Pinned rather than "latest": `--locked` only fixes the tool's own -# dependencies, not which version of the tool cargo selects. -env: - CARGO_NDK_VERSION: 4.1.2 - jobs: - parse-version: - name: Parse version - runs-on: ubuntu-latest - outputs: - version: ${{ steps.parse.outputs.version }} - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Parse version - id: parse - run: just gh release parse-version moq-ffi - - build: - name: Build moq-ffi (${{ matrix.target }}) - needs: [parse-version] - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - include: - # Android - - target: aarch64-linux-android - os: ubuntu-latest - - target: armv7-linux-androideabi - os: ubuntu-latest - - target: x86_64-linux-android - os: ubuntu-latest - # Desktop JVM - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-apple-darwin - os: macos-latest - - target: x86_64-pc-windows-msvc - os: windows-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - - name: Rust cache - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 - - - name: Install cross-compilation tools (Linux ARM64) - if: matrix.target == 'aarch64-unknown-linux-gnu' - run: | - sudo apt-get update - # g++ as well as gcc: moq-video's openh264 fallback is C++, and - # cc-rs looks for aarch64-linux-gnu-g++ by name. - sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu - - - name: Install cargo-ndk (Android) - if: contains(matrix.target, 'android') - run: cargo install --locked "cargo-ndk@$CARGO_NDK_VERSION" - - - name: Build - shell: bash - env: - BUILD_TARGET: ${{ matrix.target }} - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: | - just rs package-ffi \ - --target "$BUILD_TARGET" \ - --version "$BUILD_VERSION" \ - --output dist - - - name: Upload lib - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: kotlin-lib-${{ matrix.target }} - # Upload only the lib/ contents; package.sh expects /. - path: dist/moq-ffi-${{ needs.parse-version.outputs.version }}-${{ matrix.target }}/lib/ - - bindings: - name: Generate bindings - needs: [parse-version] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - - - name: Rust cache - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 - - - name: Generate bindings - env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: | - just rs package-ffi \ - --bindings-only \ - --version "$BUILD_VERSION" \ - --output dist - - - name: Upload bindings - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: kotlin-bindings - # Upload only the kotlin subdir. - path: dist/bindings/kotlin/ + ffi: + name: moq-ffi + uses: ./.github/workflows/release-ffi.yml + with: + language: kotlin + # Android, then the desktop JVM. + targets: >- + [ + "aarch64-linux-android", + "armv7-linux-androideabi", + "x86_64-linux-android", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc" + ] package: name: Package and publish - needs: [parse-version, build, bindings] + needs: [ffi] runs-on: ubuntu-latest env: # Picked up by com.vanniktech.maven.publish via Gradle project properties. @@ -205,7 +95,7 @@ jobs: - name: Stage native libs + bindings into the gradle module env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: | just kt package \ --version "$BUILD_VERSION" \ @@ -216,7 +106,7 @@ jobs: - name: Publish to Maven Central if: github.ref_type == 'tag' env: - MOQFFI_VERSION: ${{ needs.parse-version.outputs.version }} + MOQFFI_VERSION: ${{ needs.ffi.outputs.version }} ORG_GRADLE_PROJECT_signingInMemoryKey: ${{ secrets.SIGNING_KEY }} ORG_GRADLE_PROJECT_signingInMemoryKeyPassword: ${{ secrets.SIGNING_PASSWORD }} run: | diff --git a/.github/workflows/release-swift-ffi.yml b/.github/workflows/release-swift-ffi.yml index 0ab0eaa056..afa2fbf175 100644 --- a/.github/workflows/release-swift-ffi.yml +++ b/.github/workflows/release-swift-ffi.yml @@ -23,106 +23,16 @@ concurrency: cancel-in-progress: false jobs: - parse-version: - name: Parse version - runs-on: ubuntu-latest - outputs: - version: ${{ steps.parse.outputs.version }} - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Parse version - id: parse - run: just gh release parse-version moq-ffi - - build: - name: Build moq-ffi (${{ matrix.target }}) - needs: [parse-version] - runs-on: macos-latest - - strategy: - fail-fast: false - matrix: - target: - - aarch64-apple-ios - - aarch64-apple-ios-sim - - aarch64-apple-darwin - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - - name: Rust cache - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 - - - name: Build - shell: bash - env: - BUILD_TARGET: ${{ matrix.target }} - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - # Set iOS deployment target to avoid ___chkstk_darwin linker errors. - IPHONEOS_DEPLOYMENT_TARGET: ${{ contains(matrix.target, 'apple-ios') && '16.0' || '' }} - run: | - just rs package-ffi \ - --target "$BUILD_TARGET" \ - --version "$BUILD_VERSION" \ - --output dist - - - name: Upload lib - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: swift-lib-${{ matrix.target }} - path: dist/moq-ffi-${{ needs.parse-version.outputs.version }}-${{ matrix.target }}/lib/ - - bindings: - name: Generate bindings - needs: [parse-version] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - - - name: Rust cache - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 - - - name: Generate bindings - env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: | - just rs package-ffi \ - --bindings-only \ - --version "$BUILD_VERSION" \ - --output dist - - - name: Upload bindings - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: swift-bindings - path: dist/bindings/swift/ + ffi: + name: moq-ffi + uses: ./.github/workflows/release-ffi.yml + with: + language: swift + targets: '["aarch64-apple-ios", "aarch64-apple-ios-sim", "aarch64-apple-darwin"]' package: name: Package - needs: [parse-version, build, bindings] + needs: [ffi] runs-on: macos-latest steps: @@ -155,7 +65,7 @@ jobs: - name: Package env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: | just swift package-ffi \ --version "$BUILD_VERSION" \ @@ -175,7 +85,7 @@ jobs: # points at this asset, so it must exist before any consumer resolves the # SPM tag. name: Attach assets to GitHub release - needs: [package, parse-version] + needs: [ffi, package] if: github.ref_type == 'tag' runs-on: ubuntu-latest permissions: @@ -205,7 +115,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} RELEASE_TAG: ${{ github.ref_name }} - RELEASE_TITLE: "moq-ffi v${{ needs.parse-version.outputs.version }}" + RELEASE_TITLE: "moq-ffi v${{ needs.ffi.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} run: just gh release create artifacts @@ -215,7 +125,7 @@ jobs: # fine but SPM cannot resolve (e.g. a path-based binaryTarget slipping in # where a URL+checksum is expected). name: Verify staged package resolves - needs: [release, parse-version] + needs: [ffi, release] runs-on: macos-latest steps: @@ -233,13 +143,13 @@ jobs: - name: Resolve and build smoke consumer env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: | just swift verify-ffi --tarball "artifacts/moq-ffi-${BUILD_VERSION}-swift-ffi.tar.gz" publish: name: Publish to Swift Package mirror - needs: [verify, parse-version] + needs: [ffi, verify] runs-on: ubuntu-latest steps: @@ -268,7 +178,7 @@ jobs: - name: Publish to moq-dev/moq-swift-ffi mirror env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} # Token is minted fresh per run, expires in 1 hour, never at rest. SWIFT_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot diff --git a/quest/m1/tooling/README.md b/quest/m1/tooling/README.md index c210608dc1..2cfdc22482 100644 --- a/quest/m1/tooling/README.md +++ b/quest/m1/tooling/README.md @@ -1,4 +1,4 @@ -# Tooling: thin justfiles and CI that calls them +# [XS] Tooling: thin justfiles and CI that calls them ## Goal @@ -14,9 +14,4 @@ runs a recipe rather than a script path. `just` stays as the entry point because the vocabulary (`just check`, `just fix`) is in every doc, skill, and workflow. Its cost was -self-inflicted: logic inside recipes. The quests below are ordered and each -requires the one before it, so they land as one line of pull requests. - -## Quests - -- [FFI release workflow](/quest/m1/tooling/release-ffi.md) - the five `release-*-ffi.yml` share the moq-ffi target matrix and artifact staging +self-inflicted: logic inside recipes. diff --git a/quest/m1/tooling/release-ffi.md b/quest/m1/tooling/release-ffi.md deleted file mode 100644 index 58ac9efe08..0000000000 --- a/quest/m1/tooling/release-ffi.md +++ /dev/null @@ -1,34 +0,0 @@ -# [M] One reusable workflow for the moq-ffi releases - -## Goal - -`release-dart-ffi.yml`, `release-go-ffi.yml`, `release-kt-ffi.yml`, -`release-swift-ffi.yml`, and `release-py-ffi.yml` share half to two thirds of -their lines: the moq-ffi cross-target build matrix and the artifact staging. -That shared half becomes one reusable `release-ffi.yml`; each language keeps -only its packaging and publish steps. - -## Plan - -- Measure the shared lines first (`comm` over the sorted files shows 74 to - 176 common lines per pair) and confirm the matrix targets agree. Where a - language needs a different target set (iOS simulator for swift, Android - ABIs for kt and dart), make the target list an input rather than forcing - one matrix. -- `release-ffi.yml` on `workflow_call`: builds `rs/moq-ffi` for the requested - targets, uploads one artifact per target, and outputs the version parsed - from the tag. Callers download the artifacts and run their `just - package` and publish recipes. -- `release-dart-ffi.yml` and `release-go-ffi.yml` also run on `pull_request`, - where `GITHUB_REF` is not a `moq-ffi-v*` tag and `parse-version` fails; they - read the version from `rs/moq-ffi/Cargo.toml` instead. The reusable - workflow keeps that fallback (an input or a ref check), or those PR runs die - before building. -- Callers keep their `name:` and triggers, as `moq-cli.yml` and - `moq-relay.yml` do over `release-binary.yml`, so `alert.yml` and the - `workflow_run` chains (`release-go.yml`, `release-py.yml`, - `release-swift-lib.yml`) are untouched. A calling job must grant every - permission a called job raises to, and `alert.sh` already skips a - `workflow_call`-only file. -- Verify with `just gh check`; the next tagged `moq-ffi-v*` release is the - end-to-end check. From 3b76b6460b86f9b4a64c8d9f4b876cd0b0befa0c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 25 Sep 2026 15:58:34 -0700 Subject: [PATCH 6/6] quest(tooling): scoped checks refuse a host toolchain (#4176) Co-authored-by: Claude Opus 5.5 Co-authored-by: Grok 4.7 --- quest/m1/tooling/README.md | 6 +++++- quest/m1/tooling/nix-guard.md | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 quest/m1/tooling/nix-guard.md diff --git a/quest/m1/tooling/README.md b/quest/m1/tooling/README.md index 2cfdc22482..b29d374fa9 100644 --- a/quest/m1/tooling/README.md +++ b/quest/m1/tooling/README.md @@ -1,4 +1,4 @@ -# [XS] Tooling: thin justfiles and CI that calls them +# Tooling: thin justfiles and CI that calls them ## Goal @@ -15,3 +15,7 @@ runs a recipe rather than a script path. `just` stays as the entry point because the vocabulary (`just check`, `just fix`) is in every doc, skill, and workflow. Its cost was self-inflicted: logic inside recipes. + +## Quests + +- [Nix guard](/quest/m1/tooling/nix-guard.md) - the scoped check, fix, and test refuse to run outside the Nix dev shell unless told to diff --git a/quest/m1/tooling/nix-guard.md b/quest/m1/tooling/nix-guard.md new file mode 100644 index 0000000000..e9ab5934bc --- /dev/null +++ b/quest/m1/tooling/nix-guard.md @@ -0,0 +1,29 @@ +# [S] Scoped checks refuse a host toolchain + +## Goal + +`just check`, `just fix`, and `just ci test` stop at once with a clear "run +inside `nix develop`" message when they run outside the Nix dev shell, instead +of failing deep in a build on a host toolchain difference. The repository +already requires the dev shell; this makes the requirement loud. Set +`MOQ_ALLOW_HOST=1` to keep a deliberate host run possible. + +## Plan + +- The trigger: `moq-tokio --all-features` builds jemalloc, whose configure + adds `-Werror` to its `strerror_r` probes, so any host warning (likely + `_FORTIFY_SOURCE` at `-O0`) makes it report "cannot determine return type of + strerror_r". The dev shell disables fortify hardening, which is why CI + passes. Reproduce it once outside the shell and confirm the cause from + jemalloc's `config.log` before relying on it. +- Put the guard in `sh/dispatch.sh`, the one place the scoped verbs resolve, + next to the existing `MOQ_STRICT` missing-tool check. The verbs are `check`, + `fix`, `ci-check`, and `ci-test`. `just test` is the cross-language module + and does not reach this script, so it stays unguarded. How to detect the dev + shell (`IN_NIX_SHELL`, a variable the flake sets, or a toolchain probe) is + the implementer's call; prefer whatever direnv and `nix develop --command` + both set. +- CI already runs inside the dev shell, so it is unaffected. Verify that a + direnv-loaded shell passes the guard. + +Public API: none. Wire: none.