diff --git a/.github/workflows/sf_release.yml b/.github/workflows/sf_release.yml index 5608b9b36a..b56244c4b2 100644 --- a/.github/workflows/sf_release.yml +++ b/.github/workflows/sf_release.yml @@ -14,15 +14,86 @@ on: branches: - release/* +# Two runs on the same ref (a re-run, or a workflow_dispatch fired while a push +# run is still going) resolve the same VERSION and therefore push the same +# per-arch image tags. Serialize them so a second run cannot overwrite the +# per-arch images the first run is still assembling into a manifest. Queue +# rather than cancel: a half-published release is worse than a slow one. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} FIREHOSE_ETHEREUM_IMAGE: ${{ inputs.firehose_ethereum_image || 'ghcr.io/streamingfast/firehose-ethereum:latest' }} jobs: - push: - name: Docker build & push + prepare: + name: Resolve versions runs-on: ubuntu-24.04 + permissions: + contents: read + packages: read + outputs: + version: ${{ steps.extract-versions.outputs.VERSION }} + fireeth_image: ${{ steps.extract-versions.outputs.FIREETH_IMAGE }} + image_name: ${{ steps.image.outputs.name }} + steps: + # The digest below is resolved against the registry, so authenticate even + # though the firehose-ethereum image is public today. + - name: Log in to the Container registry + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Lowercase image name + id: image + # Container registries require lowercase repository names; github.repository + # preserves the org's original casing, so normalize it here. + run: echo "name=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT" + + - name: Extract versions + id: extract-versions + env: + FIREETH_REF: ${{ env.FIREHOSE_ETHEREUM_IMAGE }} + run: | + set -euo pipefail + + version="edge-${GITHUB_SHA::7}" + if [[ "${GITHUB_REF}" == refs/tags/* ]]; then + version=${GITHUB_REF#refs/tags/} + fi + + # Pin the Firehose Ethereum base by digest. The amd64 and arm64 legs + # build on separate runners and each resolves its own FROM, so a moving + # tag can otherwise put two different fireeth builds in one manifest. + fireeth_digest=$(docker buildx imagetools inspect "${FIREETH_REF}" --format '{{.Manifest.Digest}}') + + # Strip any tag or digest to get the bare repository. Only treat a ':' + # in the last path segment as a tag separator, so a registry port + # (host:5000/repo) survives. + fireeth_repo="${FIREETH_REF%@*}" + case "${fireeth_repo##*/}" in + *:*) fireeth_repo="${fireeth_repo%:*}" ;; + esac + + echo "VERSION=${version}" >> "$GITHUB_OUTPUT" + echo "FIREETH_IMAGE=${fireeth_repo}@${fireeth_digest}" >> "$GITHUB_OUTPUT" + + build: + name: Build & push (${{ matrix.arch }}) + needs: prepare + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} permissions: contents: read packages: write @@ -55,79 +126,168 @@ jobs: submodules: recursive - name: Log in to the Container registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract versions - id: extract-versions - run: | - version="edge-${GITHUB_SHA::7}" - if [[ "${GITHUB_REF}" == refs/tags/* ]]; then - version=${GITHUB_REF#refs/tags/} - fi - - docker pull "${{ env.FIREHOSE_ETHEREUM_IMAGE }}" - fireeth_version=$(docker inspect "${{ env.FIREHOSE_ETHEREUM_IMAGE }}" --format='{{index .Config.Labels "org.opencontainers.image.version"}}') - - echo "VERSION=${version}" >> "$GITHUB_OUTPUT" - echo "FIREETH_VERSION=${fireeth_version}" >> "$GITHUB_OUTPUT" - echo "FIREETH_DOCKER_SUFFIX=${version}-fireeth-${fireeth_version}" >> "$GITHUB_OUTPUT" - - - name: Generate docker tags/labels from github build context + - name: Generate docker labels from github build context id: meta-bare - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v6 with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=ref,event=tag - type=sha,prefix= - type=raw,value=${{ steps.extract-versions.outputs.VERSION }} - type=raw,value=${{ steps.extract-versions.outputs.VERSION }},suffix=${{ steps.extract-versions.outputs.FIREETH_DOCKER_SUFFIX }} - type=raw,enable=${{ github.ref == 'refs/heads/develop' }},value=develop + images: ${{ env.REGISTRY }}/${{ needs.prepare.outputs.image_name }} - - name: Build and push nitro Docker image - uses: docker/build-push-action@v6 + - name: Build nitro Docker image + uses: docker/build-push-action@v7 with: context: . target: nitro-node file: ./Dockerfile push: false - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-base:${{ steps.extract-versions.outputs.VERSION }} + tags: ${{ env.REGISTRY }}/${{ needs.prepare.outputs.image_name }}-base:${{ needs.prepare.outputs.version }} labels: ${{ steps.meta-bare.outputs.labels }} - - name: Build and push final (fireeth + nitro) Docker image - uses: docker/build-push-action@v6 + # The per-arch tag is a debugging handle only. The manifest job assembles + # the release from the digests exported below, never from this tag, so a + # concurrent run overwriting it cannot corrupt a published manifest. + - name: Build and push final (fireeth + nitro) per-arch image + id: build + uses: docker/build-push-action@v7 with: context: . file: ./Dockerfile.sf push: true build-args: | - NITRO_BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-base:${{ steps.extract-versions.outputs.VERSION }} - FIREHOSE_ETHEREUM_IMAGE=${{ env.FIREHOSE_ETHEREUM_IMAGE }} - tags: ${{ steps.meta-bare.outputs.tags }} + NITRO_BASE_IMAGE=${{ env.REGISTRY }}/${{ needs.prepare.outputs.image_name }}-base:${{ needs.prepare.outputs.version }} + FIREHOSE_ETHEREUM_IMAGE=${{ needs.prepare.outputs.fireeth_image }} + tags: ${{ env.REGISTRY }}/${{ needs.prepare.outputs.image_name }}:${{ needs.prepare.outputs.version }}-${{ matrix.arch }} labels: ${{ steps.meta-bare.outputs.labels }} + - name: Export image digest + env: + ARCH: ${{ matrix.arch }} + BUILD_DIGEST: ${{ steps.build.outputs.digest }} + IMAGE_TAG: ${{ env.REGISTRY }}/${{ needs.prepare.outputs.image_name }}:${{ needs.prepare.outputs.version }}-${{ matrix.arch }} + run: | + set -euo pipefail + + digest="${BUILD_DIGEST}" + if [ -z "${digest}" ]; then + digest=$(docker buildx imagetools inspect "${IMAGE_TAG}" --format '{{.Manifest.Digest}}') + fi + if [ -z "${digest}" ]; then + echo "unable to resolve the pushed image digest for ${IMAGE_TAG}" >&2 + exit 1 + fi + + mkdir -p /tmp/digests + echo "${digest}" > "/tmp/digests/${ARCH}" + + - name: Upload image digest + uses: actions/upload-artifact@v7 + with: + name: digests-${{ matrix.arch }} + path: /tmp/digests/${{ matrix.arch }} + if-no-files-found: error + retention-days: 1 + + manifest: + name: Create multi-arch manifests + needs: [prepare, build] + runs-on: ubuntu-24.04 + permissions: + contents: read + packages: write + + steps: + - name: Log in to the Container registry + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Download image digests + uses: actions/download-artifact@v8 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + + - name: Generate docker tags from github build context + id: meta-bare + uses: docker/metadata-action@v6 + with: + images: ${{ env.REGISTRY }}/${{ needs.prepare.outputs.image_name }} + tags: | + type=ref,event=tag + # Only branch builds claim the bare commit-sha tag. A tag pushed on a + # commit that is also a release/* branch head starts two runs, and + # both would otherwise publish this same tag, leaving whichever + # finished last as the winner. + type=sha,prefix=,enable=${{ !startsWith(github.ref, 'refs/tags/') }} + type=raw,value=${{ needs.prepare.outputs.version }} + type=raw,enable=${{ github.ref == 'refs/heads/develop' }},value=develop + + - name: Create and push manifests + env: + IMAGE_NAME: ${{ needs.prepare.outputs.image_name }} + TAGS: ${{ steps.meta-bare.outputs.tags }} + run: | + set -euo pipefail + + # Reference the per-arch images by digest: content-addressed, so no + # concurrent run can substitute a different image under us. + sources=() + for digest_file in /tmp/digests/*; do + sources+=("${REGISTRY}/${IMAGE_NAME}@$(cat "${digest_file}")") + done + if [ "${#sources[@]}" -eq 0 ]; then + echo "no per-arch digests were published" >&2 + exit 1 + fi + echo "manifest sources: ${sources[*]}" + + # On a tag push, type=ref and type=raw,value=VERSION both resolve to the + # tag name, so deduplicate before pushing. + while IFS= read -r tag; do + [ -n "$tag" ] || continue + docker buildx imagetools create -t "$tag" "${sources[@]}" + done < <(printf '%s\n' "$TAGS" | sort -u) + release: if: startsWith(github.ref, 'refs/tags/') - needs: push + needs: [prepare, manifest] runs-on: ubuntu-24.04 permissions: contents: write + packages: read steps: - - name: Extract image - id: image - run: | - # The run is done only on tags, so we can ref/tags/ replace directly here - echo "ID=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${GITHUB_REF#refs/tags/}" >> "$GITHUB_OUTPUT" + - name: Log in to the Container registry + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract assets + - name: Extract binaries + env: + IMAGE_NAME: ${{ needs.prepare.outputs.image_name }} + VERSION: ${{ needs.prepare.outputs.version }} run: | - # The --platform are not really needed here, but it removes the warning - docker cp $(docker create --platform=linux/amd64 ${{ steps.image.outputs.ID }}):/usr/local/bin/nitro ./nitro_linux_amd64 + set -euo pipefail + + # Read from the multi-arch manifest this run just published and let + # --platform select the variant. The container is only created, never + # started, so pulling a foreign architecture here is fine. + for arch in amd64 arm64; do + src="${REGISTRY}/${IMAGE_NAME}:${VERSION}" + container=$(docker create --platform="linux/${arch}" "$src") + docker cp "${container}:/usr/local/bin/nitro" "./nitro_linux_${arch}" + docker rm "${container}" >/dev/null + done - name: Extract Changelog id: changelog @@ -146,3 +306,4 @@ jobs: prerelease: ${{ contains(github.ref, 'beta') || contains(github.ref, 'rc') }} files: | ./nitro_linux_amd64 + ./nitro_linux_arm64 diff --git a/CHANGELOG.sf.md b/CHANGELOG.sf.md index 8c1bd463d1..640749cbf6 100644 --- a/CHANGELOG.sf.md +++ b/CHANGELOG.sf.md @@ -1,3 +1,10 @@ +## Unreleased + +* Added `linux/arm64` support: the published Docker image is now a multi-arch manifest covering `linux/amd64` and `linux/arm64`, and releases ship a `nitro_linux_arm64` binary alongside `nitro_linux_amd64`. +* Changed the Firehose Ethereum base image to be pinned by digest for the whole build, so both architectures in a manifest are guaranteed to embed the same `firehose-ethereum` build. +* Removed the `-fireeth-` image tag. Releases are identified by their version alone, e.g. `v3.11.2-fh3.0`. +* Fixed the bare commit-sha image tag being published by both the branch run and the tag run when a tag is pushed on a `release/*` branch head, where whichever finished last silently won. Only branch builds publish it now. + ## v3.11.2-fh3.0 * Bumped to [3.11.2](https://github.com/OffchainLabs/nitro/releases/tag/v3.11.2).