Arm64 support for SF releases (binary and images)

.github/workflows/sf_release.yml

@@ -14,15 +14,86 @@ on:
     branches:
       - release/*
 
+# Two runs on the same ref (a re-run, or a workflow_dispatch fired while a push
+# run is still going) resolve the same VERSION and therefore push the same
+# per-arch image tags. Serialize them so a second run cannot overwrite the
+# per-arch images the first run is still assembling into a manifest. Queue
+# rather than cancel: a half-published release is worse than a slow one.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
   FIREHOSE_ETHEREUM_IMAGE: ${{ inputs.firehose_ethereum_image || 'ghcr.io/streamingfast/firehose-ethereum:latest' }}
 
 jobs:
-  push:
-    name: Docker build & push
+  prepare:
+    name: Resolve versions
     runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      packages: read
+    outputs:
+      version: ${{ steps.extract-versions.outputs.VERSION }}
+      fireeth_image: ${{ steps.extract-versions.outputs.FIREETH_IMAGE }}
+      image_name: ${{ steps.image.outputs.name }}
+    steps:
+      # The digest below is resolved against the registry, so authenticate even
+      # though the firehose-ethereum image is public today.
+      - name: Log in to the Container registry
+        uses: docker/login-action@v4
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Lowercase image name
+        id: image
+        # Container registries require lowercase repository names; github.repository
+        # preserves the org's original casing, so normalize it here.
+        run: echo "name=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT"
+
+      - name: Extract versions
+        id: extract-versions
+        env:
+          FIREETH_REF: ${{ env.FIREHOSE_ETHEREUM_IMAGE }}
+        run: |
+          set -euo pipefail
+
+          version="edge-${GITHUB_SHA::7}"
+          if [[ "${GITHUB_REF}" == refs/tags/* ]]; then
+            version=${GITHUB_REF#refs/tags/}
+          fi
+
+          # Pin the Firehose Ethereum base by digest. The amd64 and arm64 legs
+          # build on separate runners and each resolves its own FROM, so a moving
+          # tag can otherwise put two different fireeth builds in one manifest.
+          fireeth_digest=$(docker buildx imagetools inspect "${FIREETH_REF}" --format '{{.Manifest.Digest}}')
+
+          # Strip any tag or digest to get the bare repository. Only treat a ':'
+          # in the last path segment as a tag separator, so a registry port
+          # (host:5000/repo) survives.
+          fireeth_repo="${FIREETH_REF%@*}"
+          case "${fireeth_repo##*/}" in
+            *:*) fireeth_repo="${fireeth_repo%:*}" ;;
+          esac
+
+          echo "VERSION=${version}" >> "$GITHUB_OUTPUT"
+          echo "FIREETH_IMAGE=${fireeth_repo}@${fireeth_digest}" >> "$GITHUB_OUTPUT"
+
+  build:
+    name: Build & push (${{ matrix.arch }})
+    needs: prepare
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            runner: ubuntu-24.04
+          - arch: arm64
+            runner: ubuntu-24.04-arm
+    runs-on: ${{ matrix.runner }}
     permissions:
       contents: read
       packages: write
@@ -55,79 +126,168 @@ jobs:
           submodules: recursive
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Extract versions
-        id: extract-versions
-        run: |
-          version="edge-${GITHUB_SHA::7}"
-          if [[ "${GITHUB_REF}" == refs/tags/* ]]; then
-            version=${GITHUB_REF#refs/tags/}
-          fi
-
-          docker pull "${{ env.FIREHOSE_ETHEREUM_IMAGE }}"
-          fireeth_version=$(docker inspect "${{ env.FIREHOSE_ETHEREUM_IMAGE }}" --format='{{index .Config.Labels "org.opencontainers.image.version"}}')
-
-          echo "VERSION=${version}" >> "$GITHUB_OUTPUT"
-          echo "FIREETH_VERSION=${fireeth_version}" >> "$GITHUB_OUTPUT"
-          echo "FIREETH_DOCKER_SUFFIX=${version}-fireeth-${fireeth_version}" >> "$GITHUB_OUTPUT"
-
-      - name: Generate docker tags/labels from github build context
+      - name: Generate docker labels from github build context
         id: meta-bare
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          tags: |
-            type=ref,event=tag
-            type=sha,prefix=
-            type=raw,value=${{ steps.extract-versions.outputs.VERSION }}
-            type=raw,value=${{ steps.extract-versions.outputs.VERSION }},suffix=${{ steps.extract-versions.outputs.FIREETH_DOCKER_SUFFIX }}
-            type=raw,enable=${{ github.ref == 'refs/heads/develop' }},value=develop
+          images: ${{ env.REGISTRY }}/${{ needs.prepare.outputs.image_name }}
 
-      - name: Build and push nitro Docker image
-        uses: docker/build-push-action@v6
+      - name: Build nitro Docker image
+        uses: docker/build-push-action@v7
         with:
           context: .
           target: nitro-node
           file: ./Dockerfile
           push: false
-          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-base:${{ steps.extract-versions.outputs.VERSION }}
+          tags: ${{ env.REGISTRY }}/${{ needs.prepare.outputs.image_name }}-base:${{ needs.prepare.outputs.version }}
           labels: ${{ steps.meta-bare.outputs.labels }}
 
-      - name: Build and push final (fireeth + nitro) Docker image
-        uses: docker/build-push-action@v6
+      # The per-arch tag is a debugging handle only. The manifest job assembles
+      # the release from the digests exported below, never from this tag, so a
+      # concurrent run overwriting it cannot corrupt a published manifest.
+      - name: Build and push final (fireeth + nitro) per-arch image
+        id: build
+        uses: docker/build-push-action@v7
         with:
           context: .
           file: ./Dockerfile.sf
           push: true
           build-args: |
-            NITRO_BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-base:${{ steps.extract-versions.outputs.VERSION }}
-            FIREHOSE_ETHEREUM_IMAGE=${{ env.FIREHOSE_ETHEREUM_IMAGE }}
-          tags: ${{ steps.meta-bare.outputs.tags }}
+            NITRO_BASE_IMAGE=${{ env.REGISTRY }}/${{ needs.prepare.outputs.image_name }}-base:${{ needs.prepare.outputs.version }}
+            FIREHOSE_ETHEREUM_IMAGE=${{ needs.prepare.outputs.fireeth_image }}
+          tags: ${{ env.REGISTRY }}/${{ needs.prepare.outputs.image_name }}:${{ needs.prepare.outputs.version }}-${{ matrix.arch }}
           labels: ${{ steps.meta-bare.outputs.labels }}
 
+      - name: Export image digest
+        env:
+          ARCH: ${{ matrix.arch }}
+          BUILD_DIGEST: ${{ steps.build.outputs.digest }}
+          IMAGE_TAG: ${{ env.REGISTRY }}/${{ needs.prepare.outputs.image_name }}:${{ needs.prepare.outputs.version }}-${{ matrix.arch }}
+        run: |
+          set -euo pipefail
+
+          digest="${BUILD_DIGEST}"
+          if [ -z "${digest}" ]; then
+            digest=$(docker buildx imagetools inspect "${IMAGE_TAG}" --format '{{.Manifest.Digest}}')
+          fi
+          if [ -z "${digest}" ]; then
+            echo "unable to resolve the pushed image digest for ${IMAGE_TAG}" >&2
+            exit 1
+          fi
+
+          mkdir -p /tmp/digests
+          echo "${digest}" > "/tmp/digests/${ARCH}"
+
+      - name: Upload image digest
+        uses: actions/upload-artifact@v7
+        with:
+          name: digests-${{ matrix.arch }}
+          path: /tmp/digests/${{ matrix.arch }}
+          if-no-files-found: error
+          retention-days: 1
+
+  manifest:
+    name: Create multi-arch manifests
+    needs: [prepare, build]
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Log in to the Container registry
+        uses: docker/login-action@v4
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Download image digests
+        uses: actions/download-artifact@v8
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Generate docker tags from github build context
+        id: meta-bare
+        uses: docker/metadata-action@v6
+        with:
+          images: ${{ env.REGISTRY }}/${{ needs.prepare.outputs.image_name }}
+          tags: |
+            type=ref,event=tag
+            # Only branch builds claim the bare commit-sha tag. A tag pushed on a
+            # commit that is also a release/* branch head starts two runs, and
+            # both would otherwise publish this same tag, leaving whichever
+            # finished last as the winner.
+            type=sha,prefix=,enable=${{ !startsWith(github.ref, 'refs/tags/') }}
+            type=raw,value=${{ needs.prepare.outputs.version }}
+            type=raw,enable=${{ github.ref == 'refs/heads/develop' }},value=develop
+
+      - name: Create and push manifests
+        env:
+          IMAGE_NAME: ${{ needs.prepare.outputs.image_name }}
+          TAGS: ${{ steps.meta-bare.outputs.tags }}
+        run: |
+          set -euo pipefail
+
+          # Reference the per-arch images by digest: content-addressed, so no
+          # concurrent run can substitute a different image under us.
+          sources=()
+          for digest_file in /tmp/digests/*; do
+            sources+=("${REGISTRY}/${IMAGE_NAME}@$(cat "${digest_file}")")
+          done
+          if [ "${#sources[@]}" -eq 0 ]; then
+            echo "no per-arch digests were published" >&2
+            exit 1
+          fi
+          echo "manifest sources: ${sources[*]}"
+
+          # On a tag push, type=ref and type=raw,value=VERSION both resolve to the
+          # tag name, so deduplicate before pushing.
+          while IFS= read -r tag; do
+            [ -n "$tag" ] || continue
+            docker buildx imagetools create -t "$tag" "${sources[@]}"
+          done < <(printf '%s\n' "$TAGS" | sort -u)
+
   release:
     if: startsWith(github.ref, 'refs/tags/')
-    needs: push
+    needs: [prepare, manifest]
     runs-on: ubuntu-24.04
     permissions:
       contents: write
+      packages: read
 
     steps:
-      - name: Extract image
-        id: image
-        run: |
-          # The run is done only on tags, so we can ref/tags/ replace directly here
-          echo "ID=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${GITHUB_REF#refs/tags/}" >> "$GITHUB_OUTPUT"
+      - name: Log in to the Container registry
+        uses: docker/login-action@v4
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Extract assets
+      - name: Extract binaries
+        env:
+          IMAGE_NAME: ${{ needs.prepare.outputs.image_name }}
+          VERSION: ${{ needs.prepare.outputs.version }}
         run: |
-          # The --platform are not really needed here, but it removes the warning
-          docker cp $(docker create --platform=linux/amd64 ${{ steps.image.outputs.ID }}):/usr/local/bin/nitro ./nitro_linux_amd64
+          set -euo pipefail
+
+          # Read from the multi-arch manifest this run just published and let
+          # --platform select the variant. The container is only created, never
+          # started, so pulling a foreign architecture here is fine.
+          for arch in amd64 arm64; do
+            src="${REGISTRY}/${IMAGE_NAME}:${VERSION}"
+            container=$(docker create --platform="linux/${arch}" "$src")
+            docker cp "${container}:/usr/local/bin/nitro" "./nitro_linux_${arch}"
+            docker rm "${container}" >/dev/null
+          done
 
       - name: Extract Changelog
         id: changelog
@@ -146,3 +306,4 @@ jobs:
           prerelease: ${{ contains(github.ref, 'beta') || contains(github.ref, 'rc') }}
           files: |
             ./nitro_linux_amd64
+            ./nitro_linux_arm64

CHANGELOG.sf.md

@@ -1,3 +1,10 @@
+## Unreleased
+
+* Added `linux/arm64` support: the published Docker image is now a multi-arch manifest covering `linux/amd64` and `linux/arm64`, and releases ship a `nitro_linux_arm64` binary alongside `nitro_linux_amd64`.
+* Changed the Firehose Ethereum base image to be pinned by digest for the whole build, so both architectures in a manifest are guaranteed to embed the same `firehose-ethereum` build.
+* Removed the `<version>-fireeth-<fireeth version>` image tag. Releases are identified by their version alone, e.g. `v3.11.2-fh3.0`.
+* Fixed the bare commit-sha image tag being published by both the branch run and the tag run when a tag is pushed on a `release/*` branch head, where whichever finished last silently won. Only branch builds publish it now.
+
 ## v3.11.2-fh3.0
 
 * Bumped to [3.11.2](https://github.com/OffchainLabs/nitro/releases/tag/v3.11.2).