diff --git a/.dockerignore b/.dockerignore index 3c6805bf8..69f30185f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,6 +9,7 @@ README.md docker/ !docker/entrypoint-*.sh !docker/scripts/ +!docker/genvm-source-build.nix.conf .venv/ .vscode/ .ollama/ diff --git a/.env.example b/.env.example index 13724ca3a..5ce99567f 100644 --- a/.env.example +++ b/.env.example @@ -43,6 +43,7 @@ GENVM_SOURCE_MODE="" GENVM_TAG="" # Exact release tag, e.g. v0.6.0-rc0. Mutually exclusive with GENVM_REF. GENVM_REF="" # Source git ref/SHA; set mode=source, or use : for auto source mode. GENVM_EXECUTOR_VERSION_NAME="" # Optional source-build executor label, e.g. v0.3.0-e2e-dev. +NIX_NETRC_FILE="/dev/null" # Netrc with a nix cache pull token; must exist, empty one just means no cache hits. GENVM_LLM_DEBUG="0" GENVM_WEB_DEBUG="0" diff --git a/.github/actions/genvm-runners-closure/action.yml b/.github/actions/genvm-runners-closure/action.yml index d029c0cca..31d78add6 100644 --- a/.github/actions/genvm-runners-closure/action.yml +++ b/.github/actions/genvm-runners-closure/action.yml @@ -18,15 +18,34 @@ inputs: description: Directory in the build context to write the closure into. required: false default: .genvm-nix-closure + nix_cache_pull_token: + description: >- + pull token for the GenLayer nix cache, which is private. A composite + action cannot read `secrets`, so the caller has to thread it through. + Empty means the runner closure is built from source. + required: false + default: "" + +outputs: + netrc_path: + description: >- + Netrc holding the cache pull token, written by nix-setup. The Docker + source build fetches from the same cache, so it needs the same credential + — pass it as a BuildKit secret, never as an ARG or a COPY. + value: ${{ steps.nix.outputs.netrc_path }} runs: using: composite steps: + # Substituters, trusted keys and cache auth come from the shared action; the + # sandbox settings below are this action's own and must survive it, so they + # go through extra_nix_config, whose lines are appended last and win. - name: Install Nix - uses: cachix/install-nix-action@b4b293eae0b79aac8a161bb32925a5508c9cca93 # v31 + id: nix + uses: genlayerlabs/github-actions/nix-setup@39b0a0d5e9bb27a1612d2e98b0f8509d31745157 with: + cache_pull_token: ${{ inputs.nix_cache_pull_token }} extra_nix_config: | - experimental-features = nix-command flakes sandbox = true # Without this Nix quietly builds unsandboxed when it cannot set the # sandbox up, which is the exact output this action must never export. diff --git a/.github/workflows/backend_integration_tests_pr.yml b/.github/workflows/backend_integration_tests_pr.yml index c8e72107b..c4fdc1685 100644 --- a/.github/workflows/backend_integration_tests_pr.yml +++ b/.github/workflows/backend_integration_tests_pr.yml @@ -209,16 +209,24 @@ jobs: # Source builds need the GenVM runner fixed-output derivations built under # a real Nix sandbox, which the nixos/nix build stage cannot provide. - name: Prebuild GenVM runners closure + id: genvm_closure if: env.GENVM_SOURCE_MODE == 'source' uses: ./.github/actions/genvm-runners-closure with: ref: ${{ env.GENVM_REF }} + nix_cache_pull_token: ${{ secrets.NIX_CACHE_PULL_TOKEN }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 # Build only backend images with Buildx Bake and enable GHA cache - name: Build backend images with cache (buildx bake) + # The source build inside Dockerfile.backend substitutes from the same + # private cache, so it needs the same credential. docker-compose.yml + # reads this path into the `nix_netrc` build secret; unset (a release + # build, or a fork PR with no token) falls back to /dev/null. + env: + NIX_NETRC_FILE: ${{ steps.genvm_closure.outputs.netrc_path }} uses: docker/bake-action@v6 with: # Bake defaults to the Git context, which ignores anything earlier @@ -465,16 +473,24 @@ jobs: # Source builds need the GenVM runner fixed-output derivations built under # a real Nix sandbox, which the nixos/nix build stage cannot provide. - name: Prebuild GenVM runners closure + id: genvm_closure if: env.GENVM_SOURCE_MODE == 'source' uses: ./.github/actions/genvm-runners-closure with: ref: ${{ env.GENVM_REF }} + nix_cache_pull_token: ${{ secrets.NIX_CACHE_PULL_TOKEN }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 # Build only backend images with Buildx Bake and enable GHA cache - name: Build backend images with cache (buildx bake) + # The source build inside Dockerfile.backend substitutes from the same + # private cache, so it needs the same credential. docker-compose.yml + # reads this path into the `nix_netrc` build secret; unset (a release + # build, or a fork PR with no token) falls back to /dev/null. + env: + NIX_NETRC_FILE: ${{ steps.genvm_closure.outputs.netrc_path }} uses: docker/bake-action@5be5f02ff8819ecd3092ea6b2e6261c31774f2b4 # v6 with: # Bake defaults to the Git context, which ignores anything earlier diff --git a/.github/workflows/load-test-oha.yml b/.github/workflows/load-test-oha.yml index 48bcb3e69..0d10a053c 100644 --- a/.github/workflows/load-test-oha.yml +++ b/.github/workflows/load-test-oha.yml @@ -198,15 +198,23 @@ jobs: # Source builds need the GenVM runner fixed-output derivations built under # a real Nix sandbox, which the nixos/nix build stage cannot provide. - name: Prebuild GenVM runners closure + id: genvm_closure if: env.GENVM_SOURCE_MODE == 'source' uses: ./.github/actions/genvm-runners-closure with: ref: ${{ env.GENVM_REF }} + nix_cache_pull_token: ${{ secrets.NIX_CACHE_PULL_TOKEN }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build images with cache (buildx bake) + # The source build inside Dockerfile.backend substitutes from the same + # private cache, so it needs the same credential. docker-compose.yml + # reads this path into the `nix_netrc` build secret; unset (a release + # build, or a fork PR with no token) falls back to /dev/null. + env: + NIX_NETRC_FILE: ${{ steps.genvm_closure.outputs.netrc_path }} uses: docker/bake-action@v6 with: # Bake defaults to the Git context, which ignores anything earlier diff --git a/docker-compose.yml b/docker-compose.yml index 9fa694ccd..45b56c9f2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -111,6 +111,8 @@ services: context: ./ dockerfile: ./docker/Dockerfile.backend target: prod + secrets: + - nix_netrc args: GENVM_TAG: ${GENVM_TAG:-} GENVM_REF: ${GENVM_REF:-} @@ -301,6 +303,8 @@ services: context: ./ dockerfile: ./docker/Dockerfile.backend target: ${CONSENSUS_BUILD_TARGET:-consensus-worker} + secrets: + - nix_netrc args: GENVM_TAG: ${GENVM_TAG:-} GENVM_REF: ${GENVM_REF:-} @@ -449,3 +453,11 @@ volumes: genvm_cache: postgres_data: redis_data: + +# Credential for the private nix cache, consumed only by the GenVM source build +# in Dockerfile.backend. Defaults to /dev/null — an empty netrc — so a checkout +# without a token still builds, just without cache hits. CI points it at the +# netrc `nix-setup` wrote. +secrets: + nix_netrc: + file: ${NIX_NETRC_FILE:-/dev/null} diff --git a/docker/Dockerfile.backend b/docker/Dockerfile.backend index 470d787eb..54f89da5d 100644 --- a/docker/Dockerfile.backend +++ b/docker/Dockerfile.backend @@ -35,11 +35,20 @@ SHELL ["/nix/var/nix/profiles/default/bin/bash", "-x", "-c"] # CI or scripts/prepare-genvm-source-build.sh exports genvm-manager's # `runners-all` closure here from a sandboxed Nix. COPY .genvm-nix-closure/ /genvm-nix-closure/ +COPY docker/genvm-source-build.nix.conf /genvm-source-build.nix.conf # Source builds bind Studio to a genvm-manager branch/SHA. The flake target # follows TARGETARCH (amd64-linux / arm64-linux); runners are included by # genvm-manager's combined genvm package. -RUN set -euo pipefail ; \ +# +# The cache this build substitutes from is private, so the fetch needs a +# credential. `/etc/nix/netrc` is nix's default netrc-file, so mounting there +# means nothing has to point at it. The mount is torn down when the RUN ends and +# never reaches a layer — which is why it is a secret and not an ARG or a COPY. +# `required=false`: a build without the secret gets no credential, fetches +# nothing from the cache, and builds from source instead of failing. +RUN --mount=type=secret,id=nix_netrc,target=/etc/nix/netrc,required=false \ + set -euo pipefail ; \ if [[ -n "$GENVM_TAG" && -n "$GENVM_REF" ]]; then \ echo "ERROR: GENVM_TAG and GENVM_REF are mutually exclusive" ; exit 1 ; \ fi ; \ @@ -64,10 +73,8 @@ RUN set -euo pipefail ; \ echo "ERROR: GenVM source builds require the sandboxed runners closure; run ./scripts/prepare-genvm-source-build.sh before building" ; exit 1 ; \ fi ; \ gzip -dc "$CLOSURE_FILE" | nix-store --import > /dev/null ; \ - # filter-syscalls=false: nix's seccomp filter cannot load under - # qemu/Rosetta emulation (cross-platform local builds); harmless on - # native hosts since the image already runs with sandbox disabled. - printf 'experimental-features = nix-command flakes\nfilter-syscalls = false\ncores = %s\n' "$GENVM_BUILD_CORES" >> /etc/nix/nix.conf ; \ + cat /genvm-source-build.nix.conf >> /etc/nix/nix.conf ; \ + printf 'cores = %s\n' "$GENVM_BUILD_CORES" >> /etc/nix/nix.conf ; \ git clone --recurse-submodules https://github.com/genlayerlabs/genvm-manager /src/genvm ; \ cd /src/genvm ; \ git checkout "$SOURCE_REF" ; \ diff --git a/docker/genvm-source-build.nix.conf b/docker/genvm-source-build.nix.conf new file mode 100644 index 000000000..a8613dba6 --- /dev/null +++ b/docker/genvm-source-build.nix.conf @@ -0,0 +1,13 @@ +experimental-features = nix-command flakes + +# seccomp filter cannot load under qemu/Rosetta emulation; sandbox is off anyway +filter-syscalls = false + +extra-substituters = https://nix-cache.ygr.ai/genlayer +extra-trusted-public-keys = genlayer:hMvP8BkI2v8E3BtzMB9HQyIEZz/ahNdBZo/JEQ5hpVA= + +# substituter error recovery: retry, then build locally +download-attempts = 10 +connect-timeout = 15 +stalled-download-timeout = 60 +fallback = true