From 2447b2bc9e0b64261bd3c9f337dc9480f238552b Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Sat, 10 Oct 2026 17:34:08 +0000 Subject: [PATCH 1/6] tests: cut image builds over to tailor workspace Point the Makefile, image-build pipelines, documentation, and legacy target helper at tests/tailor-images so the new tailor workspace becomes the live build path. This also renames the azl-installer pipeline staging paths to tests/tailor-images/azl-installer. That path update was not needed in the original combined branch, but it is required here because this split introduces the standalone tests/tailor-images workspace one PR earlier. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../stages/azl_installer/azl-installer.yml | 10 +- .../build_image/build-image-template.yml | 140 +++++++- .../trident_images/build-image-arm64.yml | 9 + .../stages/trident_images/build-image.yml | 9 + .../trident-testimg-template.yml | 27 +- Makefile | 91 +++-- docs/Development/Testing/E2E-Tests.md | 61 ++-- docs/Development/Testing/Functional-Tests.md | 9 +- docs/Development/Testing/Rollback-Tests.md | 6 +- docs/Development/Testing/Servicing-Tests.md | 17 +- tests/tailor-images/legacy-map.json | 263 +++++++++++++++ tests/tailor-images/legacy_targets.py | 317 ++++++++++++++++++ 12 files changed, 857 insertions(+), 102 deletions(-) create mode 100644 tests/tailor-images/legacy-map.json create mode 100644 tests/tailor-images/legacy_targets.py diff --git a/.pipelines/templates/stages/azl_installer/azl-installer.yml b/.pipelines/templates/stages/azl_installer/azl-installer.yml index cc7d807c53..e9e8d93f2d 100644 --- a/.pipelines/templates/stages/azl_installer/azl-installer.yml +++ b/.pipelines/templates/stages/azl_installer/azl-installer.yml @@ -76,15 +76,15 @@ stages: - bash: | set -ex - mkdir -p tests/images/azl-installer/iso/bin - mkdir -p tests/images/azl-installer/iso/images + mkdir -p tests/tailor-images/azl-installer/iso/bin + mkdir -p tests/tailor-images/azl-installer/iso/images # Move liveinstaller to bin directory - mv "$(Build.ArtifactStagingDirectory)/installer-tools/liveinstaller" tests/images/azl-installer/iso/bin/ - chmod +x tests/images/azl-installer/iso/bin/liveinstaller + mv "$(Build.ArtifactStagingDirectory)/installer-tools/liveinstaller" tests/tailor-images/azl-installer/iso/bin/ + chmod +x tests/tailor-images/azl-installer/iso/bin/liveinstaller # Move .cosi file to the ISO directory. When there are clones (multiple .cosi files), we pick the first one. - mv $(ls $(Build.ArtifactStagingDirectory)/testimage/*.cosi | head -n 1) tests/images/azl-installer/iso/images/trident-testimage.cosi + mv $(ls $(Build.ArtifactStagingDirectory)/testimage/*.cosi | head -n 1) tests/tailor-images/azl-installer/iso/images/trident-testimage.cosi displayName: "Prepare installer dependencies" workingDirectory: $(TRIDENT_SOURCE_DIR) diff --git a/.pipelines/templates/stages/build_image/build-image-template.yml b/.pipelines/templates/stages/build_image/build-image-template.yml index 75b3521d6b..713ffc2742 100644 --- a/.pipelines/templates/stages/build_image/build-image-template.yml +++ b/.pipelines/templates/stages/build_image/build-image-template.yml @@ -62,6 +62,10 @@ parameters: steps: - template: ../common_tasks/avoid-pypi-usage.yml + - template: ../common_tasks/cargo-auth.yml + parameters: + cargoConfigPath: $(TRIDENT_SOURCE_DIR)/.cargo/config.toml + - template: common/sfi-enforce-isolation-with-etc-hosts.yaml@platform-pipelines - script: | @@ -82,8 +86,17 @@ steps: retryCountOnTaskFailure: 3 - bash: | - ./tests/images/testimages.py show-image ${{ parameters.imageName }} base-image --devops-var baseImageType - displayName: "Get Image Info" + set -euo pipefail + python3 - <<'PY' "${{ parameters.imageName }}" + import json + import sys + from pathlib import Path + + legacy = json.loads(Path("tests/tailor-images/legacy-map.json").read_text()) + entry = legacy[sys.argv[1]] + print(f"##vso[task.setvariable variable=baseImageType]{entry['baseImage']}") + PY + displayName: "Resolve image metadata" workingDirectory: ${{ parameters.tridentSourceDirectory }} - bash: | @@ -117,10 +130,27 @@ steps: azureLinuxVersion: ${{ parameters.azureLinuxVersion }} runtimeBuildType: ${{ parameters.baseimgBuildType }} + - ${{ if eq(parameters.architecture, 'arm64') }}: + - task: DownloadPipelineArtifact@2 + displayName: "Download go-tools for direct-streaming installer" + condition: or(eq('${{ parameters.imageName }}', 'trident-direct-streaming-installer-amd64'), eq('${{ parameters.imageName }}', 'trident-direct-streaming-installer-arm64')) + inputs: + buildType: current + artifactName: go-tools-arm64 + targetPath: "$(Build.ArtifactStagingDirectory)/go-tools" + - ${{ if ne(parameters.architecture, 'arm64') }}: + - task: DownloadPipelineArtifact@2 + displayName: "Download go-tools for direct-streaming installer" + condition: or(eq('${{ parameters.imageName }}', 'trident-direct-streaming-installer-amd64'), eq('${{ parameters.imageName }}', 'trident-direct-streaming-installer-arm64')) + inputs: + buildType: current + artifactName: go-tools + targetPath: "$(Build.ArtifactStagingDirectory)/go-tools" + - bash: | set -ex - # Move base VHDX to artifacts/ (builder expects artifacts/*.vhdx) + # Move base VHDX to artifacts/ (tailor baseImages catalogue points at artifacts/*.vhdx) mkdir -p artifacts if ls "$(Build.ArtifactStagingDirectory)/images" | grep -q ".*\.vhdx$"; then mv $(Build.ArtifactStagingDirectory)/images/*.vhdx artifacts/ @@ -144,28 +174,114 @@ steps: DISTRO=azl4 fi - # Move Trident RPMs to bin/RPMS/ (builder expects bin/RPMS/*.rpm) + # Move Trident RPMs to bin/RPMS/ (tailor rpmSources point at bin/RPMS/) if [ -d "$(Build.ArtifactStagingDirectory)/trident" ]; then mkdir -p bin/RPMS find "$(Build.ArtifactStagingDirectory)/trident" -name "*${DISTRO}*.rpm" -exec mv {} bin/RPMS/ \; rm -rf "$(Build.ArtifactStagingDirectory)/trident" fi + + if [[ "${{ parameters.imageName }}" == trident-direct-streaming-installer-* ]]; then + if [ -f "$(Build.ArtifactStagingDirectory)/go-tools/rcp-agent" ]; then + cp "$(Build.ArtifactStagingDirectory)/go-tools/rcp-agent" artifacts/rcp-agent + elif [ -f bin/rcp-agent ]; then + cp bin/rcp-agent artifacts/rcp-agent + else + echo "Missing rcp-agent for direct-streaming installer" + exit 1 + fi + chmod +x artifacts/rcp-agent + cp tools/cmd/rcp-agent/rcp-agent.service artifacts/rcp-agent.service + fi displayName: "Prepare and move requirements" workingDirectory: ${{ parameters.tridentSourceDirectory }} - bash: | - set -ex + set -euo pipefail + + legacy_name='${{ parameters.imageName }}' + export PATH="$HOME/.cargo/bin:$PATH" + export CLICOLOR_FORCE=1 - EXTRA_ARGS="" + readarray -t selector_lines < <(python3 - <<'PY' "$legacy_name" + import json + import sys + from pathlib import Path + entry = json.loads(Path("tests/tailor-images/legacy-map.json").read_text())[sys.argv[1]] + print(entry["image"]) + print(entry["ext"]) + print(entry.get("tailorExt", entry["ext"])) + for key, value in entry["selectors"].items(): + print(f"{key}={value}") + PY + ) + + tailor_image="${selector_lines[0]}" + artifact_ext="${selector_lines[1]}" + source_ext="${selector_lines[2]}" + selector_args=() + if (( ${#selector_lines[@]} > 3 )); then + for selector in "${selector_lines[@]:3}"; do + selector_args+=("-s" "$selector") + done + fi + + manifest_path=tests/tailor-images/tailor.yaml if [[ "${{ parameters.micBuildType }}" == "dev" ]]; then - EXTRA_ARGS+=" --container imagecustomizer:dev" + manifest_path=tests/tailor-images/.tailor.pipeline.yaml + python3 - <<'PY' + from pathlib import Path + text = Path("tests/tailor-images/tailor.yaml").read_text() + needle = """toolchains: + default: ic + entries: + - name: ic + container: mcr.microsoft.com/azurelinux/imagecustomizer + tag: latest + """ + replacement = """toolchains: + default: ic-dev + entries: + - name: ic + container: mcr.microsoft.com/azurelinux/imagecustomizer + tag: latest + - name: ic-dev + container: imagecustomizer + tag: dev + pull: never + """ + Path("tests/tailor-images/.tailor.pipeline.yaml").write_text(text.replace(needle, replacement, 1)) + PY + fi + + matrix_json=$(cargo run --manifest-path tools/tailor/Cargo.toml --quiet -- --manifest "$manifest_path" matrix "$tailor_image" --format json "${selector_args[@]}") + + slug=$(python3 - <<'PY' "$matrix_json" + import json + import sys + cells = json.loads(sys.argv[1]) + if len(cells) != 1: + raise SystemExit(f"expected exactly one cell, got {len(cells)}") + print(cells[0]["slug"]) + PY + ) + + rm -f tests/tailor-images/artifacts/ca_cert.pem + cargo run --manifest-path tools/tailor/Cargo.toml --quiet -- --manifest "$manifest_path" build "$tailor_image" "${selector_args[@]}" --output-dir $(ob_outputDirectory) --clones ${{ parameters.clones }} + + if [[ ${{ parameters.clones }} -eq 1 ]]; then + mv "$(ob_outputDirectory)/${slug}.${source_ext}" "$(ob_outputDirectory)/${legacy_name}.${artifact_ext}" + else + for ((i=0; i<${{ parameters.clones }}; i++)); do + mv "$(ob_outputDirectory)/${slug}_clone${i}.${source_ext}" "$(ob_outputDirectory)/${legacy_name}_${i}.${artifact_ext}" + done + fi + + if [ -f tests/tailor-images/artifacts/ca_cert.pem ]; then + cp tests/tailor-images/artifacts/ca_cert.pem "$(ob_outputDirectory)/ca_cert.pem" fi - python3 ./tests/images/testimages.py build \ - "${{ parameters.imageName }}" $EXTRA_ARGS \ - --output-dir $(ob_outputDirectory) \ - --no-download \ - --clones ${{ parameters.clones }} + rm -f tests/tailor-images/.tailor.pipeline.yaml displayName: "Build ${{ parameters.imageName }}" workingDirectory: ${{ parameters.tridentSourceDirectory }} diff --git a/.pipelines/templates/stages/trident_images/build-image-arm64.yml b/.pipelines/templates/stages/trident_images/build-image-arm64.yml index 04be97e2f9..abc4ab9d3e 100644 --- a/.pipelines/templates/stages/trident_images/build-image-arm64.yml +++ b/.pipelines/templates/stages/trident_images/build-image-arm64.yml @@ -52,6 +52,15 @@ jobs: - template: ../common_tasks/checkout_trident.yml - template: ../common_tasks/avoid-pypi-usage.yml + # The Makefile's tailor-backed legacy image targets invoke + # `cargo run --manifest-path tools/tailor/Cargo.toml`, so a Rust + # toolchain and cargo registry auth are required here even though this + # job previously never needed cargo. + - template: ../common_tasks/rustup.yml + - template: ../common_tasks/cargo-auth.yml + parameters: + cargoConfigPath: $(TRIDENT_SOURCE_DIR)/.cargo/config.toml + - task: DownloadPipelineArtifact@2 inputs: buildType: current diff --git a/.pipelines/templates/stages/trident_images/build-image.yml b/.pipelines/templates/stages/trident_images/build-image.yml index 6f70dafdce..c50648e7c4 100644 --- a/.pipelines/templates/stages/trident_images/build-image.yml +++ b/.pipelines/templates/stages/trident_images/build-image.yml @@ -101,6 +101,15 @@ jobs: - template: ../common_tasks/checkout_trident.yml - template: ../common_tasks/avoid-pypi-usage.yml + # The Makefile's tailor-backed legacy image targets invoke + # `cargo run --manifest-path tools/tailor/Cargo.toml`, so a Rust + # toolchain and cargo registry auth are required here even though this + # job previously never needed cargo. + - template: ../common_tasks/rustup.yml + - template: ../common_tasks/cargo-auth.yml + parameters: + cargoConfigPath: $(TRIDENT_SOURCE_DIR)/.cargo/config.toml + - ${{ if parameters.downloadGoTools }}: - task: DownloadPipelineArtifact@2 displayName: "Download $(goToolsArtifactName)" diff --git a/.pipelines/templates/stages/trident_images/trident-testimg-template.yml b/.pipelines/templates/stages/trident_images/trident-testimg-template.yml index c2d53a4694..9b7cce61a1 100644 --- a/.pipelines/templates/stages/trident_images/trident-testimg-template.yml +++ b/.pipelines/templates/stages/trident_images/trident-testimg-template.yml @@ -135,7 +135,7 @@ steps: displayName: Copy SSH Keys - script: | - echo "##[warning]THE PIPELINE TEMPLATE trident-testimg-template.yaml IS DEPRECATED. PLEASE SWITCH TO USING testimages.py TO BUILD TEST IMAGES." + echo "##[warning]THE PIPELINE TEMPLATE trident-testimg-template.yaml IS DEPRECATED. PLEASE SWITCH TO USING the tailor workspace under tests/tailor-images." cat /etc/os-release displayName: "Report Host Info" @@ -264,6 +264,18 @@ steps: rm -rf "$(Build.ArtifactStagingDirectory)/trident" fi + if [[ "${{ parameters.target }}" == artifacts/trident-direct-streaming-installer-* ]]; then + mkdir -p '${{ parameters.tridentSourceDirectory }}/artifacts' + if [ -f "${{ parameters.tridentSourceDirectory }}/bin/rcp-agent" ]; then + cp "${{ parameters.tridentSourceDirectory }}/bin/rcp-agent" '${{ parameters.tridentSourceDirectory }}/artifacts/rcp-agent' + chmod +x '${{ parameters.tridentSourceDirectory }}/artifacts/rcp-agent' + else + echo "Missing rcp-agent in bin/ for direct-streaming installer" + exit 1 + fi + cp '${{ parameters.tridentSourceDirectory }}/tools/cmd/rcp-agent/rcp-agent.service' '${{ parameters.tridentSourceDirectory }}/artifacts/rcp-agent.service' + fi + workingDirectory: ${{ parameters.tridentSourceDirectory }} displayName: "Prepare and move requirements" @@ -280,13 +292,12 @@ steps: export MIC_CONTAINER_IMAGE="imagecustomizer:dev" fi - # Allow cross-platform (i.e. amd64 pipeline creating arm64 images) - if [[ "${{ parameters.micArchitecture }}" != "${{ parameters.pipelineArchitecture }}" ]]; then - # Export variable to tell Makefile and testimages to use `docker --platform` - export MIC_ARCHITECTURE="linux/${{ parameters.micArchitecture }}" - fi + # The Makefile's tailor-backed legacy target helper resolves the cell arch + # from the historical image name, so no extra MIC_ARCHITECTURE export is + # needed here for cross-arch direct-streaming builds. # Compress the full image & delete the uncompressed image + rm -f tests/tailor-images/artifacts/ca_cert.pem make ${{ parameters.target }} workingDirectory: ${{ parameters.tridentSourceDirectory }} displayName: "Build Test Image" @@ -302,5 +313,9 @@ steps: # Everything else is a file sudo mv -v "${{ parameters.target }}" "${{ parameters.outputDirectory }}/" fi + + if [ -f "tests/tailor-images/artifacts/ca_cert.pem" ]; then + sudo cp -v "tests/tailor-images/artifacts/ca_cert.pem" "${{ parameters.outputDirectory }}/ca_cert.pem" + fi workingDirectory: ${{ parameters.tridentSourceDirectory }} displayName: "Copy artifacts to output directory" diff --git a/Makefile b/Makefile index 28ec9dc5d7..0bc1e43a10 100644 --- a/Makefile +++ b/Makefile @@ -538,7 +538,7 @@ generate-functional-test-manifest: .cargo/config .PHONY: validate-configs validate-configs: target/release/trident - $(eval DETECTED_HC_FILES := $(shell grep -R '^storage:' . --include '*.yaml' -l | grep -E -v '\./(target|dev|azure-linux-image-tools|crates/docbuilder|tests/images|tests/azl-installer)')) + $(eval DETECTED_HC_FILES := $(shell grep -R '^storage:' . --include '*.yaml' -l | grep -E -v '\./(target|dev|azure-linux-image-tools|crates/docbuilder|tests/tailor-images|tests/azl-installer)')) @for file in $(DETECTED_HC_FILES); do \ echo "Validating $$file"; \ $< validate $$file -v info || exit 1; \ @@ -627,7 +627,7 @@ run-attendedinstaller-simulator: bin/attendedinstaller-simulator # Test image paths ARTIFACTS_TEST_IMAGE_DIR := artifacts/test-image -AZL_INSTALLER_DIR := tests/images/azl-installer +AZL_INSTALLER_DIR := tests/tailor-images/azl-installer AZL_INSTALLER_ISO_DIR := $(AZL_INSTALLER_DIR)/iso # If regular.cosi is not present, download runtime images @@ -651,7 +651,7 @@ $(ARTIFACTS_TEST_IMAGE_DIR)/azl-installer.iso: \ mkdir -p $(AZL_INSTALLER_ISO_DIR)/bin cp bin/liveinstaller $(AZL_INSTALLER_ISO_DIR)/bin/ # Build ISO - ./tests/images/testimages.py build azl-installer --output-dir $(ARTIFACTS_TEST_IMAGE_DIR) + python3 ./tests/tailor-images/legacy_targets.py build azl-installer --output-path $(ARTIFACTS_TEST_IMAGE_DIR)/azl-installer.iso $(if $(strip $(MIC_CONTAINER_IMAGE)),--container $(MIC_CONTAINER_IMAGE)) .PHONY: validate validate: $(TRIDENT_CONFIG) target/release/trident @@ -885,9 +885,15 @@ download-trident-container-installer-iso: --path artifacts/ \ --artifact-name 'trident-container-installer' -artifacts/trident-container-installer.iso: - $(MAKE) download-trident-container-installer-iso; \ - ls -l artifacts/trident-container-installer.iso +artifacts/trident-container-installer.iso: $(shell python3 ./tests/tailor-images/legacy_targets.py dependencies trident-container-installer) + @echo "Building 'trident-container-installer' [$@] from $<" + @echo "Prerequisites:" + @echo "$^" | tr ' ' '\n' | sed 's/^/ /' + @echo "Building image..." + python3 ./tests/tailor-images/legacy_targets.py build \ + trident-container-installer \ + --output-path $@ \ + $(if $(strip $(MIC_CONTAINER_IMAGE)),--container $(MIC_CONTAINER_IMAGE)) # Copies locally built runtime images from ../test-images/build to ./artifacts/test-image. # Expects that both the regular and verity Trident test images have been built. @@ -932,9 +938,9 @@ starter-configuration: bin/trident-mos.iso: \ artifacts/baremetal.vhdx \ packaging/systemd/trident-install.service \ - tests/images/trident-mos/iso.yaml \ - tests/images/trident-mos/files/* \ - tests/images/trident-mos/post-install.sh \ + tests/tailor-images/trident-mos/iso.yaml \ + tests/tailor-images/trident-mos/files/* \ + tests/tailor-images/trident-mos/post-install.sh \ packaging/selinux-policy-trident/* \ tools/cmd/rcp-agent/rcp-agent.service \ bin/rcp-agent @@ -949,7 +955,7 @@ bin/trident-mos.iso: \ --build-dir /build \ --image-file /repo/$< \ --output-image-file /repo/$@ \ - --config-file /repo/tests/images/trident-mos/iso.yaml \ + --config-file /repo/tests/tailor-images/trident-mos/iso.yaml \ --output-image-format iso .PHONY: recreate-verity-image @@ -1015,7 +1021,7 @@ validate-pipeline-website-artifact: # # Generic COSI image build target pattern # -COSI_TARGETS = $(shell ./tests/images/testimages.py list --filter-type cosi) +COSI_TARGETS = $(shell python3 ./tests/tailor-images/legacy_targets.py list --filter-type cosi) .PHONY: $(COSI_TARGETS) $(COSI_TARGETS): %: artifacts/%.cosi @@ -1026,7 +1032,7 @@ all-cosi: $(COSI_TARGETS) # # Generic ISO image build target pattern # -ISO_TARGETS = $(shell ./tests/images/testimages.py list --filter-type iso) +ISO_TARGETS = $(shell python3 ./tests/tailor-images/legacy_targets.py list --filter-type iso) .PHONY: $(ISO_TARGETS) $(ISO_TARGETS): %: artifacts/%.iso @@ -1038,31 +1044,30 @@ all-iso: $(ISO_TARGETS) # prerequisites so that we can use find to get all the files in the directory. # https://www.gnu.org/software/make/manual/make.html#Secondary-Expansion .SECONDEXPANSION: -artifacts/%.cosi artifacts/%.iso artifacts/%.vhdx: $$(shell ./tests/images/testimages.py dependencies $$*) +artifacts/%.cosi artifacts/%.iso artifacts/%.vhdx: $$(shell python3 ./tests/tailor-images/legacy_targets.py dependencies $$*) @echo "Building '$*' [$@] from $<" @echo "Prerequisites:" @echo "$^" | tr ' ' '\n' | sed 's/^/ /' @echo "Building image..." - sudo ./tests/images/testimages.py build \ + python3 ./tests/tailor-images/legacy_targets.py build \ $* \ - --output-dir ./artifacts \ - $(if $(strip $(MIC_CONTAINER_IMAGE)),--container $(MIC_CONTAINER_IMAGE)) \ - $(if $(strip $(MIC_ARCHITECTURE)),--image-architecture $(MIC_ARCHITECTURE)) + --output-path $@ \ + $(if $(strip $(MIC_CONTAINER_IMAGE)),--container $(MIC_CONTAINER_IMAGE)) -MIC_CONTAINER_IMAGE ?= $(shell ./tests/images/testimages.py show-artifact customizer-container-full) -artifacts/trident-functest.qcow2: $$(shell ./tests/images/testimages.py dependencies $$(basename $$(notdir $$@))) +MIC_CONTAINER_IMAGE ?= $(shell python3 ./tests/tailor-images/legacy_targets.py show-artifact customizer-container-full) +artifacts/trident-functest.qcow2: $$(shell python3 ./tests/tailor-images/legacy_targets.py dependencies $$(basename $$(notdir $$@))) @echo "Building '$*' [$@] from $<" @echo "Prerequisites:" @echo "$^" | tr ' ' '\n' | sed 's/^/ /' @echo "Building image..." - sudo ./tests/images/testimages.py build \ + python3 ./tests/tailor-images/legacy_targets.py build \ $(basename $(notdir $@)) \ - --output-dir ./artifacts \ + --output-path $@ \ $(if $(strip $(MIC_CONTAINER_IMAGE)),--container $(MIC_CONTAINER_IMAGE)) # TRIDENT VM UPDATE IMAGES -VM_IMAGE_PATH_PREFIX = tests/images/trident-vm-testimage/base +VM_IMAGE_PATH_PREFIX = tests/tailor-images/trident-vm-testimage/base artifacts/rpm-overrides: @mkdir -p artifacts/rpm-overrides @@ -1097,23 +1102,38 @@ $(QEMU_GUEST_IMAGE): BAREMETAL_IMAGE = artifacts/baremetal.vhdx $(BAREMETAL_IMAGE): @mkdir -p artifacts - @tests/images/testimages.py download-image baremetal + @python3 ./tests/tailor-images/legacy_targets.py download-image baremetal CORE_SELINUX_IMAGE = artifacts/core_selinux.vhdx $(CORE_SELINUX_IMAGE): @mkdir -p artifacts - @tests/images/testimages.py download-image core_selinux + @python3 ./tests/tailor-images/legacy_targets.py download-image core_selinux +CORE_ARM64_IMAGE_NAME ?= core_vhdx-arm64-3.0-stable +CORE_ARM64_IMAGE_VERSION ?= * +CORE_ARM64_IMAGE = artifacts/core_arm64.vhdx +$(CORE_ARM64_IMAGE): + @mkdir -p artifacts + @tempdir=$$(mktemp -d); result=$$(az artifacts universal download --organization "https://dev.azure.com/mariner-org/" --project "36d030d6-1d99-4ebd-878b-09af1f4f722f" --scope project --feed "AzureLinuxArtifacts" --name '$(CORE_ARM64_IMAGE_NAME)' --version '$(CORE_ARM64_IMAGE_VERSION)' --path $$tempdir) && mv $$tempdir/*.vhdx $(CORE_ARM64_IMAGE) && rm -rf $$tempdir && echo $$result | jq > $(CORE_ARM64_IMAGE).metadata.json + +artifacts/rcp-agent: bin/rcp-agent + @mkdir -p artifacts + @cp bin/rcp-agent $@ + @chmod +x $@ + +artifacts/rcp-agent.service: tools/cmd/rcp-agent/rcp-agent.service + @mkdir -p artifacts + @cp tools/cmd/rcp-agent/rcp-agent.service $@ MINIMAL_IMAGE = artifacts/minimal.vhdx $(MINIMAL_IMAGE): @mkdir -p artifacts - @tests/images/testimages.py download-image minimal + @python3 ./tests/tailor-images/legacy_targets.py download-image minimal MINIMAL_IMAGE_AARCH64 = artifacts/minimal_aarch64.vhdx $(MINIMAL_IMAGE_AARCH64): @mkdir -p artifacts - @tests/images/testimages.py download-image minimal_aarch64 + @python3 ./tests/tailor-images/legacy_targets.py download-image minimal_aarch64 artifacts/trident-vm-grub-testimage.qcow2: \ $(QEMU_GUEST_IMAGE) \ @@ -1137,7 +1157,7 @@ artifacts/trident-vm-grub-testimage.qcow2: \ --config-file /repo/$(VM_IMAGE_PATH_PREFIX)/baseimg-grub.yaml artifacts/trident-vm-grub-testimage-arm64.qcow2: \ - base/core_arm64.vhdx \ + $(CORE_ARM64_IMAGE) \ $(TRIDENT_VM_DEPENDENCIES) \ $(VM_IMAGE_PATH_PREFIX)/baseimg-grub.yaml \ $(VM_IMAGE_PATH_PREFIX)/files/id_rsa.pub @@ -1200,7 +1220,7 @@ artifacts/trident-vm-root-verity-testimage.qcow2: \ --config-file /repo/$(VM_IMAGE_PATH_PREFIX)/baseimg-root-verity.yaml artifacts/trident-vm-verity-testimage-arm64.qcow2: \ - base/core_arm64.vhdx \ + $(CORE_ARM64_IMAGE) \ $(TRIDENT_VM_DEPENDENCIES) \ $(VM_IMAGE_PATH_PREFIX)/baseimg-verity.yaml \ $(VM_IMAGE_PATH_PREFIX)/files/etc-mount.service \ @@ -1315,3 +1335,18 @@ artifacts/ubuntu_2404_arm64.vhdx: curl -LO https://cloud-images.ubuntu.com/releases/server/24.04/release/ubuntu-24.04-server-cloudimg-arm64.img qemu-img convert -O vhdx ubuntu-24.04-server-cloudimg-arm64.img artifacts/ubuntu_2404_arm64.vhdx rm -f ubuntu-24.04-server-cloudimg-arm64.img + +# GB200 base image is staged from a private Azure Linux storage account (see +# .pipelines/templates/stages/trident_images/trident-testimg-template.yml, +# "Download Base Image - gb200_2404_arm64"), not a public URL, so this rule +# mirrors the pipeline's az CLI download for local/dev builds that have +# access to the azlinuxbmpstaging storage account. +GB200_BLOB_NAME = gb200/arm64/20260318/image.vhdx +artifacts/gb200_2404_arm64.vhdx: + mkdir -p artifacts + az storage blob download \ + --auth-mode login \ + --account-name azlinuxbmpstaging \ + --container-name os-image-cache \ + --name "$(GB200_BLOB_NAME)" \ + --file artifacts/gb200_2404_arm64.vhdx diff --git a/docs/Development/Testing/E2E-Tests.md b/docs/Development/Testing/E2E-Tests.md index 13169b5e42..509840db27 100644 --- a/docs/Development/Testing/E2E-Tests.md +++ b/docs/Development/Testing/E2E-Tests.md @@ -132,8 +132,8 @@ make bin/rcp-agent # Remote control plane agent ### 4. Build Trident RPMs The **host** test images include Trident packages built from your local tree. -This step builds the RPMs into `bin/RPMS/`, which `testimages.py` passes to -Image Customizer via `--rpm-source`: +This step builds the RPMs into `bin/RPMS/`, which the `tests/tailor-images` tailor +workspace passes to Image Customizer via `rpmSources:`: ```bash make bin/trident-rpms.tar.gz @@ -168,8 +168,7 @@ make artifacts/id_rsa ### 7. Download Base Image ```bash -# Downloads baremetal.vhdx from MCR -./tests/images/testimages.py download-image baremetal +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/tailor-images/tailor.yaml bases download baremetal ``` ### 8. Build COSI Images @@ -184,29 +183,27 @@ then rename them into `artifacts/test-image/`. ```bash mkdir -p artifacts/test-image -# Build two clones (produces trident-testimage_0.cosi and trident-testimage_1.cosi) -sudo ./tests/images/testimages.py build trident-testimage \ - --output-dir ./artifacts/test-image --clones 2 +# Build two clones (produces trident-testimage_amd64_cosi_clone0.cosi and +# trident-testimage_amd64_cosi_clone1.cosi) +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/tailor-images/tailor.yaml build trident-testimage -s arch=amd64 --output-dir ./artifacts/test-image --clones 2 # Rename clones to the filenames referenced by Host Configurations -mv artifacts/test-image/trident-testimage_0.cosi artifacts/test-image/regular.cosi -mv artifacts/test-image/trident-testimage_1.cosi artifacts/test-image/regular_v2.cosi +mv artifacts/test-image/trident-testimage_amd64_cosi_clone0.cosi artifacts/test-image/regular.cosi +mv artifacts/test-image/trident-testimage_amd64_cosi_clone1.cosi artifacts/test-image/regular_v2.cosi ``` Repeat for other image types as needed: ```bash # Verity image (for root-verity configuration) -sudo ./tests/images/testimages.py build trident-verity-testimage \ - --output-dir ./artifacts/test-image --clones 2 -mv artifacts/test-image/trident-verity-testimage_0.cosi artifacts/test-image/verity.cosi -mv artifacts/test-image/trident-verity-testimage_1.cosi artifacts/test-image/verity_v2.cosi +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/tailor-images/tailor.yaml build trident-verity-testimage -s deployment=host -s mode=root --output-dir ./artifacts/test-image --clones 2 +mv artifacts/test-image/trident-verity-testimage_host_root_amd64_cosi_clone0.cosi artifacts/test-image/verity.cosi +mv artifacts/test-image/trident-verity-testimage_host_root_amd64_cosi_clone1.cosi artifacts/test-image/verity_v2.cosi # UKI/usr-verity image (for usr-verity, combined configurations) -sudo ./tests/images/testimages.py build trident-usrverity-testimage \ - --output-dir ./artifacts/test-image --clones 2 -mv artifacts/test-image/trident-usrverity-testimage_0.cosi artifacts/test-image/usrverity.cosi -mv artifacts/test-image/trident-usrverity-testimage_1.cosi artifacts/test-image/usrverity_v2.cosi +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/tailor-images/tailor.yaml build trident-verity-testimage -s deployment=host -s mode=usr --output-dir ./artifacts/test-image --clones 2 +mv artifacts/test-image/trident-verity-testimage_host_usr_amd64_cosi_clone0.cosi artifacts/test-image/usrverity.cosi +mv artifacts/test-image/trident-verity-testimage_host_usr_amd64_cosi_clone1.cosi artifacts/test-image/usrverity_v2.cosi ``` #### Container Runtime Images @@ -220,26 +217,23 @@ URLs work unchanged: mkdir -p artifacts/test-image # Regular container image -sudo ./tests/images/testimages.py build trident-container-testimage \ - --output-dir ./artifacts/test-image --clones 2 -mv artifacts/test-image/trident-container-testimage_0.cosi artifacts/test-image/regular.cosi -mv artifacts/test-image/trident-container-testimage_1.cosi artifacts/test-image/regular_v2.cosi +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/tailor-images/tailor.yaml build trident-container-testimage --output-dir ./artifacts/test-image --clones 2 +mv artifacts/test-image/trident-container-testimage_amd64_cosi_clone0.cosi artifacts/test-image/regular.cosi +mv artifacts/test-image/trident-container-testimage_amd64_cosi_clone1.cosi artifacts/test-image/regular_v2.cosi # Verity container image (for root-verity configuration) -sudo ./tests/images/testimages.py build trident-container-verity-testimage \ - --output-dir ./artifacts/test-image --clones 2 -mv artifacts/test-image/trident-container-verity-testimage_0.cosi artifacts/test-image/verity.cosi -mv artifacts/test-image/trident-container-verity-testimage_1.cosi artifacts/test-image/verity_v2.cosi +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/tailor-images/tailor.yaml build trident-verity-testimage -s deployment=container -s mode=root --output-dir ./artifacts/test-image --clones 2 +mv artifacts/test-image/trident-verity-testimage_container_root_amd64_cosi_clone0.cosi artifacts/test-image/verity.cosi +mv artifacts/test-image/trident-verity-testimage_container_root_amd64_cosi_clone1.cosi artifacts/test-image/verity_v2.cosi # UKI/usr-verity container image (for usr-verity, combined configurations) -sudo ./tests/images/testimages.py build trident-container-usrverity-testimage \ - --output-dir ./artifacts/test-image --clones 2 -mv artifacts/test-image/trident-container-usrverity-testimage_0.cosi artifacts/test-image/usrverity.cosi -mv artifacts/test-image/trident-container-usrverity-testimage_1.cosi artifacts/test-image/usrverity_v2.cosi +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/tailor-images/tailor.yaml build trident-verity-testimage -s deployment=container -s mode=usr --output-dir ./artifacts/test-image --clones 2 +mv artifacts/test-image/trident-verity-testimage_container_usr_amd64_cosi_clone0.cosi artifacts/test-image/usrverity.cosi +mv artifacts/test-image/trident-verity-testimage_container_usr_amd64_cosi_clone1.cosi artifacts/test-image/usrverity_v2.cosi ``` -The images use the Image Customizer container from -`mcr.microsoft.com/azurelinux/imagecustomizer:latest`. +The workspace defaults to the Image Customizer container declared in +`tests/tailor-images/tailor.yaml` (`mcr.microsoft.com/azurelinux/imagecustomizer:latest`). ### 9. Build the Installer ISO @@ -255,8 +249,9 @@ make bin/trident-mos.iso **Container runtime:** ```bash -sudo ./tests/images/testimages.py build trident-container-installer \ - --output-dir ./artifacts +cargo run --manifest-path tools/tailor/Cargo.toml -- \ + --manifest tests/tailor-images/tailor.yaml \ + build trident-container-installer --output-dir ./artifacts ``` This builds `artifacts/trident-container-installer.iso` using Image Customizer. diff --git a/docs/Development/Testing/Functional-Tests.md b/docs/Development/Testing/Functional-Tests.md index 2a4d79b99f..a59aecc3e1 100644 --- a/docs/Development/Testing/Functional-Tests.md +++ b/docs/Development/Testing/Functional-Tests.md @@ -47,18 +47,17 @@ and a base image also from MCR. 1. **Download the base image:** ```bash - # Downloads baremetal.vhdx from mcr.microsoft.com/azurelinux/3.0/image/baremetal:latest - ./tests/images/testimages.py download-image baremetal + cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/tailor-images/tailor.yaml bases download baremetal ``` 2. **Build the functional test image:** ```bash - sudo ./tests/images/testimages.py build trident-functest --output-dir ./artifacts + cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/tailor-images/tailor.yaml build trident-functest --output-dir ./artifacts ``` - This produces `artifacts/trident-functest.qcow2`. The image configuration is - defined in `tests/images/trident-functest/base/baseimg.yaml`. + This produces `artifacts/trident-functest.qcow2`. The image is selected from + the `trident-functest` family in `tests/tailor-images/tailor.yaml`. ## Building Test Dependencies diff --git a/docs/Development/Testing/Rollback-Tests.md b/docs/Development/Testing/Rollback-Tests.md index 385fd77ee2..cf918beaea 100644 --- a/docs/Development/Testing/Rollback-Tests.md +++ b/docs/Development/Testing/Rollback-Tests.md @@ -7,7 +7,7 @@ sidebar_position: 8 Rollback tests validate manual rollback and runtime update workflows using `storm-trident run rollback`. Like [servicing tests](Servicing-Tests.md), they start from a pre-built VM image defined in -`tests/images/trident-vm-testimage/`, but focus specifically on the rollback +`tests/tailor-images/trident-vm-testimage/`, but focus specifically on the rollback chain: A/B updates, runtime updates (sysexts and netplan), and rolling each back in sequence. @@ -21,7 +21,7 @@ extensions: | `trident-vm-usr-verity-testimage` | systemd-boot | `/usr` verity | Yes | Sysexts supported | Required | | `trident-vm-grub-verity-testimage` | grub2 | Root verity | No | Not supported | Not needed | -All image configs live under `tests/images/trident-vm-testimage/base/`. The +All image configs live under `tests/tailor-images/trident-vm-testimage/base/`. The base image is `qemu_guest` (see [Servicing Tests — Download the qemu\_guest Base Image](Servicing-Tests.md#4-download-the-qemu_guest-base-image) for how to download it from the publicly available `minimal-os` image on MCR). @@ -128,7 +128,7 @@ TEST_IMAGE_NAME="trident-vm-usr-verity-testimage" sudo rm -f artifacts/trident-vm-*-testimage.qcow2 artifacts/trident-vm-*-testimage.cosi # Build the COSI and QCOW2 -sudo ./tests/images/testimages.py build $TEST_IMAGE_NAME --output-dir ./artifacts +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/tailor-images/tailor.yaml build trident-vm-testimage -s arch=amd64 -s scenario=usr-verity --output-dir ./artifacts make artifacts/$TEST_IMAGE_NAME.qcow2 ``` diff --git a/docs/Development/Testing/Servicing-Tests.md b/docs/Development/Testing/Servicing-Tests.md index af1eb33b92..fe7ce9e4f0 100644 --- a/docs/Development/Testing/Servicing-Tests.md +++ b/docs/Development/Testing/Servicing-Tests.md @@ -10,7 +10,7 @@ Servicing tests validate multi-update workflows on pre-built VM images using from a QCOW2 image that already has Trident and an OS installed, then run repeated A/B updates with optional rollback testing. -The VM images are defined in `tests/images/trident-vm-testimage/` and built +The VM images are defined in `tests/tailor-images/trident-vm-testimage/` and built with Image Customizer from a `qemu_guest` base image downloaded from MCR. ## VM Image Types @@ -24,7 +24,7 @@ image types are: | `trident-vm-grub-verity-testimage` | grub2 | Root verity | No | `updateimg-grub-verity.yaml` | | `trident-vm-usr-verity-testimage` | systemd-boot | `/usr` verity | Yes | `baseimg-usr-verity.yaml` | -All image configs live under `tests/images/trident-vm-testimage/base/`. The +All image configs live under `tests/tailor-images/trident-vm-testimage/base/`. The base image is `qemu_guest` (see [step 4](#4-download-the-qemu_guest-base-image) for how to obtain it). @@ -105,7 +105,7 @@ step, so the base image's pre-installed package set does not matter. :::tip Internal shortcut If you have access to the internal Azure DevOps artifacts feed, the Makefile downloads `qemu_guest` automatically (requires `az login`). You can also -download it directly: `./tests/images/testimages.py download-image qemu_guest` +download it directly: `cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/tailor-images/tailor.yaml bases download qemu_guest` ::: ### 5. Build the VM Image @@ -131,15 +131,12 @@ the same filename in each directory (the update loop picks the filename from ```bash mkdir -p artifacts/update-a artifacts/update-b -# Build two clones of the COSI image (produces _0 and _1 suffixed files) -sudo ./tests/images/testimages.py build trident-vm-grub-verity-testimage \ - --output-dir ./artifacts --clones 2 +# Build two clones of the COSI image +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/tailor-images/tailor.yaml build trident-vm-testimage -s arch=amd64 -s scenario=grub-verity --output-dir ./artifacts --clones 2 # Move the clones into the update directories with the same filename -mv artifacts/trident-vm-grub-verity-testimage_0.cosi \ - artifacts/update-a/trident-vm-grub-verity-testimage.cosi -mv artifacts/trident-vm-grub-verity-testimage_1.cosi \ - artifacts/update-b/trident-vm-grub-verity-testimage.cosi +mv artifacts/trident-vm-testimage_amd64_grub-verity_cosi_clone0.cosi artifacts/update-a/trident-vm-grub-verity-testimage.cosi +mv artifacts/trident-vm-testimage_amd64_grub-verity_cosi_clone1.cosi artifacts/update-b/trident-vm-grub-verity-testimage.cosi ``` :::note diff --git a/tests/tailor-images/legacy-map.json b/tests/tailor-images/legacy-map.json new file mode 100644 index 0000000000..b87725ff5c --- /dev/null +++ b/tests/tailor-images/legacy-map.json @@ -0,0 +1,263 @@ +{ + "trident-installer": { + "image": "trident-installer", + "selectors": { + "arch": "amd64", + "variant": "default" + }, + "baseImage": "baremetal", + "ext": "iso" + }, + "trident-split-installer": { + "image": "trident-installer", + "selectors": { + "arch": "amd64", + "variant": "split" + }, + "baseImage": "baremetal", + "ext": "iso" + }, + "trident-installer-arm64": { + "image": "trident-installer", + "selectors": { + "arch": "arm64", + "variant": "default" + }, + "baseImage": "core_arm64", + "ext": "iso" + }, + "trident-container-installer": { + "image": "trident-container-installer", + "selectors": {}, + "baseImage": "baremetal", + "ext": "iso" + }, + "trident-direct-streaming-installer-amd64": { + "image": "trident-installer", + "selectors": { + "arch": "amd64", + "variant": "direct-streaming" + }, + "baseImage": "baremetal", + "ext": "iso" + }, + "trident-direct-streaming-installer-arm64": { + "image": "trident-installer", + "selectors": { + "arch": "arm64", + "variant": "direct-streaming" + }, + "baseImage": "core_arm64", + "ext": "iso" + }, + "trident-functest": { + "image": "trident-functest", + "selectors": {}, + "baseImage": "baremetal", + "ext": "qcow2" + }, + "trident-testimage": { + "image": "trident-testimage", + "selectors": { + "arch": "amd64" + }, + "baseImage": "baremetal", + "ext": "cosi" + }, + "trident-testimage-arm64": { + "image": "trident-testimage", + "selectors": { + "arch": "arm64" + }, + "baseImage": "core_arm64", + "ext": "cosi" + }, + "trident-verity-testimage": { + "image": "trident-verity-testimage", + "selectors": { + "deployment": "host", + "mode": "root" + }, + "baseImage": "baremetal", + "ext": "cosi" + }, + "trident-usrverity-testimage": { + "image": "trident-verity-testimage", + "selectors": { + "deployment": "host", + "mode": "usr" + }, + "baseImage": "baremetal", + "ext": "cosi" + }, + "trident-container-verity-testimage": { + "image": "trident-verity-testimage", + "selectors": { + "deployment": "container", + "mode": "root" + }, + "baseImage": "baremetal", + "ext": "cosi" + }, + "trident-container-usrverity-testimage": { + "image": "trident-verity-testimage", + "selectors": { + "deployment": "container", + "mode": "usr" + }, + "baseImage": "baremetal", + "ext": "cosi" + }, + "trident-container-testimage": { + "image": "trident-container-testimage", + "selectors": {}, + "baseImage": "baremetal", + "ext": "cosi" + }, + "azurelinux-direct-streaming-testimage-amd64": { + "image": "azurelinux-direct-streaming-testimage", + "selectors": { + "arch": "amd64" + }, + "baseImage": "baremetal", + "ext": "cosi", + "tailorExt": "raw" + }, + "azurelinux-direct-streaming-testimage-arm64": { + "image": "azurelinux-direct-streaming-testimage", + "selectors": { + "arch": "arm64" + }, + "baseImage": "core_arm64", + "ext": "cosi", + "tailorExt": "raw" + }, + "azl-installer": { + "image": "azl-installer", + "selectors": {}, + "baseImage": "baremetal", + "ext": "iso" + }, + "trident-vm-grub-testimage": { + "image": "trident-vm-testimage", + "selectors": { + "arch": "amd64", + "scenario": "grub" + }, + "baseImage": "qemu_guest", + "ext": "cosi" + }, + "trident-vm-grub-verity-testimage": { + "image": "trident-vm-testimage", + "selectors": { + "arch": "amd64", + "scenario": "grub-verity" + }, + "baseImage": "qemu_guest", + "ext": "cosi" + }, + "trident-vm-root-verity-testimage": { + "image": "trident-vm-testimage", + "selectors": { + "arch": "amd64", + "scenario": "root-verity" + }, + "baseImage": "qemu_guest", + "ext": "cosi" + }, + "trident-vm-usr-verity-testimage": { + "image": "trident-vm-testimage", + "selectors": { + "arch": "amd64", + "scenario": "usr-verity" + }, + "baseImage": "qemu_guest", + "ext": "cosi" + }, + "trident-vm-acl-agent-update-testimage": { + "image": "trident-vm-testimage", + "selectors": { + "arch": "amd64", + "scenario": "acl-agent" + }, + "baseImage": "qemu_guest", + "ext": "cosi" + }, + "trident-vm-grub-verity-azure-testimage": { + "image": "trident-vm-testimage", + "selectors": { + "arch": "amd64", + "scenario": "grub-verity-azure" + }, + "baseImage": "core_selinux", + "ext": "cosi" + }, + "trident-vm-grub-testimage-arm64": { + "image": "trident-vm-testimage", + "selectors": { + "arch": "arm64", + "scenario": "grub" + }, + "baseImage": "core_arm64", + "ext": "cosi" + }, + "trident-vm-grub-verity-testimage-arm64": { + "image": "trident-vm-testimage", + "selectors": { + "arch": "arm64", + "scenario": "grub-verity" + }, + "baseImage": "core_arm64", + "ext": "cosi" + }, + "ubuntu-direct-streaming-testimage-2204-amd64": { + "image": "foreign-direct-streaming-testimage", + "selectors": { + "arch": "amd64", + "source": "ubuntu-2204" + }, + "baseImage": "ubuntu_2204_amd64", + "ext": "cosi", + "tailorExt": "raw" + }, + "ubuntu-direct-streaming-testimage-2204-arm64": { + "image": "foreign-direct-streaming-testimage", + "selectors": { + "arch": "arm64", + "source": "ubuntu-2204" + }, + "baseImage": "ubuntu_2204_arm64", + "ext": "cosi", + "tailorExt": "raw" + }, + "ubuntu-direct-streaming-testimage-2404-amd64": { + "image": "foreign-direct-streaming-testimage", + "selectors": { + "arch": "amd64", + "source": "ubuntu-2404" + }, + "baseImage": "ubuntu_2404_amd64", + "ext": "cosi", + "tailorExt": "raw" + }, + "ubuntu-direct-streaming-testimage-2404-arm64": { + "image": "foreign-direct-streaming-testimage", + "selectors": { + "arch": "arm64", + "source": "ubuntu-2404" + }, + "baseImage": "ubuntu_2404_arm64", + "ext": "cosi", + "tailorExt": "raw" + }, + "gb200-direct-streaming-testimage-2404-arm64": { + "image": "foreign-direct-streaming-testimage", + "selectors": { + "arch": "arm64", + "source": "gb200-2404" + }, + "baseImage": "gb200_2404_arm64", + "ext": "cosi", + "tailorExt": "raw" + } +} diff --git a/tests/tailor-images/legacy_targets.py b/tests/tailor-images/legacy_targets.py new file mode 100644 index 0000000000..f706f7815f --- /dev/null +++ b/tests/tailor-images/legacy_targets.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +import argparse +import json +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +WORKSPACE = REPO_ROOT / "tests" / "tailor-images" / "tailor.yaml" +LEGACY_MAP = REPO_ROOT / "tests" / "tailor-images" / "legacy-map.json" +SELF = Path(__file__).resolve() + + +def load_map(): + return json.loads(LEGACY_MAP.read_text()) + + +def cargo_cmd() -> str: + cargo = shutil.which("cargo") + if cargo: + return cargo + fallback = Path.home() / ".cargo" / "bin" / "cargo" + if fallback.exists(): + return str(fallback) + raise SystemExit( + "cargo not found: install a Rust toolchain (see " + "../common_tasks/rustup.yml in CI) or ensure cargo is on PATH" + ) + + +def default_container_full() -> str: + text = WORKSPACE.read_text() + needle = """ entries:\n - name: ic\n container: """ + idx = text.find(needle) + if idx == -1: + raise SystemExit("default toolchain not found in tests/tailor-images/tailor.yaml") + rest = text[idx + len(needle) :].splitlines() + container = rest[0].strip() + tag = None + for line in rest[1:6]: + s = line.strip() + if s.startswith("tag:"): + tag = s.split(":", 1)[1].strip() + break + if tag: + return f"{container}:{tag}" + return container + + +def tailor_manifest_for_container(container_ref: str | None) -> Path: + if not container_ref or container_ref == default_container_full(): + return WORKSPACE + text = WORKSPACE.read_text() + name = "ic-override" + if ":" in container_ref.rsplit("/", 1)[-1]: + container, tag = container_ref.rsplit(":", 1) + tag_block = f" tag: {tag}\n" + else: + container = container_ref + tag_block = "" + needle = """toolchains:\n default: ic\n entries:\n - name: ic\n container: mcr.microsoft.com/azurelinux/imagecustomizer\n tag: latest\n""" + replacement = ( + "toolchains:\n" + f" default: {name}\n" + " entries:\n" + " - name: ic\n" + " container: mcr.microsoft.com/azurelinux/imagecustomizer\n" + " tag: latest\n" + f" - name: {name}\n" + f" container: {container}\n" + f"{tag_block}" + " pull: never\n" + ) + rendered = text.replace(needle, replacement, 1) + tmp = REPO_ROOT / "tests" / "tailor-images" / ".tailor.make.yaml" + tmp.write_text(rendered) + return tmp + + +def selector_args(entry: dict) -> list[str]: + args: list[str] = [] + for key, value in entry["selectors"].items(): + args.extend(["-s", f"{key}={value}"]) + return args + + +def call_tailor(*args: str, capture: bool = False) -> subprocess.CompletedProcess: + cmd = [ + cargo_cmd(), + "run", + "--manifest-path", + "tools/tailor/Cargo.toml", + "--quiet", + "--", + *args, + ] + env = dict(os.environ) + env["PATH"] = f"{Path.home() / '.cargo' / 'bin'}:{env.get('PATH','')}" + return subprocess.run( + cmd, cwd=str(REPO_ROOT), check=True, text=True, capture_output=capture, env=env + ) + + +def legacy_list(filter_type: str): + for name, entry in load_map().items(): + if entry["ext"] == filter_type: + print(name) + + +def dependencies(name: str): + mapping = load_map() + entry = mapping[name] + image_dir = REPO_ROOT / "tests" / "tailor-images" / entry["image"] + deps: list[Path] = [WORKSPACE, LEGACY_MAP, SELF] + base_dep_map = { + "baremetal": REPO_ROOT / "artifacts" / "baremetal.vhdx", + "core_selinux": REPO_ROOT / "artifacts" / "core_selinux.vhdx", + "core_arm64": REPO_ROOT / "artifacts" / "core_arm64.vhdx", + "qemu_guest": REPO_ROOT / "artifacts" / "qemu_guest.vhdx", + "ubuntu_2204_amd64": REPO_ROOT / "artifacts" / "ubuntu_2204_amd64.vhdx", + "ubuntu_2204_arm64": REPO_ROOT / "artifacts" / "ubuntu_2204_arm64.vhdx", + "ubuntu_2404_amd64": REPO_ROOT / "artifacts" / "ubuntu_2404_amd64.vhdx", + "ubuntu_2404_arm64": REPO_ROOT / "artifacts" / "ubuntu_2404_arm64.vhdx", + "gb200_2404_arm64": REPO_ROOT / "artifacts" / "gb200_2404_arm64.vhdx", + } + base_dep = base_dep_map.get(entry.get("baseImage")) + if base_dep is not None: + deps.append(base_dep) + for path in sorted(image_dir.rglob("*")): + if ( + path.is_file() + and ".rendered" not in path.parts + and path.name != ".tailor.make.yaml" + ): + deps.append(path) + needs_rpms = name not in { + "trident-functest", + "trident-container-installer", + "trident-container-testimage", + "trident-container-verity-testimage", + "trident-container-usrverity-testimage", + "ubuntu-direct-streaming-testimage-2204-amd64", + "ubuntu-direct-streaming-testimage-2204-arm64", + "ubuntu-direct-streaming-testimage-2404-amd64", + "ubuntu-direct-streaming-testimage-2404-arm64", + "gb200-direct-streaming-testimage-2404-arm64", + } + if needs_rpms: + rpm_dir = REPO_ROOT / "bin" / "RPMS" + if rpm_dir.exists(): + deps.append(rpm_dir) + deps.extend(sorted(rpm_dir.rglob("*.rpm"))) + if name in { + "trident-direct-streaming-installer-amd64", + "trident-direct-streaming-installer-arm64", + }: + for extra in [ + REPO_ROOT / "artifacts" / "rcp-agent", + REPO_ROOT / "artifacts" / "rcp-agent.service", + ]: + deps.append(extra) + if entry["image"] == "trident-vm-testimage": + # IC configs under trident-vm-testimage/ reference base/files/id_rsa.pub + # directly; the Makefile's own rule (keyed off artifacts/id_rsa[.pub], + # staged by the pipeline or `make artifacts/id_rsa`) copies the staged + # key into place. Without this dependency, make never runs that rule + # and IC fails with "failed to find SSH public key file". + ssh_key = image_dir / "base" / "files" / "id_rsa.pub" + deps.append(ssh_key) + if name == "azl-installer": + for extra in [ + REPO_ROOT + / "tests" + / "tailor-images" + / "azl-installer" + / "iso" + / "bin" + / "liveinstaller", + REPO_ROOT + / "tests" + / "tailor-images" + / "azl-installer" + / "iso" + / "images" + / "trident-testimage.cosi", + ]: + if extra.exists(): + deps.append(extra) + seen = set() + for dep in deps: + try: + s = str(dep.relative_to(REPO_ROOT)) + except ValueError: + s = str(dep) + if s not in seen: + seen.add(s) + print(s) + + +def resolve_slug(entry: dict, manifest: Path) -> str: + proc = call_tailor( + "--manifest", + str(manifest), + "matrix", + entry["image"], + "--format", + "json", + *selector_args(entry), + capture=True, + ) + cells = json.loads(proc.stdout) + if len(cells) != 1: + raise SystemExit( + f"expected exactly one cell for {entry['image']}, got {len(cells)}" + ) + return cells[0]["slug"] + + +def build(name: str, output_path: str, container: str | None): + mapping = load_map() + entry = mapping[name] + target = Path(output_path) + target.parent.mkdir(parents=True, exist_ok=True) + manifest = tailor_manifest_for_container(container) + try: + slug = resolve_slug(entry, manifest) + cert_path = REPO_ROOT / "tests" / "images" / "artifacts" / "ca_cert.pem" + cert_path.unlink(missing_ok=True) + call_tailor( + "--manifest", + str(manifest), + "build", + entry["image"], + *selector_args(entry), + "--output-dir", + str(target.parent), + ) + source_ext = entry.get("tailorExt", entry["ext"]) + built = target.parent / f"{slug}.{source_ext}" + if not built.exists(): + raise SystemExit(f"expected built artifact not found: {built}") + built.replace(target) + if cert_path.exists(): + shutil.copy2(cert_path, target.parent / "ca_cert.pem") + finally: + if manifest.name == ".tailor.make.yaml": + manifest.unlink(missing_ok=True) + + +def oras_download(name: str, out_path: Path): + ref = f"mcr.microsoft.com/azurelinux/3.0/image/{name}:latest" + with tempfile.TemporaryDirectory() as td: + subprocess.run( + ["oras", "pull", ref, "--output", td, "--platform", "linux/amd64"], + check=True, + ) + files = list(Path(td).glob("*.vhdx")) + if len(files) != 1: + raise SystemExit(f"expected one .vhdx from {ref}, got {len(files)}") + out_path.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(files[0]), str(out_path)) + + +def download_image(name: str): + target_map = { + "baremetal": REPO_ROOT / "artifacts" / "baremetal.vhdx", + "core_selinux": REPO_ROOT / "artifacts" / "core_selinux.vhdx", + "minimal": REPO_ROOT / "artifacts" / "minimal.vhdx", + "minimal_aarch64": REPO_ROOT / "artifacts" / "minimal_aarch64.vhdx", + } + if name not in target_map: + raise SystemExit(f"unsupported base image: {name}") + oras_download(name, target_map[name]) + + +def main(): + ap = argparse.ArgumentParser() + sub = ap.add_subparsers(dest="cmd", required=True) + + p = sub.add_parser("list") + p.add_argument("--filter-type", required=True) + + p = sub.add_parser("dependencies") + p.add_argument("name") + + p = sub.add_parser("build") + p.add_argument("name") + p.add_argument("--output-path", required=True) + p.add_argument("--container") + + p = sub.add_parser("show-artifact") + p.add_argument("item") + + p = sub.add_parser("download-image") + p.add_argument("name") + + args = ap.parse_args() + if args.cmd == "list": + legacy_list(args.filter_type) + elif args.cmd == "dependencies": + dependencies(args.name) + elif args.cmd == "build": + build(args.name, args.output_path, args.container) + elif args.cmd == "show-artifact": + if args.item != "customizer-container-full": + raise SystemExit(f"unsupported artifact item: {args.item}") + print(default_container_full()) + elif args.cmd == "download-image": + download_image(args.name) + + +if __name__ == "__main__": + import os + + main() From 2dc01612beb3a1d18d76af219bcb589c472f39d5 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Sat, 10 Oct 2026 22:39:56 +0000 Subject: [PATCH 2/6] Makefile: migrate the last raw docker-run VM image targets to tailor artifacts/trident-vm-{grub,grub-verity,root-verity,usr-verity,acl-agent}-testimage.qcow2 and artifacts/trident-vm-grub-verity-azure-testimage.vhd were built by a Makefile section invoking Image Customizer directly via `docker run` against the pre-tailor per-scenario YAML under tests/tailor-images/trident-vm-testimage/base/ - every one of those YAML files was deleted by the tests-cleanup commit, since nothing else referenced them, breaking these targets outright (confirmed from a live pipeline failure: "No rule to make target 'tests/tailor-images/trident-vm-testimage/base/baseimg-grub-verity.yaml'"). The previous commit (tests/tailor-images) added the missing qcow2/vhd-fixed outputs: entries to the affected scenarios, so tailor now produces these formats natively. This commit finishes the cutover: - extend the generic `artifacts/%.cosi artifacts/%.iso artifacts/%.vhdx:` pattern rule to also match `.qcow2` and `.vhd`, and delete the entire raw docker-run section (keeping VM_IMAGE_PATH_PREFIX, $(QEMU_GUEST_IMAGE)/$(CORE_ARM64_IMAGE), and the SSH-key-staging rule, which are still used by the tailor-based builds). - legacy_targets.py: a legacy name's selectors can now match more than one tailor cell (the same scenario's .cosi and .qcow2/.vhd-fixed outputs). Disambiguate using the extension of the Makefile's requested output path (already available, since the generic pattern rule's `$*` plus its own extension is exactly what's being asked for) instead of requiring every legacy name to resolve to exactly one cell unconditionally. - legacy-map.json: add the one legacy name with no existing entry at all, `trident-vm-acl-agent-testimage` (only the `-update-` variant existed). Verified: `make -n` for all 4 previously-failing targets now resolves correctly, and a real `make artifacts/trident-vm-acl-agent-testimage.qcow2` build succeeds end-to-end, producing both the existing .cosi and the new .qcow2 artifact. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Makefile | 172 +------------------------- tests/tailor-images/legacy-map.json | 9 ++ tests/tailor-images/legacy_targets.py | 30 ++++- 3 files changed, 37 insertions(+), 174 deletions(-) diff --git a/Makefile b/Makefile index 0bc1e43a10..a3cc1918c8 100644 --- a/Makefile +++ b/Makefile @@ -1044,7 +1044,7 @@ all-iso: $(ISO_TARGETS) # prerequisites so that we can use find to get all the files in the directory. # https://www.gnu.org/software/make/manual/make.html#Secondary-Expansion .SECONDEXPANSION: -artifacts/%.cosi artifacts/%.iso artifacts/%.vhdx: $$(shell python3 ./tests/tailor-images/legacy_targets.py dependencies $$*) +artifacts/%.cosi artifacts/%.iso artifacts/%.vhdx artifacts/%.qcow2 artifacts/%.vhd: $$(shell python3 ./tests/tailor-images/legacy_targets.py dependencies $$*) @echo "Building '$*' [$@] from $<" @echo "Prerequisites:" @echo "$^" | tr ' ' '\n' | sed 's/^/ /' @@ -1135,176 +1135,6 @@ $(MINIMAL_IMAGE_AARCH64): @mkdir -p artifacts @python3 ./tests/tailor-images/legacy_targets.py download-image minimal_aarch64 -artifacts/trident-vm-grub-testimage.qcow2: \ - $(QEMU_GUEST_IMAGE) \ - $(TRIDENT_VM_DEPENDENCIES) \ - $(VM_IMAGE_PATH_PREFIX)/baseimg-grub.yaml \ - $(VM_IMAGE_PATH_PREFIX)/files/id_rsa.pub \ - artifacts/rpm-overrides - @echo "Building $@ from $<" - docker run --rm \ - --privileged \ - -v ".:/repo:z" \ - -v "/dev:/dev" \ - ${MIC_CONTAINER_IMAGE} \ - --log-level debug \ - --rpm-source /repo/bin/RPMS \ - --rpm-source /repo/artifacts/rpm-overrides \ - --build-dir /build \ - --image-file /repo/$< \ - --output-image-file /repo/$@ \ - --output-image-format qcow2 \ - --config-file /repo/$(VM_IMAGE_PATH_PREFIX)/baseimg-grub.yaml - -artifacts/trident-vm-grub-testimage-arm64.qcow2: \ - $(CORE_ARM64_IMAGE) \ - $(TRIDENT_VM_DEPENDENCIES) \ - $(VM_IMAGE_PATH_PREFIX)/baseimg-grub.yaml \ - $(VM_IMAGE_PATH_PREFIX)/files/id_rsa.pub - @echo "Building $@ from $<" - docker run --rm \ - --privileged \ - -v ".:/repo:z" \ - -v "/dev:/dev" \ - ${MIC_CONTAINER_IMAGE} \ - --log-level debug \ - --rpm-source /repo/bin/RPMS \ - --build-dir /build \ - --image-file /repo/$< \ - --output-image-file /repo/$@ \ - --output-image-format qcow2 \ - --config-file /repo/$(VM_IMAGE_PATH_PREFIX)/baseimg-grub-verity.yaml - -artifacts/trident-vm-grub-verity-testimage.qcow2: \ - $(QEMU_GUEST_IMAGE) \ - $(TRIDENT_VM_DEPENDENCIES) \ - $(VM_IMAGE_PATH_PREFIX)/baseimg-grub-verity.yaml \ - $(VM_IMAGE_PATH_PREFIX)/files/etc-mount.service \ - $(VM_IMAGE_PATH_PREFIX)/files/etc-mount.sh \ - $(VM_IMAGE_PATH_PREFIX)/files/id_rsa.pub \ - artifacts/rpm-overrides - @echo "Building $@ from $<" - docker run --rm \ - --privileged \ - -v ".:/repo:z" \ - -v "/dev:/dev" \ - ${MIC_CONTAINER_IMAGE} \ - --log-level debug \ - --rpm-source /repo/bin/RPMS \ - --rpm-source /repo/artifacts/rpm-overrides \ - --build-dir /build \ - --image-file /repo/$< \ - --output-image-file /repo/$@ \ - --output-image-format qcow2 \ - --config-file /repo/$(VM_IMAGE_PATH_PREFIX)/baseimg-grub-verity.yaml - -artifacts/trident-vm-root-verity-testimage.qcow2: \ - $(QEMU_GUEST_IMAGE) \ - $(TRIDENT_VM_DEPENDENCIES) \ - $(VM_IMAGE_PATH_PREFIX)/baseimg-root-verity.yaml \ - $(VM_IMAGE_PATH_PREFIX)/files/id_rsa.pub \ - artifacts/rpm-overrides - @echo "Building $@ from $<" - docker run --rm \ - --privileged \ - -v ".:/repo:z" \ - -v "/dev:/dev" \ - ${MIC_CONTAINER_IMAGE} \ - --log-level debug \ - --rpm-source /repo/bin/RPMS \ - --rpm-source /repo/artifacts/rpm-overrides \ - --build-dir /build \ - --image-file /repo/$< \ - --output-image-file /repo/$@ \ - --output-image-format qcow2 \ - --config-file /repo/$(VM_IMAGE_PATH_PREFIX)/baseimg-root-verity.yaml - -artifacts/trident-vm-verity-testimage-arm64.qcow2: \ - $(CORE_ARM64_IMAGE) \ - $(TRIDENT_VM_DEPENDENCIES) \ - $(VM_IMAGE_PATH_PREFIX)/baseimg-verity.yaml \ - $(VM_IMAGE_PATH_PREFIX)/files/etc-mount.service \ - $(VM_IMAGE_PATH_PREFIX)/files/etc-mount.sh \ - $(VM_IMAGE_PATH_PREFIX)/files/id_rsa.pub - @echo "Building $@ from $<" - docker run --rm \ - --privileged \ - -v ".:/repo:z" \ - -v "/dev:/dev" \ - ${MIC_CONTAINER_IMAGE} \ - --log-level debug \ - --rpm-source /repo/bin/RPMS \ - --build-dir /build \ - --image-file /repo/$< \ - --output-image-file /repo/$@ \ - --output-image-format qcow2 \ - --config-file /repo/$(VM_IMAGE_PATH_PREFIX)/baseimg-verity.yaml - -artifacts/trident-vm-usr-verity-testimage.qcow2: \ - $(QEMU_GUEST_IMAGE) \ - $(TRIDENT_VM_DEPENDENCIES) \ - $(VM_IMAGE_PATH_PREFIX)/baseimg-usr-verity.yaml \ - $(VM_IMAGE_PATH_PREFIX)/files/id_rsa.pub \ - artifacts/rpm-overrides - @echo "Building $@ from $<" - docker run --rm \ - --privileged \ - -v ".:/repo:z" \ - -v "/dev:/dev" \ - ${MIC_CONTAINER_IMAGE} \ - --log-level debug \ - --rpm-source /repo/bin/RPMS \ - --rpm-source /repo/artifacts/rpm-overrides \ - --build-dir /build \ - --image-file /repo/$< \ - --output-image-file /repo/$@ \ - --output-image-format qcow2 \ - --config-file /repo/$(VM_IMAGE_PATH_PREFIX)/baseimg-usr-verity.yaml - -artifacts/trident-vm-acl-agent-testimage.qcow2: \ - $(QEMU_GUEST_IMAGE) \ - $(TRIDENT_VM_DEPENDENCIES) \ - $(VM_IMAGE_PATH_PREFIX)/baseimg-acl-agent.yaml \ - $(VM_IMAGE_PATH_PREFIX)/files/id_rsa.pub \ - $(VM_IMAGE_PATH_PREFIX)/files/trident-acl-agent-override.conf \ - artifacts/rpm-overrides - @echo "Building $@ from $<" - docker run --rm \ - --privileged \ - -v ".:/repo:z" \ - -v "/dev:/dev" \ - ${MIC_CONTAINER_IMAGE} \ - --log-level debug \ - --rpm-source /repo/bin/RPMS \ - --rpm-source /repo/artifacts/rpm-overrides \ - --build-dir /build \ - --image-file /repo/$< \ - --output-image-file /repo/$@ \ - --output-image-format qcow2 \ - --config-file /repo/$(VM_IMAGE_PATH_PREFIX)/baseimg-acl-agent.yaml - -artifacts/trident-vm-grub-verity-azure-testimage.vhd: \ - $(CORE_SELINUX_IMAGE) \ - $(TRIDENT_VM_DEPENDENCIES) \ - $(VM_IMAGE_PATH_PREFIX)/baseimg-grub-verity-azure.yaml \ - $(VM_IMAGE_PATH_PREFIX)/files/etc-mount.service \ - $(VM_IMAGE_PATH_PREFIX)/files/etc-mount.sh \ - $(VM_IMAGE_PATH_PREFIX)/files/id_rsa.pub \ - artifacts/rpm-overrides - @echo "Building $@ from $<" - docker run --rm \ - --privileged \ - -v ".:/repo:z" \ - -v "/dev:/dev" \ - ${MIC_CONTAINER_IMAGE} \ - --log-level debug \ - --rpm-source /repo/bin/RPMS \ - --rpm-source /repo/artifacts/rpm-overrides \ - --build-dir /build \ - --image-file /repo/$< \ - --output-image-file /repo/$@ \ - --output-image-format vhd-fixed \ - --config-file /repo/$(VM_IMAGE_PATH_PREFIX)/baseimg-grub-verity-azure.yaml .PHONY: imagecustomizer-dev-amd64 imagecustomizer-dev-amd64: diff --git a/tests/tailor-images/legacy-map.json b/tests/tailor-images/legacy-map.json index b87725ff5c..d5ce416b8a 100644 --- a/tests/tailor-images/legacy-map.json +++ b/tests/tailor-images/legacy-map.json @@ -183,6 +183,15 @@ "baseImage": "qemu_guest", "ext": "cosi" }, + "trident-vm-acl-agent-testimage": { + "image": "trident-vm-testimage", + "selectors": { + "arch": "amd64", + "scenario": "acl-agent" + }, + "baseImage": "qemu_guest", + "ext": "cosi" + }, "trident-vm-grub-verity-azure-testimage": { "image": "trident-vm-testimage", "selectors": { diff --git a/tests/tailor-images/legacy_targets.py b/tests/tailor-images/legacy_targets.py index f706f7815f..1088621699 100644 --- a/tests/tailor-images/legacy_targets.py +++ b/tests/tailor-images/legacy_targets.py @@ -35,7 +35,9 @@ def default_container_full() -> str: needle = """ entries:\n - name: ic\n container: """ idx = text.find(needle) if idx == -1: - raise SystemExit("default toolchain not found in tests/tailor-images/tailor.yaml") + raise SystemExit( + "default toolchain not found in tests/tailor-images/tailor.yaml" + ) rest = text[idx + len(needle) :].splitlines() container = rest[0].strip() tag = None @@ -199,7 +201,20 @@ def dependencies(name: str): print(s) -def resolve_slug(entry: dict, manifest: Path) -> str: +# A legacy name's selectors can now match more than one tailor cell when a +# scenario declares multiple outputs: formats (e.g. trident-vm-testimage's +# grub-verity scenario produces both a .cosi A/B-update payload and a +# .qcow2 "clean install" base disk from the very same selectors) - the +# Makefile's generic pattern rule already tells us which one via the +# requested output file's own extension, so use that to disambiguate +# instead of requiring every legacy name to resolve to exactly one cell +# unconditionally. +_EXT_TO_TAILOR_FORMAT = { + "vhd": "vhd-fixed", +} + + +def resolve_slug(entry: dict, manifest: Path, requested_ext: str | None = None) -> str: proc = call_tailor( "--manifest", str(manifest), @@ -211,6 +226,9 @@ def resolve_slug(entry: dict, manifest: Path) -> str: capture=True, ) cells = json.loads(proc.stdout) + if len(cells) > 1 and requested_ext is not None: + wanted_format = _EXT_TO_TAILOR_FORMAT.get(requested_ext, requested_ext) + cells = [c for c in cells if c.get("format") == wanted_format] if len(cells) != 1: raise SystemExit( f"expected exactly one cell for {entry['image']}, got {len(cells)}" @@ -224,8 +242,9 @@ def build(name: str, output_path: str, container: str | None): target = Path(output_path) target.parent.mkdir(parents=True, exist_ok=True) manifest = tailor_manifest_for_container(container) + requested_ext = target.suffix.lstrip(".") try: - slug = resolve_slug(entry, manifest) + slug = resolve_slug(entry, manifest, requested_ext) cert_path = REPO_ROOT / "tests" / "images" / "artifacts" / "ca_cert.pem" cert_path.unlink(missing_ok=True) call_tailor( @@ -238,6 +257,11 @@ def build(name: str, output_path: str, container: str | None): str(target.parent), ) source_ext = entry.get("tailorExt", entry["ext"]) + if requested_ext != entry["ext"]: + # The caller asked for a different format than this legacy name's + # default (see resolve_slug above) - trust the extension actually + # requested over legacy-map.json's single static default. + source_ext = requested_ext built = target.parent / f"{slug}.{source_ext}" if not built.exists(): raise SystemExit(f"expected built artifact not found: {built}") From 13277fbc73b1656c90f78d7117903f64693a0200 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Sun, 11 Oct 2026 00:04:38 +0000 Subject: [PATCH 3/6] tests: cut over legacy image builds to tailor Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../build_image/build-image-template.yml | 179 +++++---- Makefile | 213 +---------- tests/tailor-images/README.md | 8 +- tests/tailor-images/legacy-map.json | 272 -------------- tests/tailor-images/legacy_targets.py | 341 ------------------ tests/tailor-images/targets.mk | 113 ++++++ 6 files changed, 229 insertions(+), 897 deletions(-) delete mode 100644 tests/tailor-images/legacy-map.json delete mode 100644 tests/tailor-images/legacy_targets.py create mode 100644 tests/tailor-images/targets.mk diff --git a/.pipelines/templates/stages/build_image/build-image-template.yml b/.pipelines/templates/stages/build_image/build-image-template.yml index 713ffc2742..229624efae 100644 --- a/.pipelines/templates/stages/build_image/build-image-template.yml +++ b/.pipelines/templates/stages/build_image/build-image-template.yml @@ -87,15 +87,21 @@ steps: - bash: | set -euo pipefail - python3 - <<'PY' "${{ parameters.imageName }}" - import json - import sys - from pathlib import Path - - legacy = json.loads(Path("tests/tailor-images/legacy-map.json").read_text()) - entry = legacy[sys.argv[1]] - print(f"##vso[task.setvariable variable=baseImageType]{entry['baseImage']}") - PY + + case '${{ parameters.imageName }}' in + azl-installer|trident-container-installer|trident-container-testimage|trident-functest|trident-installer|trident-split-installer|trident-testimage|trident-verity-testimage|trident-usrverity-testimage|trident-container-verity-testimage|trident-container-usrverity-testimage|trident-direct-streaming-installer-amd64) + baseImageType=baremetal + ;; + trident-direct-streaming-installer-arm64|trident-installer-arm64|trident-testimage-arm64) + baseImageType=core_arm64 + ;; + *) + echo "Unsupported imageName for build-image-template: ${{ parameters.imageName }}" + exit 1 + ;; + esac + + echo "##vso[task.setvariable variable=baseImageType]${baseImageType}" displayName: "Resolve image metadata" workingDirectory: ${{ parameters.tridentSourceDirectory }} @@ -203,78 +209,109 @@ steps: export PATH="$HOME/.cargo/bin:$PATH" export CLICOLOR_FORCE=1 - readarray -t selector_lines < <(python3 - <<'PY' "$legacy_name" - import json - import sys - from pathlib import Path - - entry = json.loads(Path("tests/tailor-images/legacy-map.json").read_text())[sys.argv[1]] - print(entry["image"]) - print(entry["ext"]) - print(entry.get("tailorExt", entry["ext"])) - for key, value in entry["selectors"].items(): - print(f"{key}={value}") - PY - ) - - tailor_image="${selector_lines[0]}" - artifact_ext="${selector_lines[1]}" - source_ext="${selector_lines[2]}" selector_args=() - if (( ${#selector_lines[@]} > 3 )); then - for selector in "${selector_lines[@]:3}"; do - selector_args+=("-s" "$selector") - done - fi + case "$legacy_name" in + azl-installer) + tailor_image=azl-installer + artifact_ext=iso + ;; + trident-functest) + tailor_image=trident-functest + artifact_ext=qcow2 + ;; + trident-installer) + tailor_image=trident-installer + artifact_ext=iso + selector_args=(-s arch=amd64 -s variant=default) + ;; + trident-split-installer) + tailor_image=trident-installer + artifact_ext=iso + selector_args=(-s arch=amd64 -s variant=split) + ;; + trident-installer-arm64) + tailor_image=trident-installer + artifact_ext=iso + selector_args=(-s arch=arm64 -s variant=default) + ;; + trident-direct-streaming-installer-amd64) + tailor_image=trident-installer + artifact_ext=iso + selector_args=(-s arch=amd64 -s variant=direct-streaming) + ;; + trident-direct-streaming-installer-arm64) + tailor_image=trident-installer + artifact_ext=iso + selector_args=(-s arch=arm64 -s variant=direct-streaming) + ;; + trident-container-installer) + tailor_image=trident-container-installer + artifact_ext=iso + ;; + trident-testimage) + tailor_image=trident-testimage + artifact_ext=cosi + selector_args=(-s arch=amd64) + ;; + trident-testimage-arm64) + tailor_image=trident-testimage + artifact_ext=cosi + selector_args=(-s arch=arm64) + ;; + trident-container-testimage) + tailor_image=trident-container-testimage + artifact_ext=cosi + ;; + trident-verity-testimage) + tailor_image=trident-verity-testimage + artifact_ext=cosi + selector_args=(-s deployment=host -s mode=root) + ;; + trident-container-verity-testimage) + tailor_image=trident-verity-testimage + artifact_ext=cosi + selector_args=(-s deployment=container -s mode=root) + ;; + trident-usrverity-testimage) + tailor_image=trident-verity-testimage + artifact_ext=cosi + selector_args=(-s deployment=host -s mode=usr) + ;; + trident-container-usrverity-testimage) + tailor_image=trident-verity-testimage + artifact_ext=cosi + selector_args=(-s deployment=container -s mode=usr) + ;; + *) + echo "Unsupported imageName for build-image-template: ${legacy_name}" + exit 1 + ;; + esac manifest_path=tests/tailor-images/tailor.yaml if [[ "${{ parameters.micBuildType }}" == "dev" ]]; then manifest_path=tests/tailor-images/.tailor.pipeline.yaml python3 - <<'PY' - from pathlib import Path - text = Path("tests/tailor-images/tailor.yaml").read_text() - needle = """toolchains: - default: ic - entries: - - name: ic - container: mcr.microsoft.com/azurelinux/imagecustomizer - tag: latest - """ - replacement = """toolchains: - default: ic-dev - entries: - - name: ic - container: mcr.microsoft.com/azurelinux/imagecustomizer - tag: latest - - name: ic-dev - container: imagecustomizer - tag: dev - pull: never - """ - Path("tests/tailor-images/.tailor.pipeline.yaml").write_text(text.replace(needle, replacement, 1)) - PY +from pathlib import Path +text = Path('tests/tailor-images/tailor.yaml').read_text() +needle = 'toolchains:\n default: ic\n entries:\n - name: ic\n container: mcr.microsoft.com/azurelinux/imagecustomizer\n tag: latest\n' +replacement = 'toolchains:\n default: ic-dev\n entries:\n - name: ic\n container: mcr.microsoft.com/azurelinux/imagecustomizer\n tag: latest\n - name: ic-dev\n container: imagecustomizer\n tag: dev\n pull: never\n' +Path('tests/tailor-images/.tailor.pipeline.yaml').write_text(text.replace(needle, replacement, 1)) +PY fi - matrix_json=$(cargo run --manifest-path tools/tailor/Cargo.toml --quiet -- --manifest "$manifest_path" matrix "$tailor_image" --format json "${selector_args[@]}") - - slug=$(python3 - <<'PY' "$matrix_json" - import json - import sys - cells = json.loads(sys.argv[1]) - if len(cells) != 1: - raise SystemExit(f"expected exactly one cell, got {len(cells)}") - print(cells[0]["slug"]) - PY - ) - rm -f tests/tailor-images/artifacts/ca_cert.pem - cargo run --manifest-path tools/tailor/Cargo.toml --quiet -- --manifest "$manifest_path" build "$tailor_image" "${selector_args[@]}" --output-dir $(ob_outputDirectory) --clones ${{ parameters.clones }} - - if [[ ${{ parameters.clones }} -eq 1 ]]; then - mv "$(ob_outputDirectory)/${slug}.${source_ext}" "$(ob_outputDirectory)/${legacy_name}.${artifact_ext}" - else + cargo run --manifest-path tools/tailor/Cargo.toml --quiet -- \ + --manifest "$manifest_path" \ + build "$tailor_image" \ + "${selector_args[@]}" \ + --output-dir $(ob_outputDirectory) \ + --clones ${{ parameters.clones }} + + if [[ ${{ parameters.clones }} -gt 1 ]]; then for ((i=0; i<${{ parameters.clones }}; i++)); do - mv "$(ob_outputDirectory)/${slug}_clone${i}.${source_ext}" "$(ob_outputDirectory)/${legacy_name}_${i}.${artifact_ext}" + mv "$(ob_outputDirectory)/${legacy_name}_clone${i}.${artifact_ext}" \ + "$(ob_outputDirectory)/${legacy_name}_${i}.${artifact_ext}" done fi diff --git a/Makefile b/Makefile index a3cc1918c8..998d569742 100644 --- a/Makefile +++ b/Makefile @@ -633,25 +633,7 @@ AZL_INSTALLER_ISO_DIR := $(AZL_INSTALLER_DIR)/iso # If regular.cosi is not present, download runtime images $(ARTIFACTS_TEST_IMAGE_DIR)/regular.cosi: download-runtime-images -# Build the installer ISO using the builder -$(ARTIFACTS_TEST_IMAGE_DIR)/azl-installer.iso: \ - bin/RPMS \ - bin/liveinstaller \ - artifacts/baremetal.vhdx \ - $(ARTIFACTS_TEST_IMAGE_DIR)/regular.cosi \ - $(AZL_INSTALLER_DIR)/installer-iso.yaml \ - $(shell find $(AZL_INSTALLER_DIR)/ -type f 2>/dev/null) - # Prepare dependencies - # Copy runtime image - rm -rf $(AZL_INSTALLER_ISO_DIR)/images - mkdir -p $(AZL_INSTALLER_ISO_DIR)/images - cp $(ARTIFACTS_TEST_IMAGE_DIR)/regular.cosi $(AZL_INSTALLER_ISO_DIR)/images/trident-testimage.cosi - # Copy installer binary - rm -rf $(AZL_INSTALLER_ISO_DIR)/bin - mkdir -p $(AZL_INSTALLER_ISO_DIR)/bin - cp bin/liveinstaller $(AZL_INSTALLER_ISO_DIR)/bin/ - # Build ISO - python3 ./tests/tailor-images/legacy_targets.py build azl-installer --output-path $(ARTIFACTS_TEST_IMAGE_DIR)/azl-installer.iso $(if $(strip $(MIC_CONTAINER_IMAGE)),--container $(MIC_CONTAINER_IMAGE)) +include tests/tailor-images/targets.mk .PHONY: validate validate: $(TRIDENT_CONFIG) target/release/trident @@ -885,186 +867,6 @@ download-trident-container-installer-iso: --path artifacts/ \ --artifact-name 'trident-container-installer' -artifacts/trident-container-installer.iso: $(shell python3 ./tests/tailor-images/legacy_targets.py dependencies trident-container-installer) - @echo "Building 'trident-container-installer' [$@] from $<" - @echo "Prerequisites:" - @echo "$^" | tr ' ' '\n' | sed 's/^/ /' - @echo "Building image..." - python3 ./tests/tailor-images/legacy_targets.py build \ - trident-container-installer \ - --output-path $@ \ - $(if $(strip $(MIC_CONTAINER_IMAGE)),--container $(MIC_CONTAINER_IMAGE)) - -# Copies locally built runtime images from ../test-images/build to ./artifacts/test-image. -# Expects that both the regular and verity Trident test images have been built. -.PHONY: copy-runtime-images -copy-runtime-images: $(TEST_IMAGES_PATH)/build/trident-testimage/*.cosi $(TEST_IMAGES_PATH)/build/trident-verity-testimage/*.cosi - @test -d $(TEST_IMAGES_PATH) || { \ - echo "$(TEST_IMAGES_PATH) not found"; \ - exit 1; \ - } - @test -d $(TEST_IMAGES_PATH)/build/trident-testimage || { \ - echo "$(TEST_IMAGES_PATH)/build/trident-testimage not found"; \ - exit 1; \ - } - @test -d $(TEST_IMAGES_PATH)/build/trident-verity-testimage || { \ - echo "$(TEST_IMAGES_PATH)/build/trident-verity-testimage not found"; \ - exit 1; \ - } - - @rm -rf ./artifacts/test-image - @mkdir -p ./artifacts/test-image - -# Copy all COSI images from trident-testimage - @for file in $(TEST_IMAGES_PATH)/build/trident-testimage/*.cosi; do \ - cp $$file ./artifacts/test-image/$$(basename $$file); \ - echo "Copied $$file to ./artifacts/test-image/$$(basename $$file)"; \ - done - -# Copy all COSI images from trident-verity-testimage - @for file in $(TEST_IMAGES_PATH)/build/trident-verity-testimage/*.cosi; do \ - cp $$file ./artifacts/test-image/$$(basename $$file); \ - echo "Copied $$file to ./artifacts/test-image/$$(basename $$file)"; \ - done - -# Uses the simple E2E test to set up a starter Host Configuration -.PHONY: starter-configuration -starter-configuration: - @mkdir -p $$(dirname $(TRIDENT_CONFIG)) - @cp tests/e2e_tests/trident_configurations/simple/trident-config.yaml $(TRIDENT_CONFIG) - @echo "\033[33mCreated \033[36m$(TRIDENT_CONFIG)\033[33m. Please review and modify as needed! :)" - @echo "\033[33mDon't forget to add your SSH public key to the Host Configuration!" - -bin/trident-mos.iso: \ - artifacts/baremetal.vhdx \ - packaging/systemd/trident-install.service \ - tests/tailor-images/trident-mos/iso.yaml \ - tests/tailor-images/trident-mos/files/* \ - tests/tailor-images/trident-mos/post-install.sh \ - packaging/selinux-policy-trident/* \ - tools/cmd/rcp-agent/rcp-agent.service \ - bin/rcp-agent - @echo "Rebuilding Trident MOS ISO: $@ from $< because of: $?" - @mkdir -p bin - docker run --rm \ - --privileged \ - -v ".:/repo:z" \ - -v "/dev:/dev" \ - ${MIC_CONTAINER_IMAGE} \ - --log-level=debug \ - --build-dir /build \ - --image-file /repo/$< \ - --output-image-file /repo/$@ \ - --config-file /repo/tests/tailor-images/trident-mos/iso.yaml \ - --output-image-format iso - -.PHONY: recreate-verity-image -recreate-verity-image: bin/trident-rpms.tar.gz - $(MAKE) -C $(TEST_IMAGES_PATH) copy-trident-rpms - $(MAKE) -C $(TEST_IMAGES_PATH) trident-verity-testimage - make copy-runtime-images - -.PHONY: website-clear -website-clear: - cd ./website && \ - rm -rf ./docs && \ - rm -rf ./versioned_* && \ - rm -rf ./versions.json && \ - rm -rf ./node_modules - -.PHONY: website-prereqs -website-prereqs: - cd ./website && npm ci - -DOCS_CONTENTS = $(shell find ./docs -type f) -website/docs: $(DOCS_CONTENTS) - rm -rf ./website/docs && \ - cp -r ./docs ./website - -website/versions.json: - echo '[]' > website/versions.json - -.PHONY: website-build -website-build: website-prereqs website/docs website/versions.json - cd ./website && \ - npm run build - -.PHONY: website-serve -website-serve: website-build - cd ./website && \ - npm run serve -- --port $(SERVER_PORT) - -# Useful if you edit website/docs files in place and want to copy them back to ./docs -.PHONY: website-reverse -website-reverse: website/docs - rm -rf ./docs - cp -r ./website/docs ./ - -.PHONY: validate-pipeline-website-artifact -validate-pipeline-website-artifact: - if ! which gh > /dev/null; then \ - sudo apt install gh; \ - fi - $(eval STAGING_DIR := $(shell mktemp -d)) - cp -r ./website/. $(STAGING_DIR)/ - rm -rf $(STAGING_DIR)/build && \ - mkdir -p $(STAGING_DIR)/build - $(eval RUN_ID ?= $(shell gh run list --workflow 'Deploy to GitHub Pages' --repo microsoft/trident --json conclusion,databaseId --jq '.[] | select(.conclusion == "success") | .databaseId' | sort -n | tail -n 1)) - @echo "Downloading GitHub Pages artifact from $(RUN_ID)" - gh run download $(RUN_ID) --name github-pages --repo microsoft/trident --dir $(STAGING_DIR)/build - cd $(STAGING_DIR)/build && \ - tar -xvf ./artifact.tar && \ - cd $(STAGING_DIR) && \ - npm install && \ - npm run serve -- --port $(SERVER_PORT) - -# -# Generic COSI image build target pattern -# -COSI_TARGETS = $(shell python3 ./tests/tailor-images/legacy_targets.py list --filter-type cosi) - -.PHONY: $(COSI_TARGETS) -$(COSI_TARGETS): %: artifacts/%.cosi - -.PHONY: all-cosi -all-cosi: $(COSI_TARGETS) - -# -# Generic ISO image build target pattern -# -ISO_TARGETS = $(shell python3 ./tests/tailor-images/legacy_targets.py list --filter-type iso) - -.PHONY: $(ISO_TARGETS) -$(ISO_TARGETS): %: artifacts/%.iso - -.PHONY: all-iso -all-iso: $(ISO_TARGETS) - -# Fun trick to use the stem of the target (%) as a variable ($*) in the -# prerequisites so that we can use find to get all the files in the directory. -# https://www.gnu.org/software/make/manual/make.html#Secondary-Expansion -.SECONDEXPANSION: -artifacts/%.cosi artifacts/%.iso artifacts/%.vhdx artifacts/%.qcow2 artifacts/%.vhd: $$(shell python3 ./tests/tailor-images/legacy_targets.py dependencies $$*) - @echo "Building '$*' [$@] from $<" - @echo "Prerequisites:" - @echo "$^" | tr ' ' '\n' | sed 's/^/ /' - @echo "Building image..." - python3 ./tests/tailor-images/legacy_targets.py build \ - $* \ - --output-path $@ \ - $(if $(strip $(MIC_CONTAINER_IMAGE)),--container $(MIC_CONTAINER_IMAGE)) - -MIC_CONTAINER_IMAGE ?= $(shell python3 ./tests/tailor-images/legacy_targets.py show-artifact customizer-container-full) -artifacts/trident-functest.qcow2: $$(shell python3 ./tests/tailor-images/legacy_targets.py dependencies $$(basename $$(notdir $$@))) - @echo "Building '$*' [$@] from $<" - @echo "Prerequisites:" - @echo "$^" | tr ' ' '\n' | sed 's/^/ /' - @echo "Building image..." - python3 ./tests/tailor-images/legacy_targets.py build \ - $(basename $(notdir $@)) \ - --output-path $@ \ - $(if $(strip $(MIC_CONTAINER_IMAGE)),--container $(MIC_CONTAINER_IMAGE)) - # TRIDENT VM UPDATE IMAGES VM_IMAGE_PATH_PREFIX = tests/tailor-images/trident-vm-testimage/base @@ -1102,12 +904,12 @@ $(QEMU_GUEST_IMAGE): BAREMETAL_IMAGE = artifacts/baremetal.vhdx $(BAREMETAL_IMAGE): @mkdir -p artifacts - @python3 ./tests/tailor-images/legacy_targets.py download-image baremetal + $(CARGO_TAILOR) run --manifest-path tools/tailor/Cargo.toml --quiet -- --manifest tests/tailor-images/tailor.yaml bases download baremetal CORE_SELINUX_IMAGE = artifacts/core_selinux.vhdx $(CORE_SELINUX_IMAGE): @mkdir -p artifacts - @python3 ./tests/tailor-images/legacy_targets.py download-image core_selinux + $(CARGO_TAILOR) run --manifest-path tools/tailor/Cargo.toml --quiet -- --manifest tests/tailor-images/tailor.yaml bases download core_selinux CORE_ARM64_IMAGE_NAME ?= core_vhdx-arm64-3.0-stable CORE_ARM64_IMAGE_VERSION ?= * @@ -1125,15 +927,6 @@ artifacts/rcp-agent.service: tools/cmd/rcp-agent/rcp-agent.service @mkdir -p artifacts @cp tools/cmd/rcp-agent/rcp-agent.service $@ -MINIMAL_IMAGE = artifacts/minimal.vhdx -$(MINIMAL_IMAGE): - @mkdir -p artifacts - @python3 ./tests/tailor-images/legacy_targets.py download-image minimal - -MINIMAL_IMAGE_AARCH64 = artifacts/minimal_aarch64.vhdx -$(MINIMAL_IMAGE_AARCH64): - @mkdir -p artifacts - @python3 ./tests/tailor-images/legacy_targets.py download-image minimal_aarch64 .PHONY: imagecustomizer-dev-amd64 diff --git a/tests/tailor-images/README.md b/tests/tailor-images/README.md index bb9666b62d..e3d7201ab0 100644 --- a/tests/tailor-images/README.md +++ b/tests/tailor-images/README.md @@ -48,9 +48,11 @@ remote source. ## Legacy name mapping -Pipelines still accept the historical image names. `tests/tailor-images/legacy-map.json` -maps each legacy name to the tailor image family, selectors, expected base slot, -and published artifact extension. +Pipelines and Make targets still accept the historical image names, but the +mapping is now declarative: each image family emits the legacy basename via +`outputs[].name`, while the remaining glue for aliases, clone suffixes, and +raw-to-`.cosi` compatibility lives directly in the Makefile / pipeline +templates. The current families are: diff --git a/tests/tailor-images/legacy-map.json b/tests/tailor-images/legacy-map.json deleted file mode 100644 index d5ce416b8a..0000000000 --- a/tests/tailor-images/legacy-map.json +++ /dev/null @@ -1,272 +0,0 @@ -{ - "trident-installer": { - "image": "trident-installer", - "selectors": { - "arch": "amd64", - "variant": "default" - }, - "baseImage": "baremetal", - "ext": "iso" - }, - "trident-split-installer": { - "image": "trident-installer", - "selectors": { - "arch": "amd64", - "variant": "split" - }, - "baseImage": "baremetal", - "ext": "iso" - }, - "trident-installer-arm64": { - "image": "trident-installer", - "selectors": { - "arch": "arm64", - "variant": "default" - }, - "baseImage": "core_arm64", - "ext": "iso" - }, - "trident-container-installer": { - "image": "trident-container-installer", - "selectors": {}, - "baseImage": "baremetal", - "ext": "iso" - }, - "trident-direct-streaming-installer-amd64": { - "image": "trident-installer", - "selectors": { - "arch": "amd64", - "variant": "direct-streaming" - }, - "baseImage": "baremetal", - "ext": "iso" - }, - "trident-direct-streaming-installer-arm64": { - "image": "trident-installer", - "selectors": { - "arch": "arm64", - "variant": "direct-streaming" - }, - "baseImage": "core_arm64", - "ext": "iso" - }, - "trident-functest": { - "image": "trident-functest", - "selectors": {}, - "baseImage": "baremetal", - "ext": "qcow2" - }, - "trident-testimage": { - "image": "trident-testimage", - "selectors": { - "arch": "amd64" - }, - "baseImage": "baremetal", - "ext": "cosi" - }, - "trident-testimage-arm64": { - "image": "trident-testimage", - "selectors": { - "arch": "arm64" - }, - "baseImage": "core_arm64", - "ext": "cosi" - }, - "trident-verity-testimage": { - "image": "trident-verity-testimage", - "selectors": { - "deployment": "host", - "mode": "root" - }, - "baseImage": "baremetal", - "ext": "cosi" - }, - "trident-usrverity-testimage": { - "image": "trident-verity-testimage", - "selectors": { - "deployment": "host", - "mode": "usr" - }, - "baseImage": "baremetal", - "ext": "cosi" - }, - "trident-container-verity-testimage": { - "image": "trident-verity-testimage", - "selectors": { - "deployment": "container", - "mode": "root" - }, - "baseImage": "baremetal", - "ext": "cosi" - }, - "trident-container-usrverity-testimage": { - "image": "trident-verity-testimage", - "selectors": { - "deployment": "container", - "mode": "usr" - }, - "baseImage": "baremetal", - "ext": "cosi" - }, - "trident-container-testimage": { - "image": "trident-container-testimage", - "selectors": {}, - "baseImage": "baremetal", - "ext": "cosi" - }, - "azurelinux-direct-streaming-testimage-amd64": { - "image": "azurelinux-direct-streaming-testimage", - "selectors": { - "arch": "amd64" - }, - "baseImage": "baremetal", - "ext": "cosi", - "tailorExt": "raw" - }, - "azurelinux-direct-streaming-testimage-arm64": { - "image": "azurelinux-direct-streaming-testimage", - "selectors": { - "arch": "arm64" - }, - "baseImage": "core_arm64", - "ext": "cosi", - "tailorExt": "raw" - }, - "azl-installer": { - "image": "azl-installer", - "selectors": {}, - "baseImage": "baremetal", - "ext": "iso" - }, - "trident-vm-grub-testimage": { - "image": "trident-vm-testimage", - "selectors": { - "arch": "amd64", - "scenario": "grub" - }, - "baseImage": "qemu_guest", - "ext": "cosi" - }, - "trident-vm-grub-verity-testimage": { - "image": "trident-vm-testimage", - "selectors": { - "arch": "amd64", - "scenario": "grub-verity" - }, - "baseImage": "qemu_guest", - "ext": "cosi" - }, - "trident-vm-root-verity-testimage": { - "image": "trident-vm-testimage", - "selectors": { - "arch": "amd64", - "scenario": "root-verity" - }, - "baseImage": "qemu_guest", - "ext": "cosi" - }, - "trident-vm-usr-verity-testimage": { - "image": "trident-vm-testimage", - "selectors": { - "arch": "amd64", - "scenario": "usr-verity" - }, - "baseImage": "qemu_guest", - "ext": "cosi" - }, - "trident-vm-acl-agent-update-testimage": { - "image": "trident-vm-testimage", - "selectors": { - "arch": "amd64", - "scenario": "acl-agent" - }, - "baseImage": "qemu_guest", - "ext": "cosi" - }, - "trident-vm-acl-agent-testimage": { - "image": "trident-vm-testimage", - "selectors": { - "arch": "amd64", - "scenario": "acl-agent" - }, - "baseImage": "qemu_guest", - "ext": "cosi" - }, - "trident-vm-grub-verity-azure-testimage": { - "image": "trident-vm-testimage", - "selectors": { - "arch": "amd64", - "scenario": "grub-verity-azure" - }, - "baseImage": "core_selinux", - "ext": "cosi" - }, - "trident-vm-grub-testimage-arm64": { - "image": "trident-vm-testimage", - "selectors": { - "arch": "arm64", - "scenario": "grub" - }, - "baseImage": "core_arm64", - "ext": "cosi" - }, - "trident-vm-grub-verity-testimage-arm64": { - "image": "trident-vm-testimage", - "selectors": { - "arch": "arm64", - "scenario": "grub-verity" - }, - "baseImage": "core_arm64", - "ext": "cosi" - }, - "ubuntu-direct-streaming-testimage-2204-amd64": { - "image": "foreign-direct-streaming-testimage", - "selectors": { - "arch": "amd64", - "source": "ubuntu-2204" - }, - "baseImage": "ubuntu_2204_amd64", - "ext": "cosi", - "tailorExt": "raw" - }, - "ubuntu-direct-streaming-testimage-2204-arm64": { - "image": "foreign-direct-streaming-testimage", - "selectors": { - "arch": "arm64", - "source": "ubuntu-2204" - }, - "baseImage": "ubuntu_2204_arm64", - "ext": "cosi", - "tailorExt": "raw" - }, - "ubuntu-direct-streaming-testimage-2404-amd64": { - "image": "foreign-direct-streaming-testimage", - "selectors": { - "arch": "amd64", - "source": "ubuntu-2404" - }, - "baseImage": "ubuntu_2404_amd64", - "ext": "cosi", - "tailorExt": "raw" - }, - "ubuntu-direct-streaming-testimage-2404-arm64": { - "image": "foreign-direct-streaming-testimage", - "selectors": { - "arch": "arm64", - "source": "ubuntu-2404" - }, - "baseImage": "ubuntu_2404_arm64", - "ext": "cosi", - "tailorExt": "raw" - }, - "gb200-direct-streaming-testimage-2404-arm64": { - "image": "foreign-direct-streaming-testimage", - "selectors": { - "arch": "arm64", - "source": "gb200-2404" - }, - "baseImage": "gb200_2404_arm64", - "ext": "cosi", - "tailorExt": "raw" - } -} diff --git a/tests/tailor-images/legacy_targets.py b/tests/tailor-images/legacy_targets.py deleted file mode 100644 index 1088621699..0000000000 --- a/tests/tailor-images/legacy_targets.py +++ /dev/null @@ -1,341 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import json -import shutil -import subprocess -import sys -import tempfile -from pathlib import Path - -REPO_ROOT = Path(__file__).resolve().parents[2] -WORKSPACE = REPO_ROOT / "tests" / "tailor-images" / "tailor.yaml" -LEGACY_MAP = REPO_ROOT / "tests" / "tailor-images" / "legacy-map.json" -SELF = Path(__file__).resolve() - - -def load_map(): - return json.loads(LEGACY_MAP.read_text()) - - -def cargo_cmd() -> str: - cargo = shutil.which("cargo") - if cargo: - return cargo - fallback = Path.home() / ".cargo" / "bin" / "cargo" - if fallback.exists(): - return str(fallback) - raise SystemExit( - "cargo not found: install a Rust toolchain (see " - "../common_tasks/rustup.yml in CI) or ensure cargo is on PATH" - ) - - -def default_container_full() -> str: - text = WORKSPACE.read_text() - needle = """ entries:\n - name: ic\n container: """ - idx = text.find(needle) - if idx == -1: - raise SystemExit( - "default toolchain not found in tests/tailor-images/tailor.yaml" - ) - rest = text[idx + len(needle) :].splitlines() - container = rest[0].strip() - tag = None - for line in rest[1:6]: - s = line.strip() - if s.startswith("tag:"): - tag = s.split(":", 1)[1].strip() - break - if tag: - return f"{container}:{tag}" - return container - - -def tailor_manifest_for_container(container_ref: str | None) -> Path: - if not container_ref or container_ref == default_container_full(): - return WORKSPACE - text = WORKSPACE.read_text() - name = "ic-override" - if ":" in container_ref.rsplit("/", 1)[-1]: - container, tag = container_ref.rsplit(":", 1) - tag_block = f" tag: {tag}\n" - else: - container = container_ref - tag_block = "" - needle = """toolchains:\n default: ic\n entries:\n - name: ic\n container: mcr.microsoft.com/azurelinux/imagecustomizer\n tag: latest\n""" - replacement = ( - "toolchains:\n" - f" default: {name}\n" - " entries:\n" - " - name: ic\n" - " container: mcr.microsoft.com/azurelinux/imagecustomizer\n" - " tag: latest\n" - f" - name: {name}\n" - f" container: {container}\n" - f"{tag_block}" - " pull: never\n" - ) - rendered = text.replace(needle, replacement, 1) - tmp = REPO_ROOT / "tests" / "tailor-images" / ".tailor.make.yaml" - tmp.write_text(rendered) - return tmp - - -def selector_args(entry: dict) -> list[str]: - args: list[str] = [] - for key, value in entry["selectors"].items(): - args.extend(["-s", f"{key}={value}"]) - return args - - -def call_tailor(*args: str, capture: bool = False) -> subprocess.CompletedProcess: - cmd = [ - cargo_cmd(), - "run", - "--manifest-path", - "tools/tailor/Cargo.toml", - "--quiet", - "--", - *args, - ] - env = dict(os.environ) - env["PATH"] = f"{Path.home() / '.cargo' / 'bin'}:{env.get('PATH','')}" - return subprocess.run( - cmd, cwd=str(REPO_ROOT), check=True, text=True, capture_output=capture, env=env - ) - - -def legacy_list(filter_type: str): - for name, entry in load_map().items(): - if entry["ext"] == filter_type: - print(name) - - -def dependencies(name: str): - mapping = load_map() - entry = mapping[name] - image_dir = REPO_ROOT / "tests" / "tailor-images" / entry["image"] - deps: list[Path] = [WORKSPACE, LEGACY_MAP, SELF] - base_dep_map = { - "baremetal": REPO_ROOT / "artifacts" / "baremetal.vhdx", - "core_selinux": REPO_ROOT / "artifacts" / "core_selinux.vhdx", - "core_arm64": REPO_ROOT / "artifacts" / "core_arm64.vhdx", - "qemu_guest": REPO_ROOT / "artifacts" / "qemu_guest.vhdx", - "ubuntu_2204_amd64": REPO_ROOT / "artifacts" / "ubuntu_2204_amd64.vhdx", - "ubuntu_2204_arm64": REPO_ROOT / "artifacts" / "ubuntu_2204_arm64.vhdx", - "ubuntu_2404_amd64": REPO_ROOT / "artifacts" / "ubuntu_2404_amd64.vhdx", - "ubuntu_2404_arm64": REPO_ROOT / "artifacts" / "ubuntu_2404_arm64.vhdx", - "gb200_2404_arm64": REPO_ROOT / "artifacts" / "gb200_2404_arm64.vhdx", - } - base_dep = base_dep_map.get(entry.get("baseImage")) - if base_dep is not None: - deps.append(base_dep) - for path in sorted(image_dir.rglob("*")): - if ( - path.is_file() - and ".rendered" not in path.parts - and path.name != ".tailor.make.yaml" - ): - deps.append(path) - needs_rpms = name not in { - "trident-functest", - "trident-container-installer", - "trident-container-testimage", - "trident-container-verity-testimage", - "trident-container-usrverity-testimage", - "ubuntu-direct-streaming-testimage-2204-amd64", - "ubuntu-direct-streaming-testimage-2204-arm64", - "ubuntu-direct-streaming-testimage-2404-amd64", - "ubuntu-direct-streaming-testimage-2404-arm64", - "gb200-direct-streaming-testimage-2404-arm64", - } - if needs_rpms: - rpm_dir = REPO_ROOT / "bin" / "RPMS" - if rpm_dir.exists(): - deps.append(rpm_dir) - deps.extend(sorted(rpm_dir.rglob("*.rpm"))) - if name in { - "trident-direct-streaming-installer-amd64", - "trident-direct-streaming-installer-arm64", - }: - for extra in [ - REPO_ROOT / "artifacts" / "rcp-agent", - REPO_ROOT / "artifacts" / "rcp-agent.service", - ]: - deps.append(extra) - if entry["image"] == "trident-vm-testimage": - # IC configs under trident-vm-testimage/ reference base/files/id_rsa.pub - # directly; the Makefile's own rule (keyed off artifacts/id_rsa[.pub], - # staged by the pipeline or `make artifacts/id_rsa`) copies the staged - # key into place. Without this dependency, make never runs that rule - # and IC fails with "failed to find SSH public key file". - ssh_key = image_dir / "base" / "files" / "id_rsa.pub" - deps.append(ssh_key) - if name == "azl-installer": - for extra in [ - REPO_ROOT - / "tests" - / "tailor-images" - / "azl-installer" - / "iso" - / "bin" - / "liveinstaller", - REPO_ROOT - / "tests" - / "tailor-images" - / "azl-installer" - / "iso" - / "images" - / "trident-testimage.cosi", - ]: - if extra.exists(): - deps.append(extra) - seen = set() - for dep in deps: - try: - s = str(dep.relative_to(REPO_ROOT)) - except ValueError: - s = str(dep) - if s not in seen: - seen.add(s) - print(s) - - -# A legacy name's selectors can now match more than one tailor cell when a -# scenario declares multiple outputs: formats (e.g. trident-vm-testimage's -# grub-verity scenario produces both a .cosi A/B-update payload and a -# .qcow2 "clean install" base disk from the very same selectors) - the -# Makefile's generic pattern rule already tells us which one via the -# requested output file's own extension, so use that to disambiguate -# instead of requiring every legacy name to resolve to exactly one cell -# unconditionally. -_EXT_TO_TAILOR_FORMAT = { - "vhd": "vhd-fixed", -} - - -def resolve_slug(entry: dict, manifest: Path, requested_ext: str | None = None) -> str: - proc = call_tailor( - "--manifest", - str(manifest), - "matrix", - entry["image"], - "--format", - "json", - *selector_args(entry), - capture=True, - ) - cells = json.loads(proc.stdout) - if len(cells) > 1 and requested_ext is not None: - wanted_format = _EXT_TO_TAILOR_FORMAT.get(requested_ext, requested_ext) - cells = [c for c in cells if c.get("format") == wanted_format] - if len(cells) != 1: - raise SystemExit( - f"expected exactly one cell for {entry['image']}, got {len(cells)}" - ) - return cells[0]["slug"] - - -def build(name: str, output_path: str, container: str | None): - mapping = load_map() - entry = mapping[name] - target = Path(output_path) - target.parent.mkdir(parents=True, exist_ok=True) - manifest = tailor_manifest_for_container(container) - requested_ext = target.suffix.lstrip(".") - try: - slug = resolve_slug(entry, manifest, requested_ext) - cert_path = REPO_ROOT / "tests" / "images" / "artifacts" / "ca_cert.pem" - cert_path.unlink(missing_ok=True) - call_tailor( - "--manifest", - str(manifest), - "build", - entry["image"], - *selector_args(entry), - "--output-dir", - str(target.parent), - ) - source_ext = entry.get("tailorExt", entry["ext"]) - if requested_ext != entry["ext"]: - # The caller asked for a different format than this legacy name's - # default (see resolve_slug above) - trust the extension actually - # requested over legacy-map.json's single static default. - source_ext = requested_ext - built = target.parent / f"{slug}.{source_ext}" - if not built.exists(): - raise SystemExit(f"expected built artifact not found: {built}") - built.replace(target) - if cert_path.exists(): - shutil.copy2(cert_path, target.parent / "ca_cert.pem") - finally: - if manifest.name == ".tailor.make.yaml": - manifest.unlink(missing_ok=True) - - -def oras_download(name: str, out_path: Path): - ref = f"mcr.microsoft.com/azurelinux/3.0/image/{name}:latest" - with tempfile.TemporaryDirectory() as td: - subprocess.run( - ["oras", "pull", ref, "--output", td, "--platform", "linux/amd64"], - check=True, - ) - files = list(Path(td).glob("*.vhdx")) - if len(files) != 1: - raise SystemExit(f"expected one .vhdx from {ref}, got {len(files)}") - out_path.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(files[0]), str(out_path)) - - -def download_image(name: str): - target_map = { - "baremetal": REPO_ROOT / "artifacts" / "baremetal.vhdx", - "core_selinux": REPO_ROOT / "artifacts" / "core_selinux.vhdx", - "minimal": REPO_ROOT / "artifacts" / "minimal.vhdx", - "minimal_aarch64": REPO_ROOT / "artifacts" / "minimal_aarch64.vhdx", - } - if name not in target_map: - raise SystemExit(f"unsupported base image: {name}") - oras_download(name, target_map[name]) - - -def main(): - ap = argparse.ArgumentParser() - sub = ap.add_subparsers(dest="cmd", required=True) - - p = sub.add_parser("list") - p.add_argument("--filter-type", required=True) - - p = sub.add_parser("dependencies") - p.add_argument("name") - - p = sub.add_parser("build") - p.add_argument("name") - p.add_argument("--output-path", required=True) - p.add_argument("--container") - - p = sub.add_parser("show-artifact") - p.add_argument("item") - - p = sub.add_parser("download-image") - p.add_argument("name") - - args = ap.parse_args() - if args.cmd == "list": - legacy_list(args.filter_type) - elif args.cmd == "dependencies": - dependencies(args.name) - elif args.cmd == "build": - build(args.name, args.output_path, args.container) - elif args.cmd == "show-artifact": - if args.item != "customizer-container-full": - raise SystemExit(f"unsupported artifact item: {args.item}") - print(default_container_full()) - elif args.cmd == "download-image": - download_image(args.name) - - -if __name__ == "__main__": - import os - - main() diff --git a/tests/tailor-images/targets.mk b/tests/tailor-images/targets.mk new file mode 100644 index 0000000000..3065afb702 --- /dev/null +++ b/tests/tailor-images/targets.mk @@ -0,0 +1,113 @@ +CARGO_TAILOR ?= $(if $(shell command -v cargo 2>/dev/null),cargo,$(HOME)/.cargo/bin/cargo) +TAILOR_DIR := $(patsubst %/,%,$(dir $(lastword $(MAKEFILE_LIST)))) +TAILOR_WORKSPACE := $(TAILOR_DIR)/tailor.yaml +TAILOR_RPM_DEPS := bin/RPMS $(shell find bin/RPMS -name '*.rpm' 2>/dev/null) +VM_IMAGE_PATH_PREFIX := $(TAILOR_DIR)/trident-vm-testimage/base + +TRIDENT_INSTALLER_DEPS := $(shell find $(TAILOR_DIR)/trident-installer -type f 2>/dev/null) +TRIDENT_TESTIMAGE_DEPS := $(shell find $(TAILOR_DIR)/trident-testimage -type f 2>/dev/null) +TRIDENT_VERITY_TESTIMAGE_DEPS := $(shell find $(TAILOR_DIR)/trident-verity-testimage -type f 2>/dev/null) +TRIDENT_CONTAINER_INSTALLER_DEPS := $(shell find $(TAILOR_DIR)/trident-container-installer -type f 2>/dev/null) +TRIDENT_CONTAINER_TESTIMAGE_DEPS := $(shell find $(TAILOR_DIR)/trident-container-testimage -type f 2>/dev/null) +TRIDENT_FUNCTEST_DEPS := $(shell find $(TAILOR_DIR)/trident-functest -type f 2>/dev/null) +AZL_INSTALLER_TAILOR_DEPS := $(shell find $(AZL_INSTALLER_DIR) -type f 2>/dev/null) +AZURELINUX_DIRECT_STREAMING_DEPS := $(shell find $(TAILOR_DIR)/azurelinux-direct-streaming-testimage -type f 2>/dev/null) +FOREIGN_DIRECT_STREAMING_DEPS := $(shell find $(TAILOR_DIR)/foreign-direct-streaming-testimage -type f 2>/dev/null) +TRIDENT_VM_TESTIMAGE_DEPS := $(shell find $(TAILOR_DIR)/trident-vm-testimage -type f 2>/dev/null) $(VM_IMAGE_PATH_PREFIX)/files/id_rsa.pub + +COSI_TARGETS = \ + trident-testimage \ + trident-testimage-arm64 \ + trident-verity-testimage \ + trident-usrverity-testimage \ + trident-container-verity-testimage \ + trident-container-usrverity-testimage \ + trident-container-testimage \ + azurelinux-direct-streaming-testimage-amd64 \ + azurelinux-direct-streaming-testimage-arm64 \ + trident-vm-grub-testimage \ + trident-vm-grub-verity-testimage \ + trident-vm-root-verity-testimage \ + trident-vm-usr-verity-testimage \ + trident-vm-acl-agent-update-testimage \ + trident-vm-grub-verity-azure-testimage \ + trident-vm-grub-testimage-arm64 \ + trident-vm-grub-verity-testimage-arm64 \ + ubuntu-direct-streaming-testimage-2204-amd64 \ + ubuntu-direct-streaming-testimage-2204-arm64 \ + ubuntu-direct-streaming-testimage-2404-amd64 \ + ubuntu-direct-streaming-testimage-2404-arm64 \ + gb200-direct-streaming-testimage-2404-arm64 + +ISO_TARGETS = \ + trident-installer \ + trident-split-installer \ + trident-installer-arm64 \ + trident-container-installer \ + trident-direct-streaming-installer-amd64 \ + trident-direct-streaming-installer-arm64 \ + azl-installer + +.PHONY: $(COSI_TARGETS) $(ISO_TARGETS) all-cosi all-iso +$(COSI_TARGETS): %: artifacts/%.cosi +$(filter-out azl-installer,$(ISO_TARGETS)): %: artifacts/%.iso +azl-installer: $(ARTIFACTS_TEST_IMAGE_DIR)/azl-installer.iso + +all-cosi: $(COSI_TARGETS) +all-iso: $(ISO_TARGETS) + +define declare_tailor_target +$1: $(TAILOR_WORKSPACE) $4 + @echo "Building '$$(@F)' [$@]" + @echo "Prerequisites:" + @echo "$$^" | tr ' ' '\n' | sed 's/^/ /' + rm -f $(TAILOR_DIR)/artifacts/ca_cert.pem + $(CARGO_TAILOR) run --manifest-path tools/tailor/Cargo.toml --quiet -- --manifest $(TAILOR_WORKSPACE) build $2 $3 --output-dir $(dir $1) + @if [ -f $(TAILOR_DIR)/artifacts/ca_cert.pem ]; then cp $(TAILOR_DIR)/artifacts/ca_cert.pem "$(dir $1)/ca_cert.pem"; fi + @if [ "$1" != "$5" ]; then cp "$5" "$1"; fi + @$6 +endef + +$(eval $(call declare_tailor_target,artifacts/trident-installer.iso,trident-installer,-s arch=amd64 -s variant=default,$(TRIDENT_INSTALLER_DEPS) $(TAILOR_RPM_DEPS) artifacts/baremetal.vhdx,artifacts/trident-installer.iso,:)) +$(eval $(call declare_tailor_target,artifacts/trident-split-installer.iso,trident-installer,-s arch=amd64 -s variant=split,$(TRIDENT_INSTALLER_DEPS) $(TAILOR_RPM_DEPS) artifacts/baremetal.vhdx,artifacts/trident-split-installer.iso,:)) +$(eval $(call declare_tailor_target,artifacts/trident-installer-arm64.iso,trident-installer,-s arch=arm64 -s variant=default,$(TRIDENT_INSTALLER_DEPS) $(TAILOR_RPM_DEPS) artifacts/core_arm64.vhdx,artifacts/trident-installer-arm64.iso,:)) +$(eval $(call declare_tailor_target,artifacts/trident-direct-streaming-installer-amd64.iso,trident-installer,-s arch=amd64 -s variant=direct-streaming,$(TRIDENT_INSTALLER_DEPS) $(TAILOR_RPM_DEPS) artifacts/baremetal.vhdx artifacts/rcp-agent artifacts/rcp-agent.service,artifacts/trident-direct-streaming-installer-amd64.iso,:)) +$(eval $(call declare_tailor_target,artifacts/trident-direct-streaming-installer-arm64.iso,trident-installer,-s arch=arm64 -s variant=direct-streaming,$(TRIDENT_INSTALLER_DEPS) $(TAILOR_RPM_DEPS) artifacts/core_arm64.vhdx artifacts/rcp-agent artifacts/rcp-agent.service,artifacts/trident-direct-streaming-installer-arm64.iso,:)) +$(eval $(call declare_tailor_target,artifacts/trident-container-installer.iso,trident-container-installer,,$(TRIDENT_CONTAINER_INSTALLER_DEPS) artifacts/baremetal.vhdx,artifacts/trident-container-installer.iso,:)) +$(eval $(call declare_tailor_target,artifacts/trident-functest.qcow2,trident-functest,,$(TRIDENT_FUNCTEST_DEPS) artifacts/baremetal.vhdx,artifacts/trident-functest.qcow2,:)) +$(eval $(call declare_tailor_target,artifacts/trident-testimage.cosi,trident-testimage,-s arch=amd64,$(TRIDENT_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/baremetal.vhdx,artifacts/trident-testimage.cosi,:)) +$(eval $(call declare_tailor_target,artifacts/trident-testimage-arm64.cosi,trident-testimage,-s arch=arm64,$(TRIDENT_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/core_arm64.vhdx,artifacts/trident-testimage-arm64.cosi,:)) +$(eval $(call declare_tailor_target,artifacts/trident-verity-testimage.cosi,trident-verity-testimage,-s deployment=host -s mode=root,$(TRIDENT_VERITY_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/baremetal.vhdx,artifacts/trident-verity-testimage.cosi,:)) +$(eval $(call declare_tailor_target,artifacts/trident-usrverity-testimage.cosi,trident-verity-testimage,-s deployment=host -s mode=usr,$(TRIDENT_VERITY_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/baremetal.vhdx,artifacts/trident-usrverity-testimage.cosi,:)) +$(eval $(call declare_tailor_target,artifacts/trident-container-verity-testimage.cosi,trident-verity-testimage,-s deployment=container -s mode=root,$(TRIDENT_VERITY_TESTIMAGE_DEPS) artifacts/baremetal.vhdx,artifacts/trident-container-verity-testimage.cosi,:)) +$(eval $(call declare_tailor_target,artifacts/trident-container-usrverity-testimage.cosi,trident-verity-testimage,-s deployment=container -s mode=usr,$(TRIDENT_VERITY_TESTIMAGE_DEPS) artifacts/baremetal.vhdx,artifacts/trident-container-usrverity-testimage.cosi,:)) +$(eval $(call declare_tailor_target,artifacts/trident-container-testimage.cosi,trident-container-testimage,,$(TRIDENT_CONTAINER_TESTIMAGE_DEPS) artifacts/baremetal.vhdx,artifacts/trident-container-testimage.cosi,:)) +$(eval $(call declare_tailor_target,artifacts/azurelinux-direct-streaming-testimage-amd64.cosi,azurelinux-direct-streaming-testimage,-s arch=amd64,$(AZURELINUX_DIRECT_STREAMING_DEPS) $(TAILOR_RPM_DEPS) artifacts/baremetal.vhdx,artifacts/azurelinux-direct-streaming-testimage-amd64.raw,:)) +$(eval $(call declare_tailor_target,artifacts/azurelinux-direct-streaming-testimage-arm64.cosi,azurelinux-direct-streaming-testimage,-s arch=arm64,$(AZURELINUX_DIRECT_STREAMING_DEPS) $(TAILOR_RPM_DEPS) artifacts/core_arm64.vhdx,artifacts/azurelinux-direct-streaming-testimage-arm64.raw,:)) +$(eval $(call declare_tailor_target,artifacts/ubuntu-direct-streaming-testimage-2204-amd64.cosi,foreign-direct-streaming-testimage,-s arch=amd64 -s source=ubuntu-2204,$(FOREIGN_DIRECT_STREAMING_DEPS) artifacts/ubuntu_2204_amd64.vhdx,artifacts/ubuntu-direct-streaming-testimage-2204-amd64.raw,:)) +$(eval $(call declare_tailor_target,artifacts/ubuntu-direct-streaming-testimage-2204-arm64.cosi,foreign-direct-streaming-testimage,-s arch=arm64 -s source=ubuntu-2204,$(FOREIGN_DIRECT_STREAMING_DEPS) artifacts/ubuntu_2204_arm64.vhdx,artifacts/ubuntu-direct-streaming-testimage-2204-arm64.raw,:)) +$(eval $(call declare_tailor_target,artifacts/ubuntu-direct-streaming-testimage-2404-amd64.cosi,foreign-direct-streaming-testimage,-s arch=amd64 -s source=ubuntu-2404,$(FOREIGN_DIRECT_STREAMING_DEPS) artifacts/ubuntu_2404_amd64.vhdx,artifacts/ubuntu-direct-streaming-testimage-2404-amd64.raw,:)) +$(eval $(call declare_tailor_target,artifacts/ubuntu-direct-streaming-testimage-2404-arm64.cosi,foreign-direct-streaming-testimage,-s arch=arm64 -s source=ubuntu-2404,$(FOREIGN_DIRECT_STREAMING_DEPS) artifacts/ubuntu_2404_arm64.vhdx,artifacts/ubuntu-direct-streaming-testimage-2404-arm64.raw,:)) +$(eval $(call declare_tailor_target,artifacts/gb200-direct-streaming-testimage-2404-arm64.cosi,foreign-direct-streaming-testimage,-s arch=arm64 -s source=gb200-2404,$(FOREIGN_DIRECT_STREAMING_DEPS) artifacts/gb200_2404_arm64.vhdx,artifacts/gb200-direct-streaming-testimage-2404-arm64.raw,:)) +$(eval $(call declare_tailor_target,artifacts/trident-vm-grub-testimage.cosi,trident-vm-testimage,-s arch=amd64 -s scenario=grub,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/qemu_guest.vhdx,artifacts/trident-vm-grub-testimage.cosi,:)) +$(eval $(call declare_tailor_target,artifacts/trident-vm-grub-verity-testimage.cosi,trident-vm-testimage,-s arch=amd64 -s scenario=grub-verity,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/qemu_guest.vhdx,artifacts/trident-vm-grub-verity-testimage.cosi,:)) +$(eval $(call declare_tailor_target,artifacts/trident-vm-grub-verity-testimage.qcow2,trident-vm-testimage,-s arch=amd64 -s scenario=grub-verity,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/qemu_guest.vhdx,artifacts/trident-vm-grub-verity-testimage.qcow2,:)) +$(eval $(call declare_tailor_target,artifacts/trident-vm-root-verity-testimage.cosi,trident-vm-testimage,-s arch=amd64 -s scenario=root-verity,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/qemu_guest.vhdx,artifacts/trident-vm-root-verity-testimage.cosi,:)) +$(eval $(call declare_tailor_target,artifacts/trident-vm-usr-verity-testimage.cosi,trident-vm-testimage,-s arch=amd64 -s scenario=usr-verity,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/qemu_guest.vhdx,artifacts/trident-vm-usr-verity-testimage.cosi,:)) +$(eval $(call declare_tailor_target,artifacts/trident-vm-usr-verity-testimage.qcow2,trident-vm-testimage,-s arch=amd64 -s scenario=usr-verity,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/qemu_guest.vhdx,artifacts/trident-vm-usr-verity-testimage.qcow2,:)) +$(eval $(call declare_tailor_target,artifacts/trident-vm-acl-agent-update-testimage.cosi,trident-vm-testimage,-s arch=amd64 -s scenario=acl-agent,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/qemu_guest.vhdx,artifacts/trident-vm-acl-agent-update-testimage.cosi,:)) +$(eval $(call declare_tailor_target,artifacts/trident-vm-acl-agent-testimage.qcow2,trident-vm-testimage,-s arch=amd64 -s scenario=acl-agent,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/qemu_guest.vhdx,artifacts/trident-vm-acl-agent-testimage.qcow2,:)) +$(eval $(call declare_tailor_target,artifacts/trident-vm-grub-verity-azure-testimage.cosi,trident-vm-testimage,-s arch=amd64 -s scenario=grub-verity-azure,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/core_selinux.vhdx,artifacts/trident-vm-grub-verity-azure-testimage.cosi,:)) +$(eval $(call declare_tailor_target,artifacts/trident-vm-grub-verity-azure-testimage.vhd,trident-vm-testimage,-s arch=amd64 -s scenario=grub-verity-azure,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/core_selinux.vhdx,artifacts/trident-vm-grub-verity-azure-testimage.vhd,:)) +$(eval $(call declare_tailor_target,artifacts/trident-vm-grub-testimage-arm64.cosi,trident-vm-testimage,-s arch=arm64 -s scenario=grub,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/core_arm64.vhdx,artifacts/trident-vm-grub-testimage-arm64.cosi,:)) +$(eval $(call declare_tailor_target,artifacts/trident-vm-grub-verity-testimage-arm64.cosi,trident-vm-testimage,-s arch=arm64 -s scenario=grub-verity,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/core_arm64.vhdx,artifacts/trident-vm-grub-verity-testimage-arm64.cosi,:)) + +$(ARTIFACTS_TEST_IMAGE_DIR)/azl-installer.iso: $(TAILOR_WORKSPACE) $(AZL_INSTALLER_TAILOR_DEPS) $(TAILOR_RPM_DEPS) bin/liveinstaller artifacts/baremetal.vhdx $(ARTIFACTS_TEST_IMAGE_DIR)/regular.cosi + rm -rf $(AZL_INSTALLER_ISO_DIR)/images + mkdir -p $(AZL_INSTALLER_ISO_DIR)/images + cp $(ARTIFACTS_TEST_IMAGE_DIR)/regular.cosi $(AZL_INSTALLER_ISO_DIR)/images/trident-testimage.cosi + rm -rf $(AZL_INSTALLER_ISO_DIR)/bin + mkdir -p $(AZL_INSTALLER_ISO_DIR)/bin + cp bin/liveinstaller $(AZL_INSTALLER_ISO_DIR)/bin/ + rm -f $(TAILOR_DIR)/artifacts/ca_cert.pem + $(CARGO_TAILOR) run --manifest-path tools/tailor/Cargo.toml --quiet -- --manifest $(TAILOR_WORKSPACE) build azl-installer --output-dir $(ARTIFACTS_TEST_IMAGE_DIR) From 558d8d079d7eeaa06faab465e7d9645ab22395ae Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Sun, 11 Oct 2026 00:22:03 +0000 Subject: [PATCH 4/6] fix(pipelines): indent inline python heredoc in build-image-template.yml The dev-MIC-override python heredoc body (patching tests/tailor-images/tailor.yaml to add a local "ic-dev" toolchain entry) was written flush-left at column 0 inside a `bash: |` block literal. YAML only strips the common leading indentation established by the block's first line; a line with less indentation than that (here 0 vs the required 8 spaces) ends the block scalar early and gets parsed as a new top-level mapping key, producing: While scanning a simple key, could not find expected ':' This exact bug was already found and fixed once before in this same file (see 6c8a6ec0 "fix(pipelines): indent inline python heredocs in build-image-template.yml") for a since-removed heredoc; this round's Makefile/legacy_targets.py elimination added a new heredoc with the same mistake. Indent the heredoc body and its `PY` terminator to match the block's indentation - YAML strips the shared indent back off before bash/python ever see the text, so the resulting script is unchanged. Verified: every file under .pipelines/**/*.yml (85 files) parses as valid YAML, confirmed via `ado-pipelines_write` (action=run_pipeline, previewRun=true), which exercises ADO's own real template-evaluation path, not just a generic YAML parser. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../stages/build_image/build-image-template.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pipelines/templates/stages/build_image/build-image-template.yml b/.pipelines/templates/stages/build_image/build-image-template.yml index 229624efae..ec73f531fc 100644 --- a/.pipelines/templates/stages/build_image/build-image-template.yml +++ b/.pipelines/templates/stages/build_image/build-image-template.yml @@ -292,12 +292,12 @@ steps: if [[ "${{ parameters.micBuildType }}" == "dev" ]]; then manifest_path=tests/tailor-images/.tailor.pipeline.yaml python3 - <<'PY' -from pathlib import Path -text = Path('tests/tailor-images/tailor.yaml').read_text() -needle = 'toolchains:\n default: ic\n entries:\n - name: ic\n container: mcr.microsoft.com/azurelinux/imagecustomizer\n tag: latest\n' -replacement = 'toolchains:\n default: ic-dev\n entries:\n - name: ic\n container: mcr.microsoft.com/azurelinux/imagecustomizer\n tag: latest\n - name: ic-dev\n container: imagecustomizer\n tag: dev\n pull: never\n' -Path('tests/tailor-images/.tailor.pipeline.yaml').write_text(text.replace(needle, replacement, 1)) -PY + from pathlib import Path + text = Path('tests/tailor-images/tailor.yaml').read_text() + needle = 'toolchains:\n default: ic\n entries:\n - name: ic\n container: mcr.microsoft.com/azurelinux/imagecustomizer\n tag: latest\n' + replacement = 'toolchains:\n default: ic-dev\n entries:\n - name: ic\n container: mcr.microsoft.com/azurelinux/imagecustomizer\n tag: latest\n - name: ic-dev\n container: imagecustomizer\n tag: dev\n pull: never\n' + Path('tests/tailor-images/.tailor.pipeline.yaml').write_text(text.replace(needle, replacement, 1)) + PY fi rm -f tests/tailor-images/artifacts/ca_cert.pem From 082a674e90e183cb74ea7c3ff9e84c9e00c42092 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Sun, 11 Oct 2026 00:41:23 +0000 Subject: [PATCH 5/6] fix(pipelines): heredoc terminator must match the bash block's own base indent, not its nesting level My previous fix (fc6b6c38) indented the dev-MIC-override python heredoc's body and `PY` terminator to 8 spaces, matching the `python3 - <<'PY'` line itself - but that line sits one level deeper (inside an `if` block) than the overall `bash: |` block's own first line (`set -euo pipefail`, at 6 spaces). YAML's block-scalar dedent strips a flat amount equal to the *block's* base indentation from every line, not each line's own relative indentation - so after stripping 6 spaces, the heredoc terminator still carried 2 residual leading spaces (" PY"), and bash's `<<'PY'` (unquoted, non-dash) requires an exact, whitespace-sensitive match at the start of the line. This produced, at actual pipeline runtime (not caught by a generic YAML parse, which doesn't care about this): "warning: here-document ... delimited by end-of-file" followed by "syntax error: unexpected end of file" - confirmed from a real failed build (1221778, job "Build trident-container-installer"). Fix: indent the heredoc body/terminator to the block's base level (6 spaces) instead of the `if`-block's nesting level (8 spaces), so they end up flush-left after YAML's dedent, exactly like the original working instance of this same pattern (6c8a6ec0, which had no extra nesting to get wrong). Verified this time by actually replicating ADO's own dedent behavior (strip the block's first-line indentation from every line) and running `bash -n` on the result, not just a generic YAML parse - this is the fix that was missing from fc6b6c38's verification. Also wrote a one-off check script that does the same dedent+`bash -n` extraction for every bash/script block the migration actually touched (Makefile, legacy_targets.py already deleted, build-image-template.yml, and the other `trident_images`/`azl_installer` templates) - all clean. Full `.pipelines/**/*.yml` (85 files) still parses as valid YAML. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../stages/build_image/build-image-template.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pipelines/templates/stages/build_image/build-image-template.yml b/.pipelines/templates/stages/build_image/build-image-template.yml index ec73f531fc..249fa5d30d 100644 --- a/.pipelines/templates/stages/build_image/build-image-template.yml +++ b/.pipelines/templates/stages/build_image/build-image-template.yml @@ -292,12 +292,12 @@ steps: if [[ "${{ parameters.micBuildType }}" == "dev" ]]; then manifest_path=tests/tailor-images/.tailor.pipeline.yaml python3 - <<'PY' - from pathlib import Path - text = Path('tests/tailor-images/tailor.yaml').read_text() - needle = 'toolchains:\n default: ic\n entries:\n - name: ic\n container: mcr.microsoft.com/azurelinux/imagecustomizer\n tag: latest\n' - replacement = 'toolchains:\n default: ic-dev\n entries:\n - name: ic\n container: mcr.microsoft.com/azurelinux/imagecustomizer\n tag: latest\n - name: ic-dev\n container: imagecustomizer\n tag: dev\n pull: never\n' - Path('tests/tailor-images/.tailor.pipeline.yaml').write_text(text.replace(needle, replacement, 1)) - PY + from pathlib import Path + text = Path('tests/tailor-images/tailor.yaml').read_text() + needle = 'toolchains:\n default: ic\n entries:\n - name: ic\n container: mcr.microsoft.com/azurelinux/imagecustomizer\n tag: latest\n' + replacement = 'toolchains:\n default: ic-dev\n entries:\n - name: ic\n container: mcr.microsoft.com/azurelinux/imagecustomizer\n tag: latest\n - name: ic-dev\n container: imagecustomizer\n tag: dev\n pull: never\n' + Path('tests/tailor-images/.tailor.pipeline.yaml').write_text(text.replace(needle, replacement, 1)) + PY fi rm -f tests/tailor-images/artifacts/ca_cert.pem From 633b2eb6b6a14a8018af31c3ad15e4a08bddea6c Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Sun, 11 Oct 2026 22:04:49 +0000 Subject: [PATCH 6/6] fix: map qemu grub-verity qcow2 to base layout Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/tailor-images/targets.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tailor-images/targets.mk b/tests/tailor-images/targets.mk index 3065afb702..f7e3ea708c 100644 --- a/tests/tailor-images/targets.mk +++ b/tests/tailor-images/targets.mk @@ -91,7 +91,7 @@ $(eval $(call declare_tailor_target,artifacts/ubuntu-direct-streaming-testimage- $(eval $(call declare_tailor_target,artifacts/gb200-direct-streaming-testimage-2404-arm64.cosi,foreign-direct-streaming-testimage,-s arch=arm64 -s source=gb200-2404,$(FOREIGN_DIRECT_STREAMING_DEPS) artifacts/gb200_2404_arm64.vhdx,artifacts/gb200-direct-streaming-testimage-2404-arm64.raw,:)) $(eval $(call declare_tailor_target,artifacts/trident-vm-grub-testimage.cosi,trident-vm-testimage,-s arch=amd64 -s scenario=grub,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/qemu_guest.vhdx,artifacts/trident-vm-grub-testimage.cosi,:)) $(eval $(call declare_tailor_target,artifacts/trident-vm-grub-verity-testimage.cosi,trident-vm-testimage,-s arch=amd64 -s scenario=grub-verity,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/qemu_guest.vhdx,artifacts/trident-vm-grub-verity-testimage.cosi,:)) -$(eval $(call declare_tailor_target,artifacts/trident-vm-grub-verity-testimage.qcow2,trident-vm-testimage,-s arch=amd64 -s scenario=grub-verity,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/qemu_guest.vhdx,artifacts/trident-vm-grub-verity-testimage.qcow2,:)) +$(eval $(call declare_tailor_target,artifacts/trident-vm-grub-verity-testimage.qcow2,trident-vm-testimage,-s arch=amd64 -s scenario=grub-verity-base,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/qemu_guest.vhdx,artifacts/trident-vm-grub-verity-testimage.qcow2,:)) $(eval $(call declare_tailor_target,artifacts/trident-vm-root-verity-testimage.cosi,trident-vm-testimage,-s arch=amd64 -s scenario=root-verity,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/qemu_guest.vhdx,artifacts/trident-vm-root-verity-testimage.cosi,:)) $(eval $(call declare_tailor_target,artifacts/trident-vm-usr-verity-testimage.cosi,trident-vm-testimage,-s arch=amd64 -s scenario=usr-verity,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/qemu_guest.vhdx,artifacts/trident-vm-usr-verity-testimage.cosi,:)) $(eval $(call declare_tailor_target,artifacts/trident-vm-usr-verity-testimage.qcow2,trident-vm-testimage,-s arch=amd64 -s scenario=usr-verity,$(TRIDENT_VM_TESTIMAGE_DEPS) $(TAILOR_RPM_DEPS) artifacts/qemu_guest.vhdx,artifacts/trident-vm-usr-verity-testimage.qcow2,:))