diff --git a/.pipelines/templates/stages/build_image/build-image-template.yml b/.pipelines/templates/stages/build_image/build-image-template.yml index 75b3521d6b..68a3688e24 100644 --- a/.pipelines/templates/stages/build_image/build-image-template.yml +++ b/.pipelines/templates/stages/build_image/build-image-template.yml @@ -62,6 +62,10 @@ parameters: steps: - template: ../common_tasks/avoid-pypi-usage.yml + - template: ../common_tasks/cargo-auth.yml + parameters: + cargoConfigPath: $(TRIDENT_SOURCE_DIR)/.cargo/config.toml + - template: common/sfi-enforce-isolation-with-etc-hosts.yaml@platform-pipelines - script: | @@ -82,8 +86,17 @@ steps: retryCountOnTaskFailure: 3 - bash: | - ./tests/images/testimages.py show-image ${{ parameters.imageName }} base-image --devops-var baseImageType - displayName: "Get Image Info" + set -euo pipefail + python3 - <<'PY' "${{ parameters.imageName }}" + import json + import sys + from pathlib import Path + + legacy = json.loads(Path("tests/images/legacy-map.json").read_text()) + entry = legacy[sys.argv[1]] + print(f"##vso[task.setvariable variable=baseImageType]{entry['baseImage']}") + PY + displayName: "Resolve image metadata" workingDirectory: ${{ parameters.tridentSourceDirectory }} - bash: | @@ -117,10 +130,27 @@ steps: azureLinuxVersion: ${{ parameters.azureLinuxVersion }} runtimeBuildType: ${{ parameters.baseimgBuildType }} + - ${{ if eq(parameters.architecture, 'arm64') }}: + - task: DownloadPipelineArtifact@2 + displayName: "Download go-tools for direct-streaming installer" + condition: or(eq('${{ parameters.imageName }}', 'trident-direct-streaming-installer-amd64'), eq('${{ parameters.imageName }}', 'trident-direct-streaming-installer-arm64')) + inputs: + buildType: current + artifactName: go-tools-arm64 + targetPath: "$(Build.ArtifactStagingDirectory)/go-tools" + - ${{ if ne(parameters.architecture, 'arm64') }}: + - task: DownloadPipelineArtifact@2 + displayName: "Download go-tools for direct-streaming installer" + condition: or(eq('${{ parameters.imageName }}', 'trident-direct-streaming-installer-amd64'), eq('${{ parameters.imageName }}', 'trident-direct-streaming-installer-arm64')) + inputs: + buildType: current + artifactName: go-tools + targetPath: "$(Build.ArtifactStagingDirectory)/go-tools" + - bash: | set -ex - # Move base VHDX to artifacts/ (builder expects artifacts/*.vhdx) + # Move base VHDX to artifacts/ (tailor baseImages catalogue points at artifacts/*.vhdx) mkdir -p artifacts if ls "$(Build.ArtifactStagingDirectory)/images" | grep -q ".*\.vhdx$"; then mv $(Build.ArtifactStagingDirectory)/images/*.vhdx artifacts/ @@ -144,28 +174,114 @@ steps: DISTRO=azl4 fi - # Move Trident RPMs to bin/RPMS/ (builder expects bin/RPMS/*.rpm) + # Move Trident RPMs to bin/RPMS/ (tailor rpmSources point at bin/RPMS/) if [ -d "$(Build.ArtifactStagingDirectory)/trident" ]; then mkdir -p bin/RPMS find "$(Build.ArtifactStagingDirectory)/trident" -name "*${DISTRO}*.rpm" -exec mv {} bin/RPMS/ \; rm -rf "$(Build.ArtifactStagingDirectory)/trident" fi + + if [[ "${{ parameters.imageName }}" == trident-direct-streaming-installer-* ]]; then + if [ -f "$(Build.ArtifactStagingDirectory)/go-tools/rcp-agent" ]; then + cp "$(Build.ArtifactStagingDirectory)/go-tools/rcp-agent" artifacts/rcp-agent + elif [ -f bin/rcp-agent ]; then + cp bin/rcp-agent artifacts/rcp-agent + else + echo "Missing rcp-agent for direct-streaming installer" + exit 1 + fi + chmod +x artifacts/rcp-agent + cp tools/cmd/rcp-agent/rcp-agent.service artifacts/rcp-agent.service + fi displayName: "Prepare and move requirements" workingDirectory: ${{ parameters.tridentSourceDirectory }} - bash: | - set -ex + set -euo pipefail + + legacy_name='${{ parameters.imageName }}' + export PATH="$HOME/.cargo/bin:$PATH" + export CLICOLOR_FORCE=1 - EXTRA_ARGS="" + readarray -t selector_lines < <(python3 - <<'PY' "$legacy_name" + import json + import sys + from pathlib import Path + entry = json.loads(Path("tests/images/legacy-map.json").read_text())[sys.argv[1]] + print(entry["image"]) + print(entry["ext"]) + print(entry.get("tailorExt", entry["ext"])) + for key, value in entry["selectors"].items(): + print(f"{key}={value}") + PY + ) + + tailor_image="${selector_lines[0]}" + artifact_ext="${selector_lines[1]}" + source_ext="${selector_lines[2]}" + selector_args=() + if (( ${#selector_lines[@]} > 3 )); then + for selector in "${selector_lines[@]:3}"; do + selector_args+=("-s" "$selector") + done + fi + + manifest_path=tests/images/tailor.yaml if [[ "${{ parameters.micBuildType }}" == "dev" ]]; then - EXTRA_ARGS+=" --container imagecustomizer:dev" + manifest_path=tests/images/.tailor.pipeline.yaml + python3 - <<'PY' + from pathlib import Path + text = Path("tests/images/tailor.yaml").read_text() + needle = """toolchains: + default: ic + entries: + - name: ic + container: mcr.microsoft.com/azurelinux/imagecustomizer + tag: latest + """ + replacement = """toolchains: + default: ic-dev + entries: + - name: ic + container: mcr.microsoft.com/azurelinux/imagecustomizer + tag: latest + - name: ic-dev + container: imagecustomizer + tag: dev + pull: never + """ + Path("tests/images/.tailor.pipeline.yaml").write_text(text.replace(needle, replacement, 1)) + PY + fi + + matrix_json=$(cargo run --manifest-path tools/tailor/Cargo.toml --quiet -- --manifest "$manifest_path" matrix "$tailor_image" --format json "${selector_args[@]}") + + slug=$(python3 - <<'PY' "$matrix_json" + import json + import sys + cells = json.loads(sys.argv[1]) + if len(cells) != 1: + raise SystemExit(f"expected exactly one cell, got {len(cells)}") + print(cells[0]["slug"]) + PY + ) + + rm -f tests/images/artifacts/ca_cert.pem + cargo run --manifest-path tools/tailor/Cargo.toml --quiet -- --manifest "$manifest_path" build "$tailor_image" "${selector_args[@]}" --output-dir $(ob_outputDirectory) --clones ${{ parameters.clones }} + + if [[ ${{ parameters.clones }} -eq 1 ]]; then + mv "$(ob_outputDirectory)/${slug}.${source_ext}" "$(ob_outputDirectory)/${legacy_name}.${artifact_ext}" + else + for ((i=0; i<${{ parameters.clones }}; i++)); do + mv "$(ob_outputDirectory)/${slug}_clone${i}.${source_ext}" "$(ob_outputDirectory)/${legacy_name}_${i}.${artifact_ext}" + done + fi + + if [ -f tests/images/artifacts/ca_cert.pem ]; then + cp tests/images/artifacts/ca_cert.pem "$(ob_outputDirectory)/ca_cert.pem" fi - python3 ./tests/images/testimages.py build \ - "${{ parameters.imageName }}" $EXTRA_ARGS \ - --output-dir $(ob_outputDirectory) \ - --no-download \ - --clones ${{ parameters.clones }} + rm -f tests/images/.tailor.pipeline.yaml displayName: "Build ${{ parameters.imageName }}" workingDirectory: ${{ parameters.tridentSourceDirectory }} diff --git a/.pipelines/templates/stages/trident_images/build-image-arm64.yml b/.pipelines/templates/stages/trident_images/build-image-arm64.yml index 04be97e2f9..abc4ab9d3e 100644 --- a/.pipelines/templates/stages/trident_images/build-image-arm64.yml +++ b/.pipelines/templates/stages/trident_images/build-image-arm64.yml @@ -52,6 +52,15 @@ jobs: - template: ../common_tasks/checkout_trident.yml - template: ../common_tasks/avoid-pypi-usage.yml + # The Makefile's tailor-backed legacy image targets invoke + # `cargo run --manifest-path tools/tailor/Cargo.toml`, so a Rust + # toolchain and cargo registry auth are required here even though this + # job previously never needed cargo. + - template: ../common_tasks/rustup.yml + - template: ../common_tasks/cargo-auth.yml + parameters: + cargoConfigPath: $(TRIDENT_SOURCE_DIR)/.cargo/config.toml + - task: DownloadPipelineArtifact@2 inputs: buildType: current diff --git a/.pipelines/templates/stages/trident_images/build-image.yml b/.pipelines/templates/stages/trident_images/build-image.yml index 6f70dafdce..c50648e7c4 100644 --- a/.pipelines/templates/stages/trident_images/build-image.yml +++ b/.pipelines/templates/stages/trident_images/build-image.yml @@ -101,6 +101,15 @@ jobs: - template: ../common_tasks/checkout_trident.yml - template: ../common_tasks/avoid-pypi-usage.yml + # The Makefile's tailor-backed legacy image targets invoke + # `cargo run --manifest-path tools/tailor/Cargo.toml`, so a Rust + # toolchain and cargo registry auth are required here even though this + # job previously never needed cargo. + - template: ../common_tasks/rustup.yml + - template: ../common_tasks/cargo-auth.yml + parameters: + cargoConfigPath: $(TRIDENT_SOURCE_DIR)/.cargo/config.toml + - ${{ if parameters.downloadGoTools }}: - task: DownloadPipelineArtifact@2 displayName: "Download $(goToolsArtifactName)" diff --git a/.pipelines/templates/stages/trident_images/trident-testimg-template.yml b/.pipelines/templates/stages/trident_images/trident-testimg-template.yml index c2d53a4694..ced262a44c 100644 --- a/.pipelines/templates/stages/trident_images/trident-testimg-template.yml +++ b/.pipelines/templates/stages/trident_images/trident-testimg-template.yml @@ -135,7 +135,7 @@ steps: displayName: Copy SSH Keys - script: | - echo "##[warning]THE PIPELINE TEMPLATE trident-testimg-template.yaml IS DEPRECATED. PLEASE SWITCH TO USING testimages.py TO BUILD TEST IMAGES." + echo "##[warning]THE PIPELINE TEMPLATE trident-testimg-template.yaml IS DEPRECATED. PLEASE SWITCH TO USING the tailor workspace under tests/images." cat /etc/os-release displayName: "Report Host Info" @@ -264,6 +264,18 @@ steps: rm -rf "$(Build.ArtifactStagingDirectory)/trident" fi + if [[ "${{ parameters.target }}" == artifacts/trident-direct-streaming-installer-* ]]; then + mkdir -p '${{ parameters.tridentSourceDirectory }}/artifacts' + if [ -f "${{ parameters.tridentSourceDirectory }}/bin/rcp-agent" ]; then + cp "${{ parameters.tridentSourceDirectory }}/bin/rcp-agent" '${{ parameters.tridentSourceDirectory }}/artifacts/rcp-agent' + chmod +x '${{ parameters.tridentSourceDirectory }}/artifacts/rcp-agent' + else + echo "Missing rcp-agent in bin/ for direct-streaming installer" + exit 1 + fi + cp '${{ parameters.tridentSourceDirectory }}/tools/cmd/rcp-agent/rcp-agent.service' '${{ parameters.tridentSourceDirectory }}/artifacts/rcp-agent.service' + fi + workingDirectory: ${{ parameters.tridentSourceDirectory }} displayName: "Prepare and move requirements" @@ -280,13 +292,12 @@ steps: export MIC_CONTAINER_IMAGE="imagecustomizer:dev" fi - # Allow cross-platform (i.e. amd64 pipeline creating arm64 images) - if [[ "${{ parameters.micArchitecture }}" != "${{ parameters.pipelineArchitecture }}" ]]; then - # Export variable to tell Makefile and testimages to use `docker --platform` - export MIC_ARCHITECTURE="linux/${{ parameters.micArchitecture }}" - fi + # The Makefile's tailor-backed legacy target helper resolves the cell arch + # from the historical image name, so no extra MIC_ARCHITECTURE export is + # needed here for cross-arch direct-streaming builds. # Compress the full image & delete the uncompressed image + rm -f tests/images/artifacts/ca_cert.pem make ${{ parameters.target }} workingDirectory: ${{ parameters.tridentSourceDirectory }} displayName: "Build Test Image" @@ -302,5 +313,9 @@ steps: # Everything else is a file sudo mv -v "${{ parameters.target }}" "${{ parameters.outputDirectory }}/" fi + + if [ -f "tests/images/artifacts/ca_cert.pem" ]; then + sudo cp -v "tests/images/artifacts/ca_cert.pem" "${{ parameters.outputDirectory }}/ca_cert.pem" + fi workingDirectory: ${{ parameters.tridentSourceDirectory }} displayName: "Copy artifacts to output directory" diff --git a/Makefile b/Makefile index 28ec9dc5d7..3df8e54533 100644 --- a/Makefile +++ b/Makefile @@ -651,7 +651,7 @@ $(ARTIFACTS_TEST_IMAGE_DIR)/azl-installer.iso: \ mkdir -p $(AZL_INSTALLER_ISO_DIR)/bin cp bin/liveinstaller $(AZL_INSTALLER_ISO_DIR)/bin/ # Build ISO - ./tests/images/testimages.py build azl-installer --output-dir $(ARTIFACTS_TEST_IMAGE_DIR) + python3 ./tests/images/legacy_targets.py build azl-installer --output-path $(ARTIFACTS_TEST_IMAGE_DIR)/azl-installer.iso $(if $(strip $(MIC_CONTAINER_IMAGE)),--container $(MIC_CONTAINER_IMAGE)) .PHONY: validate validate: $(TRIDENT_CONFIG) target/release/trident @@ -885,9 +885,15 @@ download-trident-container-installer-iso: --path artifacts/ \ --artifact-name 'trident-container-installer' -artifacts/trident-container-installer.iso: - $(MAKE) download-trident-container-installer-iso; \ - ls -l artifacts/trident-container-installer.iso +artifacts/trident-container-installer.iso: $(shell python3 ./tests/images/legacy_targets.py dependencies trident-container-installer) + @echo "Building 'trident-container-installer' [$@] from $<" + @echo "Prerequisites:" + @echo "$^" | tr ' ' '\n' | sed 's/^/ /' + @echo "Building image..." + python3 ./tests/images/legacy_targets.py build \ + trident-container-installer \ + --output-path $@ \ + $(if $(strip $(MIC_CONTAINER_IMAGE)),--container $(MIC_CONTAINER_IMAGE)) # Copies locally built runtime images from ../test-images/build to ./artifacts/test-image. # Expects that both the regular and verity Trident test images have been built. @@ -1015,7 +1021,7 @@ validate-pipeline-website-artifact: # # Generic COSI image build target pattern # -COSI_TARGETS = $(shell ./tests/images/testimages.py list --filter-type cosi) +COSI_TARGETS = $(shell python3 ./tests/images/legacy_targets.py list --filter-type cosi) .PHONY: $(COSI_TARGETS) $(COSI_TARGETS): %: artifacts/%.cosi @@ -1026,7 +1032,7 @@ all-cosi: $(COSI_TARGETS) # # Generic ISO image build target pattern # -ISO_TARGETS = $(shell ./tests/images/testimages.py list --filter-type iso) +ISO_TARGETS = $(shell python3 ./tests/images/legacy_targets.py list --filter-type iso) .PHONY: $(ISO_TARGETS) $(ISO_TARGETS): %: artifacts/%.iso @@ -1038,26 +1044,25 @@ all-iso: $(ISO_TARGETS) # prerequisites so that we can use find to get all the files in the directory. # https://www.gnu.org/software/make/manual/make.html#Secondary-Expansion .SECONDEXPANSION: -artifacts/%.cosi artifacts/%.iso artifacts/%.vhdx: $$(shell ./tests/images/testimages.py dependencies $$*) +artifacts/%.cosi artifacts/%.iso artifacts/%.vhdx: $$(shell python3 ./tests/images/legacy_targets.py dependencies $$*) @echo "Building '$*' [$@] from $<" @echo "Prerequisites:" @echo "$^" | tr ' ' '\n' | sed 's/^/ /' @echo "Building image..." - sudo ./tests/images/testimages.py build \ + python3 ./tests/images/legacy_targets.py build \ $* \ - --output-dir ./artifacts \ - $(if $(strip $(MIC_CONTAINER_IMAGE)),--container $(MIC_CONTAINER_IMAGE)) \ - $(if $(strip $(MIC_ARCHITECTURE)),--image-architecture $(MIC_ARCHITECTURE)) + --output-path $@ \ + $(if $(strip $(MIC_CONTAINER_IMAGE)),--container $(MIC_CONTAINER_IMAGE)) -MIC_CONTAINER_IMAGE ?= $(shell ./tests/images/testimages.py show-artifact customizer-container-full) -artifacts/trident-functest.qcow2: $$(shell ./tests/images/testimages.py dependencies $$(basename $$(notdir $$@))) +MIC_CONTAINER_IMAGE ?= $(shell python3 ./tests/images/legacy_targets.py show-artifact customizer-container-full) +artifacts/trident-functest.qcow2: $$(shell python3 ./tests/images/legacy_targets.py dependencies $$(basename $$(notdir $$@))) @echo "Building '$*' [$@] from $<" @echo "Prerequisites:" @echo "$^" | tr ' ' '\n' | sed 's/^/ /' @echo "Building image..." - sudo ./tests/images/testimages.py build \ + python3 ./tests/images/legacy_targets.py build \ $(basename $(notdir $@)) \ - --output-dir ./artifacts \ + --output-path $@ \ $(if $(strip $(MIC_CONTAINER_IMAGE)),--container $(MIC_CONTAINER_IMAGE)) # TRIDENT VM UPDATE IMAGES @@ -1097,23 +1102,38 @@ $(QEMU_GUEST_IMAGE): BAREMETAL_IMAGE = artifacts/baremetal.vhdx $(BAREMETAL_IMAGE): @mkdir -p artifacts - @tests/images/testimages.py download-image baremetal + @python3 ./tests/images/legacy_targets.py download-image baremetal CORE_SELINUX_IMAGE = artifacts/core_selinux.vhdx $(CORE_SELINUX_IMAGE): @mkdir -p artifacts - @tests/images/testimages.py download-image core_selinux + @python3 ./tests/images/legacy_targets.py download-image core_selinux +CORE_ARM64_IMAGE_NAME ?= core_vhdx-arm64-3.0-stable +CORE_ARM64_IMAGE_VERSION ?= * +CORE_ARM64_IMAGE = artifacts/core_arm64.vhdx +$(CORE_ARM64_IMAGE): + @mkdir -p artifacts + @tempdir=$$(mktemp -d); result=$$(az artifacts universal download --organization "https://dev.azure.com/mariner-org/" --project "36d030d6-1d99-4ebd-878b-09af1f4f722f" --scope project --feed "AzureLinuxArtifacts" --name '$(CORE_ARM64_IMAGE_NAME)' --version '$(CORE_ARM64_IMAGE_VERSION)' --path $$tempdir) && mv $$tempdir/*.vhdx $(CORE_ARM64_IMAGE) && rm -rf $$tempdir && echo $$result | jq > $(CORE_ARM64_IMAGE).metadata.json + +artifacts/rcp-agent: bin/rcp-agent + @mkdir -p artifacts + @cp bin/rcp-agent $@ + @chmod +x $@ + +artifacts/rcp-agent.service: tools/cmd/rcp-agent/rcp-agent.service + @mkdir -p artifacts + @cp tools/cmd/rcp-agent/rcp-agent.service $@ MINIMAL_IMAGE = artifacts/minimal.vhdx $(MINIMAL_IMAGE): @mkdir -p artifacts - @tests/images/testimages.py download-image minimal + @python3 ./tests/images/legacy_targets.py download-image minimal MINIMAL_IMAGE_AARCH64 = artifacts/minimal_aarch64.vhdx $(MINIMAL_IMAGE_AARCH64): @mkdir -p artifacts - @tests/images/testimages.py download-image minimal_aarch64 + @python3 ./tests/images/legacy_targets.py download-image minimal_aarch64 artifacts/trident-vm-grub-testimage.qcow2: \ $(QEMU_GUEST_IMAGE) \ @@ -1137,7 +1157,7 @@ artifacts/trident-vm-grub-testimage.qcow2: \ --config-file /repo/$(VM_IMAGE_PATH_PREFIX)/baseimg-grub.yaml artifacts/trident-vm-grub-testimage-arm64.qcow2: \ - base/core_arm64.vhdx \ + $(CORE_ARM64_IMAGE) \ $(TRIDENT_VM_DEPENDENCIES) \ $(VM_IMAGE_PATH_PREFIX)/baseimg-grub.yaml \ $(VM_IMAGE_PATH_PREFIX)/files/id_rsa.pub @@ -1200,7 +1220,7 @@ artifacts/trident-vm-root-verity-testimage.qcow2: \ --config-file /repo/$(VM_IMAGE_PATH_PREFIX)/baseimg-root-verity.yaml artifacts/trident-vm-verity-testimage-arm64.qcow2: \ - base/core_arm64.vhdx \ + $(CORE_ARM64_IMAGE) \ $(TRIDENT_VM_DEPENDENCIES) \ $(VM_IMAGE_PATH_PREFIX)/baseimg-verity.yaml \ $(VM_IMAGE_PATH_PREFIX)/files/etc-mount.service \ @@ -1315,3 +1335,18 @@ artifacts/ubuntu_2404_arm64.vhdx: curl -LO https://cloud-images.ubuntu.com/releases/server/24.04/release/ubuntu-24.04-server-cloudimg-arm64.img qemu-img convert -O vhdx ubuntu-24.04-server-cloudimg-arm64.img artifacts/ubuntu_2404_arm64.vhdx rm -f ubuntu-24.04-server-cloudimg-arm64.img + +# GB200 base image is staged from a private Azure Linux storage account (see +# .pipelines/templates/stages/trident_images/trident-testimg-template.yml, +# "Download Base Image - gb200_2404_arm64"), not a public URL, so this rule +# mirrors the pipeline's az CLI download for local/dev builds that have +# access to the azlinuxbmpstaging storage account. +GB200_BLOB_NAME = gb200/arm64/20260318/image.vhdx +artifacts/gb200_2404_arm64.vhdx: + mkdir -p artifacts + az storage blob download \ + --auth-mode login \ + --account-name azlinuxbmpstaging \ + --container-name os-image-cache \ + --name "$(GB200_BLOB_NAME)" \ + --file artifacts/gb200_2404_arm64.vhdx diff --git a/docs/Development/Testing/E2E-Tests.md b/docs/Development/Testing/E2E-Tests.md index 13169b5e42..0ec172c2a8 100644 --- a/docs/Development/Testing/E2E-Tests.md +++ b/docs/Development/Testing/E2E-Tests.md @@ -132,8 +132,8 @@ make bin/rcp-agent # Remote control plane agent ### 4. Build Trident RPMs The **host** test images include Trident packages built from your local tree. -This step builds the RPMs into `bin/RPMS/`, which `testimages.py` passes to -Image Customizer via `--rpm-source`: +This step builds the RPMs into `bin/RPMS/`, which the `tests/images` tailor +workspace passes to Image Customizer via `rpmSources:`: ```bash make bin/trident-rpms.tar.gz @@ -168,8 +168,7 @@ make artifacts/id_rsa ### 7. Download Base Image ```bash -# Downloads baremetal.vhdx from MCR -./tests/images/testimages.py download-image baremetal +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml bases download baremetal ``` ### 8. Build COSI Images @@ -184,29 +183,27 @@ then rename them into `artifacts/test-image/`. ```bash mkdir -p artifacts/test-image -# Build two clones (produces trident-testimage_0.cosi and trident-testimage_1.cosi) -sudo ./tests/images/testimages.py build trident-testimage \ - --output-dir ./artifacts/test-image --clones 2 +# Build two clones (produces trident-testimage_amd64_cosi_clone0.cosi and +# trident-testimage_amd64_cosi_clone1.cosi) +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml build trident-testimage -s arch=amd64 --output-dir ./artifacts/test-image --clones 2 # Rename clones to the filenames referenced by Host Configurations -mv artifacts/test-image/trident-testimage_0.cosi artifacts/test-image/regular.cosi -mv artifacts/test-image/trident-testimage_1.cosi artifacts/test-image/regular_v2.cosi +mv artifacts/test-image/trident-testimage_amd64_cosi_clone0.cosi artifacts/test-image/regular.cosi +mv artifacts/test-image/trident-testimage_amd64_cosi_clone1.cosi artifacts/test-image/regular_v2.cosi ``` Repeat for other image types as needed: ```bash # Verity image (for root-verity configuration) -sudo ./tests/images/testimages.py build trident-verity-testimage \ - --output-dir ./artifacts/test-image --clones 2 -mv artifacts/test-image/trident-verity-testimage_0.cosi artifacts/test-image/verity.cosi -mv artifacts/test-image/trident-verity-testimage_1.cosi artifacts/test-image/verity_v2.cosi +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml build trident-verity-testimage -s deployment=host -s mode=root --output-dir ./artifacts/test-image --clones 2 +mv artifacts/test-image/trident-verity-testimage_host_root_amd64_cosi_clone0.cosi artifacts/test-image/verity.cosi +mv artifacts/test-image/trident-verity-testimage_host_root_amd64_cosi_clone1.cosi artifacts/test-image/verity_v2.cosi # UKI/usr-verity image (for usr-verity, combined configurations) -sudo ./tests/images/testimages.py build trident-usrverity-testimage \ - --output-dir ./artifacts/test-image --clones 2 -mv artifacts/test-image/trident-usrverity-testimage_0.cosi artifacts/test-image/usrverity.cosi -mv artifacts/test-image/trident-usrverity-testimage_1.cosi artifacts/test-image/usrverity_v2.cosi +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml build trident-verity-testimage -s deployment=host -s mode=usr --output-dir ./artifacts/test-image --clones 2 +mv artifacts/test-image/trident-verity-testimage_host_usr_amd64_cosi_clone0.cosi artifacts/test-image/usrverity.cosi +mv artifacts/test-image/trident-verity-testimage_host_usr_amd64_cosi_clone1.cosi artifacts/test-image/usrverity_v2.cosi ``` #### Container Runtime Images @@ -220,26 +217,23 @@ URLs work unchanged: mkdir -p artifacts/test-image # Regular container image -sudo ./tests/images/testimages.py build trident-container-testimage \ - --output-dir ./artifacts/test-image --clones 2 -mv artifacts/test-image/trident-container-testimage_0.cosi artifacts/test-image/regular.cosi -mv artifacts/test-image/trident-container-testimage_1.cosi artifacts/test-image/regular_v2.cosi +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml build trident-container-testimage --output-dir ./artifacts/test-image --clones 2 +mv artifacts/test-image/trident-container-testimage_amd64_cosi_clone0.cosi artifacts/test-image/regular.cosi +mv artifacts/test-image/trident-container-testimage_amd64_cosi_clone1.cosi artifacts/test-image/regular_v2.cosi # Verity container image (for root-verity configuration) -sudo ./tests/images/testimages.py build trident-container-verity-testimage \ - --output-dir ./artifacts/test-image --clones 2 -mv artifacts/test-image/trident-container-verity-testimage_0.cosi artifacts/test-image/verity.cosi -mv artifacts/test-image/trident-container-verity-testimage_1.cosi artifacts/test-image/verity_v2.cosi +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml build trident-verity-testimage -s deployment=container -s mode=root --output-dir ./artifacts/test-image --clones 2 +mv artifacts/test-image/trident-verity-testimage_container_root_amd64_cosi_clone0.cosi artifacts/test-image/verity.cosi +mv artifacts/test-image/trident-verity-testimage_container_root_amd64_cosi_clone1.cosi artifacts/test-image/verity_v2.cosi # UKI/usr-verity container image (for usr-verity, combined configurations) -sudo ./tests/images/testimages.py build trident-container-usrverity-testimage \ - --output-dir ./artifacts/test-image --clones 2 -mv artifacts/test-image/trident-container-usrverity-testimage_0.cosi artifacts/test-image/usrverity.cosi -mv artifacts/test-image/trident-container-usrverity-testimage_1.cosi artifacts/test-image/usrverity_v2.cosi +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml build trident-verity-testimage -s deployment=container -s mode=usr --output-dir ./artifacts/test-image --clones 2 +mv artifacts/test-image/trident-verity-testimage_container_usr_amd64_cosi_clone0.cosi artifacts/test-image/usrverity.cosi +mv artifacts/test-image/trident-verity-testimage_container_usr_amd64_cosi_clone1.cosi artifacts/test-image/usrverity_v2.cosi ``` -The images use the Image Customizer container from -`mcr.microsoft.com/azurelinux/imagecustomizer:latest`. +The workspace defaults to the Image Customizer container declared in +`tests/images/tailor.yaml` (`mcr.microsoft.com/azurelinux/imagecustomizer:latest`). ### 9. Build the Installer ISO @@ -255,8 +249,9 @@ make bin/trident-mos.iso **Container runtime:** ```bash -sudo ./tests/images/testimages.py build trident-container-installer \ - --output-dir ./artifacts +cargo run --manifest-path tools/tailor/Cargo.toml -- \ + --manifest tests/images/tailor.yaml \ + build trident-container-installer --output-dir ./artifacts ``` This builds `artifacts/trident-container-installer.iso` using Image Customizer. diff --git a/docs/Development/Testing/Functional-Tests.md b/docs/Development/Testing/Functional-Tests.md index 2a4d79b99f..f4bdd1115b 100644 --- a/docs/Development/Testing/Functional-Tests.md +++ b/docs/Development/Testing/Functional-Tests.md @@ -47,18 +47,17 @@ and a base image also from MCR. 1. **Download the base image:** ```bash - # Downloads baremetal.vhdx from mcr.microsoft.com/azurelinux/3.0/image/baremetal:latest - ./tests/images/testimages.py download-image baremetal + cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml bases download baremetal ``` 2. **Build the functional test image:** ```bash - sudo ./tests/images/testimages.py build trident-functest --output-dir ./artifacts + cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml build trident-functest --output-dir ./artifacts ``` - This produces `artifacts/trident-functest.qcow2`. The image configuration is - defined in `tests/images/trident-functest/base/baseimg.yaml`. + This produces `artifacts/trident-functest.qcow2`. The image is selected from + the `trident-functest` family in `tests/images/tailor.yaml`. ## Building Test Dependencies diff --git a/docs/Development/Testing/Rollback-Tests.md b/docs/Development/Testing/Rollback-Tests.md index 385fd77ee2..cf7e694f56 100644 --- a/docs/Development/Testing/Rollback-Tests.md +++ b/docs/Development/Testing/Rollback-Tests.md @@ -128,7 +128,7 @@ TEST_IMAGE_NAME="trident-vm-usr-verity-testimage" sudo rm -f artifacts/trident-vm-*-testimage.qcow2 artifacts/trident-vm-*-testimage.cosi # Build the COSI and QCOW2 -sudo ./tests/images/testimages.py build $TEST_IMAGE_NAME --output-dir ./artifacts +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml build trident-vm-testimage -s arch=amd64 -s scenario=usr-verity --output-dir ./artifacts make artifacts/$TEST_IMAGE_NAME.qcow2 ``` diff --git a/docs/Development/Testing/Servicing-Tests.md b/docs/Development/Testing/Servicing-Tests.md index af1eb33b92..dd3cc5de14 100644 --- a/docs/Development/Testing/Servicing-Tests.md +++ b/docs/Development/Testing/Servicing-Tests.md @@ -105,7 +105,7 @@ step, so the base image's pre-installed package set does not matter. :::tip Internal shortcut If you have access to the internal Azure DevOps artifacts feed, the Makefile downloads `qemu_guest` automatically (requires `az login`). You can also -download it directly: `./tests/images/testimages.py download-image qemu_guest` +download it directly: `cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml bases download qemu_guest` ::: ### 5. Build the VM Image @@ -131,15 +131,12 @@ the same filename in each directory (the update loop picks the filename from ```bash mkdir -p artifacts/update-a artifacts/update-b -# Build two clones of the COSI image (produces _0 and _1 suffixed files) -sudo ./tests/images/testimages.py build trident-vm-grub-verity-testimage \ - --output-dir ./artifacts --clones 2 +# Build two clones of the COSI image +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml build trident-vm-testimage -s arch=amd64 -s scenario=grub-verity --output-dir ./artifacts --clones 2 # Move the clones into the update directories with the same filename -mv artifacts/trident-vm-grub-verity-testimage_0.cosi \ - artifacts/update-a/trident-vm-grub-verity-testimage.cosi -mv artifacts/trident-vm-grub-verity-testimage_1.cosi \ - artifacts/update-b/trident-vm-grub-verity-testimage.cosi +mv artifacts/trident-vm-testimage_amd64_grub-verity_cosi_clone0.cosi artifacts/update-a/trident-vm-grub-verity-testimage.cosi +mv artifacts/trident-vm-testimage_amd64_grub-verity_cosi_clone1.cosi artifacts/update-b/trident-vm-grub-verity-testimage.cosi ``` :::note diff --git a/tests/images/README.md b/tests/images/README.md new file mode 100644 index 0000000000..fbd8ac44df --- /dev/null +++ b/tests/images/README.md @@ -0,0 +1,66 @@ +# Test images with tailor + +`tests/images/` is now a [tailor](../../tools/tailor/README.md) workspace. +The legacy `testimages.py` / `builder/` flow is removed. + +## Build from the repo root + +Use the in-tree tailor binary via Cargo: + +```bash +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml matrix trident-installer --format json +``` + +Build examples: + +```bash +# trident-testimage +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml build trident-testimage -s arch=amd64 --output-dir ./artifacts + +# trident-testimage-arm64 +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml build trident-testimage -s arch=arm64 --output-dir ./artifacts + +# trident-direct-streaming-installer-arm64 +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml build trident-installer -s arch=arm64 -s variant=direct-streaming --output-dir ./artifacts + +# trident-container-usrverity-testimage (signed) +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml build trident-verity-testimage -s deployment=container -s mode=usr --output-dir ./artifacts +``` + +## Base images + +The workspace uses a `baseImages:` catalogue in `tests/images/tailor.yaml`. +For the Azure Linux base slots you can materialize the defaults locally with: + +```bash +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml bases download baremetal core_arm64 core_selinux qemu_guest +``` + +Verify that every referenced base file exists: + +```bash +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml bases verify +``` + +Ubuntu / GB200 direct-streaming inputs still come from the pipeline-managed blob +staging flow; those slots are intentionally catalogued without an in-workspace +remote source. + +## Legacy name mapping + +Pipelines still accept the historical image names. `tests/images/legacy-map.json` +maps each legacy name to the tailor image family, selectors, expected base slot, +and published artifact extension. + +The current families are: + +- `azl-installer` +- `azurelinux-direct-streaming-testimage` +- `foreign-direct-streaming-testimage` +- `trident-container-installer` +- `trident-container-testimage` +- `trident-functest` +- `trident-installer` +- `trident-testimage` +- `trident-verity-testimage` +- `trident-vm-testimage` diff --git a/tests/images/azl-installer/image.yaml b/tests/images/azl-installer/image.yaml new file mode 100644 index 0000000000..0622ba3131 --- /dev/null +++ b/tests/images/azl-installer/image.yaml @@ -0,0 +1,18 @@ +name: azl-installer + +base: + ref: baremetal + +outputs: + - format: iso + +rpmSources: + - ../../../bin/RPMS + +extraDependencies: + - ./installer-iso.yaml + - ./mos + - ./iso + +config: + $include: ./installer-iso.yaml diff --git a/tests/images/azurelinux-direct-streaming-testimage/by-arch/arm64.yaml b/tests/images/azurelinux-direct-streaming-testimage/by-arch/arm64.yaml new file mode 100644 index 0000000000..6fe6f6f38d --- /dev/null +++ b/tests/images/azurelinux-direct-streaming-testimage/by-arch/arm64.yaml @@ -0,0 +1,3 @@ +base: + $set: + ref: core_arm64 diff --git a/tests/images/azurelinux-direct-streaming-testimage/image.yaml b/tests/images/azurelinux-direct-streaming-testimage/image.yaml new file mode 100644 index 0000000000..c6cca2f107 --- /dev/null +++ b/tests/images/azurelinux-direct-streaming-testimage/image.yaml @@ -0,0 +1,19 @@ +name: azurelinux-direct-streaming-testimage + +matrix: + arch: [amd64, arm64] + +base: + ref: baremetal + +outputs: + - format: baremetal-image + +rpmSources: + - ../../../bin/RPMS + +extraDependencies: + - ./base + +config: + $include: ./tailor/baseimg.yaml diff --git a/tests/images/azurelinux-direct-streaming-testimage/tailor/baseimg.yaml b/tests/images/azurelinux-direct-streaming-testimage/tailor/baseimg.yaml new file mode 100644 index 0000000000..a6af1726ae --- /dev/null +++ b/tests/images/azurelinux-direct-streaming-testimage/tailor/baseimg.yaml @@ -0,0 +1,68 @@ +storage: + bootType: efi + disks: + - partitionTableType: gpt + maxSize: 15G + partitions: + - id: esp + type: esp + size: 50M + label: esp + - id: root-a + size: 4G + type: root + label: root-a + - id: root-b + size: 4G + type: root + label: root-b + - id: swap + size: 2G + type: swap + label: swap + filesystems: + - deviceId: esp + type: fat32 + mountPoint: + path: /boot/efi + options: umask=0077 + - deviceId: root-a + type: ext4 + mountPoint: + path: / +os: + bootloader: + resetType: hard-reset + hostname: trident-rawcosi-testimage + selinux: + mode: permissive + kernelCommandLine: + extraCommandLine: + - console=tty0 + - console=ttyS0 + - rd.info + - log_buf_len=1M + packages: + install: + - curl + - dnf + - efibootmgr + - iproute + - iptables + - lsof + - mdadm + - netplan + - openssh-server + - tpm2-tools + - vim + - audit + - device-mapper + - dosfstools + - lvm2 + - veritysetup + - ntfs-3g + - ntfsprogs + - selinux-policy + services: + enable: + - sshd diff --git a/tests/images/builder/README.md b/tests/images/builder/README.md deleted file mode 100644 index 0fafbd66a9..0000000000 --- a/tests/images/builder/README.md +++ /dev/null @@ -1,130 +0,0 @@ -# Builder - -Builder is a Python tool to declare, customize, and build images with AZL Image Customizer locally and in the pipelines. - -## Overview - -Builder is a builder system designed around declarative image definitions that wraps around the AZL Image Customizer concepts and API. - -`testimages.py` script in the top-level directory contains declarative definitions of all images and artifacts, such as the Image Customizer container. It is a convenient entry point for building an image locally or in the pipelines. To learn more about the supported commands, run: - -```bash -python3 ./testimages.py --help -``` - -## Directory Structure - -```md -builder/ -├── __init__.py # Definitions of wrappers around Image Customizer concepts -├── builder.py # High-level logic for building image clones -├── cli.py # Command-line interface and command execution -├── context_managers.py # Utilities for resource cleanup -├── customize.py # Image Customizer API wrapper -├── download.py # Utilities for image download -├── README.md # README -├── run.py # Core build functions and orchestration -└── sign.py # Utilities for image signing -``` - -## Key Components - -### `__init__.py` - -Defines foundational data structures and enums used throughout Builder. Specifically, defines `ImageConfig`, which represents an Image Customizer config, and `ArtifactManifest`, which describes the Image Cuztomizer container to be used for building images. - -Also, contains a series of other definitions that represent the base image type, output format, system architecture, etc. - -### `cli.py` - -Implements Builder's command-line interface and executes commands such as `build()`. Most of the high-level logic happens in `cli.init()`, which orchestrates the entire build process. - -### `run.py` - -Contains the implementations of the core functions supported by Builder, such as `build()` or `generate_matrix()`. - -### `builder.py` - -This is where the high-level logic around building images, signed and unsigned, lives. This file calls into Image Customizer APIs inside `customize.py` to build images, using cloning and parallel processing. - -### `customize.py` - -Wrapper around the AZL Image Customizer API. Only container-based execution is now supported since running IC as a raw binary is no longer supported. - -Specifically, provides APIs for (1) building an image and (2) injecting signed boot artifacts into an image via the preview feature `inject-files`. - -### Utility Files - -#### `sign.py` - -Utility functions needed for signing an image built via Image Customizer. This is needed for enabling `SecureBoot` in a host. - -#### `download.py` - -Utility functions for downloading images as AZ artifacts. - -#### `context_managers.py` - -Utility functions for resource cleanup. - -## Key Concepts - -### Image Cloning - -The builder supports creating multiple clones of the same image with different UUIDs. This is essential for testing updates where you need identical images with unique identifiers: - -### Parallel Processing - -The system uses Python's `multiprocessing` package to build image clones in parallel, significantly reducing build times. Each clone is built in its own process with a deep copy of the `ImageConfig` object to avoid race conditions. - -### Resource Management - -The builder uses `ExitStack()` context managers to ensure proper cleanup of temporary resources. - -## Running Builder - -Builder can be run locally as a stand-alone tool: - -```bash - python3 ./testimages.py -``` - -The following commands are supported: - -```bash - # List images available for building - python3 ./testimages.py list - # Download a base image: baremetal, core_selinux, qemu_guest, or minimal - python3 ./testimages.py download-image - # List required dependencies for an image - python3 ./testimages.py dependencies - # Build an image locally - python3 ./testimages.py build - # Show key info about the image, such as name, source, config path, etc. - python3 ./testimages.py show-image - # Show info on key artifacts, such as the Image Customizer version or container image - python3 ./testimages.py show-artifact - # List images that have been built and are available to be used - python3 ./testimages.py list-files -``` - -To build an image with Builder or test your changes to Builder, follow these steps: - -1. If necessary, make changes to the Builder source code. -1. Download the base image: - -```bash - ./testimages.py download-image -``` - -1. If necessary, update the Image Customizer config for the image you want to build, by modifying the corresponding YAML in `test-images/platform-integration-images`. You can also find the relevant YAML by running: - -```bash - python3 ./testimages.py show-image config-file -``` - -1. Build the image: - -```bash - python3 ./testimages.py build --output-dir --no-download --clones -``` diff --git a/tests/images/builder/__init__.py b/tests/images/builder/__init__.py deleted file mode 100644 index ca82f58db9..0000000000 --- a/tests/images/builder/__init__.py +++ /dev/null @@ -1,272 +0,0 @@ -import yaml - -from dataclasses import dataclass, field, fields -from enum import Enum -from pathlib import Path -from typing import List, Optional - - -@dataclass -class BaseImageData: - name: str - path: Path - - -class BaseImage(Enum): - BAREMETAL = BaseImageData("baremetal", Path("artifacts/baremetal.vhdx")) - CORE_SELINUX = BaseImageData("core_selinux", Path("artifacts/core_selinux.vhdx")) - QEMU_GUEST = BaseImageData("qemu_guest", Path("artifacts/qemu_guest.vhdx")) - CORE_ARM64 = BaseImageData("core_arm64", Path("artifacts/core_arm64.vhdx")) - MINIMAL = BaseImageData("minimal", Path("artifacts/minimal.vhdx")) - MINIMAL_AARCH64 = BaseImageData( - "minimal_aarch64", Path("artifacts/minimal_aarch64.vhdx") - ) - UBUNTU_2204_AMD64 = BaseImageData( - "ubuntu_2204_amd64", Path("artifacts/ubuntu_2204_amd64.vhdx") - ) - UBUNTU_2204_ARM64 = BaseImageData( - "ubuntu_2204_arm64", Path("artifacts/ubuntu_2204_arm64.vhdx") - ) - UBUNTU_2404_AMD64 = BaseImageData( - "ubuntu_2404_amd64", Path("artifacts/ubuntu_2404_amd64.vhdx") - ) - UBUNTU_2404_ARM64 = BaseImageData( - "ubuntu_2404_arm64", Path("artifacts/ubuntu_2404_arm64.vhdx") - ) - GB200_2404_ARM64 = BaseImageData( - "gb200_2404_arm64", Path("artifacts/gb200_2404_arm64.vhdx") - ) - - @property - def path(self) -> Path: - return self.value.path - - @property - def name(self) -> str: - return self.value.name - - def __str__(self) -> str: - return self.value.name - - -@dataclass -class BaseImageManifest: - image: BaseImage - package_name: str - version: str - org: str = "https://dev.azure.com/mariner-org/" - project: str = "36d030d6-1d99-4ebd-878b-09af1f4f722f" - feed: str = "AzureLinuxArtifacts" - glob: str = "*.vhdx" - - -class OutputFormat(Enum): - BAREMETAL_IMAGE = "baremetal-image" - COSI = "cosi" - VHDX = "vhdx" - RAW = "raw" - QCOW2 = "qcow2" - ISO = "iso" - VHD = "vhd" - VHD_FIXED = "vhd-fixed" - - def ic_name(self): - """Return the Image Customizer name for this format.""" - return self.value - - def ext(self) -> str: - """Return the file extension for this format.""" - if self == OutputFormat.VHD_FIXED: - return "vhd" - elif self == OutputFormat.BAREMETAL_IMAGE: - return "cosi" - return self.value - - -class RpmSources(Enum): - TRIDENT = Path("bin/RPMS") - DHCP = Path("artifacts/dhcp") - RPM_OVERRIDES = Path("artifacts/rpm-overrides") - - def path(self) -> Path: - return self.value - - -class SystemArchitecture(Enum): - AMD64 = "amd64" - ARM64 = "arm64" - - def __str__(self) -> str: - return self.value - - -@dataclass -class ImageConfig: - # Friendly name of the image - name: str - - # Top level config dir - source: str = "tests/images" - - # Second-level dir, generally same as name - config: str = None - - # YAML config file inside the config dir - config_file: Path = Path("base/baseimg.yaml") - - # The base image to use - base_image: BaseImage = BaseImage.BAREMETAL - - # Whether the image requires Trident RPMs - requires_trident: bool = True - - # Whether the image requires DHCP RPMs - requires_dhcp: bool = False - - # Desired output format for this image - output_format: OutputFormat = OutputFormat.COSI - - # Extra dependencies for this image - extra_dependencies: List[Path] = field(default_factory=list) - - # Requires ukify to be present on the host - requires_ukify: bool = False - - # When present, path to write a public SSH key to for customizer to consume - # into the image. Both keys will be written to the output directory. - ssh_key: Optional[Path] = None - - # Architecture of the image - architecture: SystemArchitecture = SystemArchitecture.AMD64 - - # Use ImageCustomizer convert command rather than customize - image_customizer_convert: bool = False - - @classmethod - def kebab_fields(cls) -> List[str]: - """Return a list of fields in kebab-case.""" - return [f.name.replace("_", "-") for f in fields(cls)] - - def __post_init__(self): - self.suffix = None - if not self.config: - self.config = self.name - - # Update the ssh key to be a Path object if it's a string - if isinstance(self.ssh_key, str): - self.ssh_key = Path(self.ssh_key) - - # Update config_file to be a Path object if it's a string - if isinstance(self.config_file, str): - self.config_file = Path(self.config_file) - - # Automatically set the architecture to arm64 if the base image is ARM64 - if self.base_image == BaseImage.CORE_ARM64: - self.architecture = SystemArchitecture.ARM64 - - # Placeholder for the loaded base Image Customizer config - self.__base_ic_config = None - - @property - def base_ic_config(self) -> dict: - """Lazy-load and return the base Image Customizer config as a dict.""" - if self.__base_ic_config is None: - try: - with open(self.full_yaml_path(), "r") as f: - self.__base_ic_config = yaml.safe_load(f) - except Exception as e: - raise RuntimeError( - f"Error loading image config '{self.full_yaml_path()}': {e}" - ) from e - return self.__base_ic_config - - def base_dir(self) -> Path: - return Path(self.source) / self.config - - def full_yaml_path(self) -> Path: - return self.base_dir() / self.config_file - - def dependencies(self) -> List[Path]: - deps = [self.base_image.path] - if not self.image_customizer_convert: - deps.append(self.full_yaml_path()) - for file in self.base_dir().rglob("*"): - if file.is_file(): - deps.append(file) - self.base_dir().glob - if self.requires_trident: - deps.append(RpmSources.TRIDENT.path()) - deps.extend(RpmSources.TRIDENT.path().rglob("*.rpm")) - if self.requires_dhcp: - deps.append(RpmSources.DHCP.path()) - deps.extend(RpmSources.DHCP.path().rglob("*.rpm")) - deps.extend(self.extra_dependencies) - return deps - - def file_name(self) -> str: - """ - Returns the file name for the image. - """ - return f"{self.id}.{self.output_format.ext()}" - - def file_name_unsigned_raw(self) -> str: - """Returns the file name for the unsigned raw image.""" - return f"{self.id}-unsigned.{OutputFormat.RAW.ext()}" - - def set_suffix(self, suffix: str) -> None: - self.suffix = suffix - - @property - def id(self) -> str: - """Return the image ID.""" - if self.suffix is None: - return self.name - return f"{self.name}_{self.suffix}" - - def get_output_artifacts_dir(self) -> Optional[str]: - """ - Return the output.artifacts.path from the image configuration YAML. - - Throws: - ValueError if the path is present but empty. - """ - path = self.base_ic_config.get("output", {}).get("artifacts", {}).get("path") - if path is not None and not path: - raise ValueError("output.artifacts.path cannot be empty") - return path - - def get_items_to_sign(self) -> List[str]: - """Return the list of items to sign from the image configuration YAML.""" - return ( - self.base_ic_config.get("output", {}).get("artifacts", {}).get("items", []) - ) - - -# IMPORTANT: THESE NAMES ARE EXPOSED IN THE CLI, MAKE SURE TO UPDATE ALL -# REFERENCES IF YOU CHANGE THEM! -@dataclass -class ArtifactManifest: - customizer_version: str - customizer_container: str - customizer_container_full: str = None - base_images: List[BaseImageManifest] = field(default_factory=list) - - def __post_init__(self): - if self.customizer_container_full is None: - self.customizer_container_full = self.customizer_container - if ":" not in self.customizer_container_full: - self.customizer_container_full = ( - f"{self.customizer_container}:{self.customizer_version}" - ) - - @classmethod - def kebab_fields(cls) -> List[str]: - """Return a list of fields in kebab-case.""" - return [f.name.replace("_", "-") for f in fields(cls)] - - def find_base_image(self, img: BaseImage) -> Optional[BaseImageManifest]: - """Find a base image by its name.""" - for base_image in self.base_images: - if base_image.image == img: - return base_image - return None diff --git a/tests/images/builder/builder.py b/tests/images/builder/builder.py deleted file mode 100644 index 3c066d0216..0000000000 --- a/tests/images/builder/builder.py +++ /dev/null @@ -1,501 +0,0 @@ -import copy -import logging -import multiprocessing -import multiprocessing.dummy -import re -import shutil -import yaml -import threading -import tempfile - -from contextlib import ExitStack -from cryptography.hazmat.backends import default_backend as crypto_default_backend -from cryptography.hazmat.primitives import serialization as crypto_serialization -from cryptography.hazmat.primitives.asymmetric import rsa, ed25519 -from pathlib import Path -from typing import List, Optional, Tuple - -from builder import ( - ArtifactManifest, - ImageConfig, - OutputFormat, - convert, - customize, - sign, -) -from builder.context_managers import temp_dir, temp_file - -logging.basicConfig(level=logging.INFO) -log = logging.getLogger(__name__) - -# Name of helper configuration file generated by Image Customizer under the same directory as -# output artifacts, when output.artifacts feature is used. This file is used to inject signed -# artifacts back into an image. Docs: -# https://microsoft.github.io/azure-linux-image-tools/imagecustomizer/api/configuration/injectFilesConfig.html. -INJECT_FILES_YAML = "inject-files.yaml" - - -def build_image( - *, - container_image: str, - image: ImageConfig, - output_dir: Path, - artifacts: ArtifactManifest, - clones: int = 1, - rpm_sources: List[Path] = [], - image_architecture: Optional[str] = None, - dry_run: bool = False, - force: bool = False, -): - """Build the image using the specified Image Customizer container.""" - - if not dry_run: - # Create the output directory if it doesn't exist - output_dir.mkdir(parents=True, exist_ok=True) - log.info(f"Output directory: {output_dir}") - - ssh_file = generate_ssh_keys( - output_dir, - image, - dry_run=dry_run, - ) - - # If image needs signing, generate a CA certificate - ca_tmp_dir, ca_nss_key_db = None, None - if image.get_output_artifacts_dir(): - ca_tmp_dir = Path(tempfile.mkdtemp(prefix="ca_")) - ca_nss_key_db = sign.generate_ca_certificate(ca_tmp_dir) - - try: - if clones == 1: - build_one( - container_image, - image, - output_dir, - rpm_sources, - image_architecture, - dry_run, - force, - ca_nss_key_db=ca_nss_key_db, - ) - else: - # Parallel execution - build_with_clones( - container_image, - image, - output_dir, - clones, - rpm_sources, - image_architecture, - dry_run, - force, - ca_nss_key_db=ca_nss_key_db, - ) - - # Publish the CA certificate to the output dir - if ca_nss_key_db and not dry_run: - sign.publish_ca_certificate(ca_nss_key_db, output_dir) - - finally: - if ssh_file: - # Clean up the SSH key file - ssh_file.unlink(missing_ok=True) - log.info(f"SSH key file {ssh_file} removed.") - - # Clean up the CA temporary directory - if ca_tmp_dir: - shutil.rmtree(ca_tmp_dir, ignore_errors=True) - log.info(f"CA temporary directory {ca_tmp_dir} removed.") - - -def build_one( - container_image: str, - image: ImageConfig, - output_dir: Path, - rpm_sources: List[Path] = [], - image_architecture: Optional[str] = None, - dry_run: bool = False, - force: bool = False, - ca_nss_key_db: Optional[Path] = None, -): - for dep in image.dependencies(): - if not dep.exists(): - raise FileNotFoundError(f"Dependency '{dep}' does not exist.") - - # Check if final image output exists, to see if building is necessary - output_file = output_dir / image.file_name() - if output_file.exists(): - log.info(f"Output file {output_file} already exists.") - if force: - log.info(f"Force build requested for '{image.id}', rebuilding.") - else: - dest_mod_time = output_file.stat().st_mtime - for dep in image.dependencies(): - if dep.is_dir(): - # Ignore directories - continue - if dep.stat().st_mtime > dest_mod_time: - log.info(f"Dependency '{dep}' is newer than output file.") - break - else: - log.info(f"Output file {output_file} is up to date.") - return output_file - - if image.image_customizer_convert: - # If 'convert' is requested, run Image Customizer convert subcommand - convert.convert_image( - container_image, - image.id, - image.base_image.path, - image.output_format.ic_name(), - output_file, - image_architecture, - dry_run, - ) - else: - # Copy RPM sources to standalone temporary directories - with ExitStack() as stack: - tmp_rpm_sources = [] - for rpm_src in rpm_sources: - # Create a temporary directory for each RPM source, delete with sudo because - # createrepo is run as root. - tmp = stack.enter_context( - temp_dir(prefix=f"rpm-{rpm_src.stem}-", sudo=True) - ) - tmp_rpm_sources.append(tmp) - log.debug(f"Copying RPM source {rpm_src} to {tmp}") - shutil.copytree(rpm_src, tmp, dirs_exist_ok=True) - - if image.get_output_artifacts_dir(): - # If config YAML contains output.artifacts, then need to output signed image. First, build - # an unsigned image; then, sign boot artifacts, and inject the signed copies back, to build - # a signed image as final output. - build_signed_image( - stack, - container_image, - image, - tmp_rpm_sources, - output_dir, - output_file, - ca_nss_key_db, - image_architecture, - dry_run, - ) - else: - # Otherwise, only build an unsigned image - customize.build_config( - container_image, - image.id, - image.full_yaml_path(), - image.base_image.path, - image.output_format.ic_name(), - output_file, - tmp_rpm_sources, - image_architecture, - dry_run, - ) - - output_file = output_dir / image.file_name() - log.info(f"Image '{image.id}' built successfully in {output_file}") - - -def build_with_clones( - container_image: str, - image: ImageConfig, - output_dir: Path, - clones: int, - rpm_sources: List[Path] = [], - image_architecture: Optional[str] = None, - dry_run: bool = False, - force: bool = False, - ca_nss_key_db: Optional[Path] = None, -): - # Build all the clones in parallel - build_clones( - container_image, - image, - output_dir, - clones, - rpm_sources, - image_architecture, - dry_run, - force, - ca_nss_key_db=ca_nss_key_db, - ) - - # In the future, when enabled by Prism, we want to build an intermediate - # image and then build the clones from that image. Leaving that code here - # for future reference. - - # # Extract kernel command line config - # with open(yaml_path, "r") as f: - # config = yaml.load(f, Loader=yaml.Loader) - # kernel_cmdline = ( - # config.get("os", {}).get("kernelCommandLine", {}).get("extraCommandLine", []) - # ) - - # # Run the build process once for the intermediate image. - # intermediate_img = build_one( - # build_runtime, - # f"{config_name}-intermediate", - # yaml_path, - # base_image, - # OutputFormat.RAW, - # output_dir, - # rpm_sources, - # dry_run, - # ) - - # log.info(f"Intermediate image created: {intermediate_img}") - - # cfg = tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", suffix=".yaml") - # try: - # final_yaml = yaml.dump( - # { - # "storage": { - # "resetPartitionsUuidsType": "reset-all", - # }, - # "os": { - # "bootloader": { - # "resetType": "hard-reset", - # }, - # # Preserve the kernel command line from the original config - # "kernelCommandLine": { - # "extraCommandLine": kernel_cmdline, - # }, - # }, - # }, - # Dumper=yaml.Dumper, - # ) - # log.debug(f"Final YAML:\n{final_yaml}") - # cfg.write(final_yaml) - # cfg.flush() - # cfg.seek(0) - - # build_clones( - # build_runtime, - # config_name, - # Path(cfg.name), - # intermediate_img, - # img_format, - # output_dir, - # clones, - # rpm_sources, - # dry_run, - # ) - # finally: - # # Clean up the temporary files - # cfg.close() - # intermediate_img.unlink(missing_ok=True) - - -def build_clones( - container_image: str, - image: ImageConfig, - output_dir: Path, - clones: int, - rpm_sources: List[Path] = [], - image_architecture: Optional[str] = None, - dry_run: bool = False, - force: bool = False, - ca_nss_key_db: Optional[Path] = None, -): - def clone_image(image: ImageConfig, clone_index: int) -> ImageConfig: - img = copy.deepcopy(image) - img.set_suffix(str(clone_index)) - return img - - pool = multiprocessing.dummy.Pool(clones) - try: - log.info(f"Building {clones} clones of {image.name}") - results = pool.starmap( - build_one, - [ - ( - container_image, - clone_image(image, i), - output_dir, - rpm_sources, - image_architecture, - dry_run, - force, - ca_nss_key_db, - ) - for i in range(clones) - ], - ) - return results - finally: - pool.close() - pool.join() - - -def generate_ssh_keys( - output_dir: Path, - image: ImageConfig, - dry_run: bool = False, -) -> Optional[Path]: - """Generate SSH keys for the image.""" - if not image.ssh_key: - return - - # the artifact path is relative to the image's YAML file - artifact_path = image.full_yaml_path().parent / image.ssh_key - - if artifact_path.exists(): - log.warning( - f"SSH key file {artifact_path} already exists, not generating a new key!" - ) - return artifact_path - - filename = image.ssh_key.name - m = re.match(r"^id_(rsa|ed25519).pub$", filename) - if not m: - raise ValueError(f"Invalid SSH key name: {filename}") - - key_type = m.group(1) - - if key_type == "rsa": - key = rsa.generate_private_key( - backend=crypto_default_backend(), public_exponent=65537, key_size=2048 - ) - elif key_type == "ed25519": - key = ed25519.Ed25519PrivateKey.generate() - else: - raise ValueError(f"Unsupported SSH key type: {key_type}") - - private_key = key.private_bytes( - crypto_serialization.Encoding.PEM, - crypto_serialization.PrivateFormat.TraditionalOpenSSL, - crypto_serialization.NoEncryption(), - ) - - public_key = key.public_key().public_bytes( - crypto_serialization.Encoding.OpenSSH, crypto_serialization.PublicFormat.OpenSSH - ) - - if dry_run: - print(f"Dry run: would generate SSH keys for {image.name}") - print(f"Private key: {private_key.decode()}") - print(f"Public key: {public_key.decode()}") - return - - # Write the private key to a file - private_key_path = output_dir / f"key_{image.name}_private.pem" - public_key_path = output_dir / f"key_{image.name}_public.pub" - with open(private_key_path, "wb") as f: - f.write(private_key) - with open(public_key_path, "wb") as f: - f.write(public_key) - - with open(artifact_path, "wb") as f: - f.write(public_key) - log.info(f"SSH keys generated for {image.name}") - - return artifact_path - - -def build_signed_image( - stack: ExitStack, - container_image: str, - image: ImageConfig, - tmp_rpm_sources: List[Path], - output_dir: Path, - output_file: Path, - ca_nss_key_db: Path, - image_architecture: Optional[str] = None, - dry_run: bool = False, -): - - # Set up thread-specific YAML and output artifacts directory - working_yaml_path, output_artifacts_dir = setup_temp_yaml_and_dir(stack, image) - - # Construct the path to the unsigned image - unsigned_output_file = output_artifacts_dir / image.file_name_unsigned_raw() - log.debug( - f"Process with PID {threading.get_ident()} will write unsigned image and output artifacts to {output_artifacts_dir.absolute()}" - ) - - # Build the unsigned image - customize.build_config( - container_image, - image.id, - working_yaml_path, - image.base_image.path, - # Set output format of unsigned image to raw file since that's the format used - # internally by Image Customizer for file injection - OutputFormat.RAW.ic_name(), - unsigned_output_file, - tmp_rpm_sources, - image_architecture, - dry_run, - ) - - log.info(f"Building signed image: {output_file}") - - # Generate a leaf certificate for this clone using the CA certificate. The leaf certificate - # must be generated in the same NSS key database as the CA certificate. - leaf_key_name = sign.generate_leaf_certificate(ca_nss_key_db, image.id) - - # Construct full path of inject-files.yaml - inject_files_yaml_path = output_artifacts_dir / INJECT_FILES_YAML - # Sign boot artifacts that Image Customizer output when building the unsigned image - sign.sign_boot_artifacts( - ca_nss_key_db, - leaf_key_name, - inject_files_yaml_path, - output_artifacts_dir, - ) - - # Run inject-files via Image Customizer to inject the signed UKI back into the image - log.info( - f"Running imagecustomizer inject-files using YAML: {inject_files_yaml_path}" - ) - log.debug( - f"Contents of output artifacts directory: {output_artifacts_dir}:\n" - + "\n".join([str(p) for p in output_artifacts_dir.rglob("*") if p.is_file()]) - ) - customize.inject_files( - container_image, - inject_files_yaml_path, - unsigned_output_file, - image.output_format.ic_name(), - output_file, - dry_run, - ) - - -def setup_temp_yaml_and_dir( - stack: ExitStack, - image: ImageConfig, -) -> Tuple[Path, Path]: - """ - Sets up a temporary YAML file and output artifacts directory for building a signed image. - Returns the path to the temporary YAML file and the output artifacts directory. - """ - - # Figure out the directory of the main YAML file - yaml_dir = image.full_yaml_path().parent.absolute() - - # Create a temp dir inside yaml_dir to store the unsigned image and output artifacts - output_artifacts_dir = stack.enter_context( - temp_dir(prefix=f".{image.id}-", dir=yaml_dir, sudo=True) - ) - - # Now figure out the relative path from yaml_dir to output_artifacts_dir - tmp_rel_path = output_artifacts_dir.absolute().relative_to(yaml_dir) - - # Update the in-memory config, this is safe because every thread has its own copy of the YAML. - # We use the relative path because the path should be relative to the YAML location. - image.base_ic_config["output"]["artifacts"]["path"] = f"./{tmp_rel_path}" - - # Create a thread-specific copy of YAML in the same dir as main YAML - working_yaml_path = yaml_dir / f".config_{image.id}.yaml" - - with open(working_yaml_path, "w") as f: - yaml.safe_dump(image.base_ic_config, f) - - # Ensure the temp YAML file is deleted at the end - stack.enter_context(temp_file(working_yaml_path)) - - return working_yaml_path, output_artifacts_dir diff --git a/tests/images/builder/cli.py b/tests/images/builder/cli.py deleted file mode 100644 index 741f0c2396..0000000000 --- a/tests/images/builder/cli.py +++ /dev/null @@ -1,303 +0,0 @@ -import argparse -from enum import Enum -import logging -from pathlib import Path - -from typing import List - -from builder import ( - ImageConfig, - ArtifactManifest, - SystemArchitecture, - run, -) - - -logging.basicConfig(level=logging.INFO) -log = logging.getLogger("trident-testimages") - - -def positive_int(value) -> int: - ivalue = int(value) - if ivalue < 1: - raise argparse.ArgumentTypeError(f"{value} is an invalid positive int value") - return ivalue - - -class SubCommand(Enum): - LIST = "list" - DEPENDENCIES = "dependencies" - BUILD = "build" - LIST_FILES = "list-files" - SHOW_ARTIFACT = "show-artifact" - SHOW_IMAGE = "show-image" - DOWNLOAD_IMAGE = "download-image" - MATRIX = "matrix" - - -def init(configs: List[ImageConfig], artifacts: ArtifactManifest) -> None: - parser = argparse.ArgumentParser( - description="Get dependency list for specific image" - ) - subparsers = parser.add_subparsers( - help="Thing to do", dest="command", required=True, metavar="COMMAND" - ) - - setup_parser_list(subparsers) - setup_parser_show(subparsers, artifacts) - setup_subparser_dependencies(subparsers) - setup_parser_build(subparsers, configs, artifacts) - setup_parser_list_files(subparsers) - setup_parser_download_image(subparsers, artifacts) - setup_parser_matrix(subparsers) - setup_parser_image_info(subparsers, configs) - - args = parser.parse_args() - subcommand = SubCommand(args.command) - - run_cmd(configs, artifacts, subcommand, args) - - -def positive_int(value) -> int: - ivalue = int(value) - if ivalue < 1: - raise argparse.ArgumentTypeError(f"{value} is an invalid positive int value") - return ivalue - - -def setup_parser_list(subparsers: argparse._SubParsersAction) -> None: - parser_list = subparsers.add_parser( - SubCommand.LIST.value, help="List all image definitions" - ) - parser_list.add_argument( - "--filter-type", - default=None, - type=str, - help="Filter images by type", - ) - - -def setup_parser_show( - subparsers: argparse._SubParsersAction, - artifacts: ArtifactManifest, -) -> None: - parser_show = subparsers.add_parser( - SubCommand.SHOW_ARTIFACT.value, - help="Show default artifact configuration.", - ) - parser_show.set_defaults(artifacts=artifacts) - parser_show.add_argument( - "item", - choices=ArtifactManifest.kebab_fields(), - help="The item to show", - ) - - -def setup_subparser_dependencies( - subparsers: argparse._SubParsersAction, -) -> None: - parser_dependencies = subparsers.add_parser( - SubCommand.DEPENDENCIES.value, - help="List all dependencies for a specific image", - ) - parser_dependencies.add_argument("image", help="The image to get dependencies for") - - -def setup_parser_build( - subparsers: argparse._SubParsersAction, - configs: List[ImageConfig], - artifacts: ArtifactManifest, -) -> None: - parser_build = subparsers.add_parser( - SubCommand.BUILD.value, help="Build a specific image" - ) - - parser_build.set_defaults(artifacts=artifacts) - - parser_build.add_argument( - "image", help="The image to build", choices=[c.name for c in configs] - ) - parser_build.add_argument( - "--output-dir", - help="Where to write the output image.", - default=Path.cwd() / "artifacts", - type=Path, - ) - parser_build.add_argument( - "--dry-run", action="store_true", help="Do not run the command" - ) - parser_build.add_argument( - "--container", - default=artifacts.customizer_container_full, - type=str, - help="Configure Prism container image", - ) - parser_build.add_argument( - "--image-architecture", - default=None, - type=str, - help="Provide image architecture override for cross-compile", - ) - parser_build.add_argument( - "--clones", - type=positive_int, - default=1, - help="Number of clones of this image to create. " - "The default is 1, which means no cloning will be done. When more than 1 is requested, " - "the image file names will be suffixed with `_` for each clone, starting with 0." - " Clones are built in parallel, requesting multiple clones will become very resource " - "intensive.", - ) - parser_build.add_argument( - "-f", - "--force", - action="store_true", - help="Force the build even if the image already exists and it is up to date", - ) - parser_build.add_argument( - "--no-download", - action="store_false", - dest="download", - help="By default, the builder will try to download any missing artifacts, " - "this flag will disable that behavior.", - ) - - -def setup_parser_list_files( - subparsers: argparse._SubParsersAction, -) -> None: - parser_targets = subparsers.add_parser( - SubCommand.LIST_FILES.value, help="List all images as file targets" - ) - parser_targets.add_argument( - "--output-dir", - help="Where to write the output image.", - default=Path("build"), - type=Path, - ) - - -def setup_parser_download_image( - subparsers: argparse._SubParsersAction, - artifacts: ArtifactManifest, -) -> None: - parser_download_img = subparsers.add_parser( - SubCommand.DOWNLOAD_IMAGE.value, - help="Download a base image from the Azure DevOps feed", - ) - parser_download_img.set_defaults(artifacts=artifacts) - parser_download_img.add_argument( - "image", - help="The image to download", - choices=[c.image.name for c in artifacts.base_images], - ) - - -def setup_parser_matrix( - subparsers: argparse._SubParsersAction, -) -> None: - parser_matrix = subparsers.add_parser( - SubCommand.MATRIX.value, - help="Generate ADO Pipeline matrix for all images", - ) - parser_matrix.add_argument( - "-a", - "--arch", - help=f"Architecture to build for. '{SystemArchitecture.AMD64.value}' or " - f"'{SystemArchitecture.ARM64.value}'", - default=SystemArchitecture.AMD64.value, - type=SystemArchitecture, - ) - parser_matrix.add_argument( - "--indent", - help="Indentation for the matrix", - default=None, - type=int, - ) - - -def setup_parser_image_info( - subparsers: argparse._SubParsersAction, - configs: List[ImageConfig], -) -> None: - parser_image_info = subparsers.add_parser( - SubCommand.SHOW_IMAGE.value, - help="Show image information", - ) - parser_image_info.add_argument( - "image", - help="The image to show", - choices=[c.name for c in configs], - ) - parser_image_info.add_argument( - "field", - help="The field to show", - choices=ImageConfig.kebab_fields(), - ) - parser_image_info.add_argument( - "--devops-var", - help="Output the field as a DevOps variable with the given name", - default=None, - type=str, - ) - - -def run_cmd( - configs: List[ImageConfig], - artifacts: ArtifactManifest, - subcommand: SubCommand, - args: argparse.Namespace, -): - if subcommand == SubCommand.LIST: - run.list_configs( - configs=configs, - filter_type=args.filter_type, - ) - elif subcommand == SubCommand.DEPENDENCIES: - run.list_dependencies( - configs=configs, - name=args.image, - ) - elif subcommand == SubCommand.BUILD: - run.build( - artifacts=args.artifacts, - configs=configs, - name=args.image, - container_name=args.container, - output_dir=args.output_dir, - clones=args.clones, - image_architecture=args.image_architecture, - dry_run=args.dry_run, - force=args.force, - download=args.download, - ) - elif subcommand == SubCommand.LIST_FILES: - run.list_files( - configs=configs, - output_dir=args.output_dir, - ) - elif subcommand == SubCommand.SHOW_ARTIFACT: - run.show_artifact( - artifacts=args.artifacts, - item=args.item, - ) - elif subcommand == SubCommand.DOWNLOAD_IMAGE: - run.download_base_image( - artifacts=args.artifacts, - name=args.image, - ) - elif subcommand == SubCommand.MATRIX: - run.generate_matrix( - configs=configs, - arch=args.arch, - indent=args.indent, - ) - elif subcommand == SubCommand.SHOW_IMAGE: - run.show_image( - configs=configs, - name=args.image, - field_name=args.field, - devops_var=args.devops_var, - ) - else: - raise ValueError(f"Unknown subcommand: {subcommand}") diff --git a/tests/images/builder/context_managers.py b/tests/images/builder/context_managers.py deleted file mode 100644 index 8b957653f7..0000000000 --- a/tests/images/builder/context_managers.py +++ /dev/null @@ -1,63 +0,0 @@ -import logging -import os -import shutil -import subprocess -import tempfile -from typing import Optional - -from contextlib import contextmanager -from pathlib import Path - -logging.basicConfig(level=logging.DEBUG) -log = logging.getLogger(__name__) - - -@contextmanager -def temp_dir( - prefix: Optional[str] = None, dir: Optional[Path] = None, sudo: bool = False -): - """ - Context manager for temporary directory cleanup. - - Args: - prefix: Prefix for the temp directory name - dir: Parent directory in which to create the temp dir (default None, system temp dir) - sudo: Whether to use sudo for cleanup - - Yields: - Path: Path of temp dir - """ - parent = str(dir) if dir is not None else None - build_dir = tempfile.mkdtemp(prefix=prefix, dir=parent) - try: - yield Path(build_dir) - finally: - log.debug(f"Cleaning up build dir: {build_dir}") - if sudo: - log.debug(f"Removing build dir as root: {build_dir}") - subprocess.run(["sudo", "rm", "-rf", build_dir], check=True) - else: - shutil.rmtree(build_dir) - - -@contextmanager -def temp_file(path: Path, sudo: bool = False): - """ - Context manager that deletes the specified file upon exit. - - Args: - path: Path of the temporary file to remove - sudo: Whether to use sudo for cleanup - - Yields: - Path: Path of the temp file (optional, you can just yield) - """ - try: - yield path - finally: - if path.exists(): - log.debug(f"Cleaning up temp file: {path}") - if sudo: - subprocess.run(["sudo", "rm", "-f", str(path)], check=True) - else: - os.remove(path) diff --git a/tests/images/builder/convert.py b/tests/images/builder/convert.py deleted file mode 100644 index 20f2cc9e55..0000000000 --- a/tests/images/builder/convert.py +++ /dev/null @@ -1,87 +0,0 @@ -import logging -from pathlib import Path -import subprocess -import sys -from typing import Optional - -from builder import utils - -logging.basicConfig(level=logging.DEBUG) -log = logging.getLogger(__name__ if __name__ != "__main__" else "convert-image") - - -def convert_image( - container_image: str, - config_name: str, - base_image: Path, - img_format: str, - output_file: Path, - image_architecture: Optional[str] = None, - dry_run: bool = False, -): - """ - Convert an image to a `baremetal-image` using AZL Image Customizer `convert` via Docker container. - - Args: - container_image: Docker container image for Image Customizer - config_name: Name of Image Customizer config - base_image: Path to the base image file to customize - img_format: Output image format - output_file: Path where the customized image will be saved - dry_run: If True, only log the command without executing it - - Raises: - Exception: If the Image Customizer container execution fails - """ - log.info(f"Building '{config_name}'") - - base_cmd = [ - "docker", - "run", - "--rm", - "--privileged", - "-v", - f"/:{utils.HOST_PATH}", - "-v", - "/dev:/dev", - ] - - if image_architecture: - base_cmd.append("--platform") - base_cmd.append(image_architecture) - - base_cmd.extend( - [ - container_image, - "convert", - "--log-level", - "debug", - "--build-dir", - utils.BUILD_DIR, - "--image-file", - utils.build_path(base_image), - "--output-image-format", - img_format, - "--output-image-file", - utils.build_path(output_file), - ] - ) - - # Stringify all the args - base_cmd = [str(x) for x in base_cmd] - - cmd = " \\\n ".join(base_cmd) - - log.debug(f"Running:\n {cmd}") - - if dry_run: - log.info("Dry run, not executing command") - return - - # Run the command - try: - result = subprocess.run(base_cmd, stdout=sys.stdout, stderr=sys.stderr) - result.check_returncode() - except Exception as e: - log.error(f"Error building config '{config_name}': {e}") - raise e diff --git a/tests/images/builder/customize.py b/tests/images/builder/customize.py deleted file mode 100644 index 7fd4d00004..0000000000 --- a/tests/images/builder/customize.py +++ /dev/null @@ -1,169 +0,0 @@ -import logging -from pathlib import Path -import subprocess -import sys -from typing import List, Optional - -from builder import utils - -logging.basicConfig(level=logging.DEBUG) -log = logging.getLogger(__name__ if __name__ != "__main__" else "customize-image") - - -def build_config( - container_image: str, - config_name: str, - yaml_path: Path, - base_image: Path, - img_format: str, - output_file: Path, - rpm_sources: List[Path] = [], - image_architecture: Optional[str] = None, - dry_run: bool = False, -): - """ - Build a custom image using AZL Image Customizer via Docker container. - - Note: Image Customizer no longer supports running as a raw binary and must be - executed within a Docker container. This function orchestrates the containerized - build process. - - Args: - container_image: Docker container image for Image Customizer - config_name: Name of Image Customizer config - yaml_path: Path to the Image Customizer YAML configuration file - base_image: Path to the base image file to customize - img_format: Output image format - output_file: Path where the customized image will be saved - rpm_sources: List of paths to additional RPM source directories - dry_run: If True, only log the command without executing it - - Raises: - Exception: If the Image Customizer container execution fails - """ - log.info(f"Building '{config_name}'") - log.info(f"Using YAML: {yaml_path}") - - base_cmd = [ - "docker", - "run", - "--rm", - "--privileged", - "-v", - f"/:{utils.HOST_PATH}", - "-v", - "/dev:/dev", - ] - - if image_architecture: - base_cmd.append("--platform") - base_cmd.append(image_architecture) - - base_cmd.extend( - [ - container_image, - "--config-file", - utils.build_path(yaml_path), - "--log-level", - "debug", - "--build-dir", - utils.BUILD_DIR, - "--image-file", - utils.build_path(base_image), - "--output-image-format", - img_format, - "--output-image-file", - utils.build_path(output_file), - ] - ) - - for _, rpm in enumerate(rpm_sources): - base_cmd.append("--rpm-source") - base_cmd.append(utils.build_path(rpm)) - - # Stringify all the args - base_cmd = [str(x) for x in base_cmd] - - cmd = " \\\n ".join(base_cmd) - - log.debug(f"Running:\n {cmd}") - - if dry_run: - log.info("Dry run, not executing command") - return - - # Run the command - try: - result = subprocess.run(base_cmd, stdout=sys.stdout, stderr=sys.stderr) - result.check_returncode() - except Exception as e: - log.error(f"Error building config '{config_name}': {e}") - raise e - - -def inject_files( - container_image: str, - inject_files_yaml_path: Path, - unsigned_image_file: Path, - img_format: str, - output_image_file: Path, - dry_run: bool = False, -): - """ - Run the imagecustomizer inject-files command to inject files into the image. - - Args: - container_image: Image Customizer container image to run the command in - inject_files_yaml_path: Path to the inject-files YAML configuration file listing the signed - and unsigned sources - unsigned_image_file: Path to the unsigned image - img_format: Format of the output image - output_image_file: Path to the signed output image - dry_run: If True, do not run the command - - Raises: - Exception: If docker command fails. - """ - base_cmd = [ - "docker", - "run", - "--rm", - "--privileged", - "-v", - f"/:{utils.HOST_PATH}", - "-v", - "/dev:/dev", - container_image, - "inject-files", - "--config-file", - utils.build_path(inject_files_yaml_path), - "--log-level", - "debug", - "--build-dir", - utils.BUILD_DIR, - "--image-file", - utils.build_path(unsigned_image_file), - "--output-image-format", - img_format, - "--output-image-file", - utils.build_path(output_image_file), - ] - - # Stringify all the args - base_cmd = [str(x) for x in base_cmd] - - cmd = " \\\n ".join(base_cmd) - - log.debug(f"Running:\n {cmd}") - - if dry_run: - log.info("Dry run, not executing command") - return - - # Run the command - try: - result = subprocess.run(base_cmd, stdout=sys.stdout, stderr=sys.stderr) - result.check_returncode() - except Exception as e: - log.error(f"Error running inject-files using YAML: {inject_files_yaml_path}") - raise e diff --git a/tests/images/builder/download.py b/tests/images/builder/download.py deleted file mode 100644 index 6f9db4c9f0..0000000000 --- a/tests/images/builder/download.py +++ /dev/null @@ -1,41 +0,0 @@ -from pathlib import Path -import shutil -import subprocess -import tempfile - -from builder import BaseImageManifest - - -def download_base_image(image: BaseImageManifest) -> None: - """Download the base image from MCR.""" - with tempfile.TemporaryDirectory() as tempdir: - subprocess.run( - [ - "oras", - "pull", - f"mcr.microsoft.com/azurelinux/3.0/image/{image.image.name}:latest", - "--output", - tempdir, - "--platform", - "linux/amd64", - ], - check=True, - ) - - # Find and copy the .vhdx file to the target location - tempdir_path = Path(tempdir) - vhdx_files = list(tempdir_path.glob("*.vhdx")) - if not vhdx_files: - raise RuntimeError( - f"No .vhdx file found in downloaded image for {image.image.name}" - ) - if len(vhdx_files) > 1: - raise RuntimeError( - f"Multiple .vhdx files found in downloaded image for {image.image.name}" - ) - - # Ensure the parent directory exists - image.image.path.parent.mkdir(parents=True, exist_ok=True) - - # Copy the .vhdx file to the target location - shutil.copy2(vhdx_files[0], image.image.path) diff --git a/tests/images/builder/run.py b/tests/images/builder/run.py deleted file mode 100644 index d465beb2f2..0000000000 --- a/tests/images/builder/run.py +++ /dev/null @@ -1,175 +0,0 @@ -import logging -from pathlib import Path -import json -from typing import List, Optional - -from builder import ImageConfig, RpmSources, ArtifactManifest -from .builder import build_image -from .convert import convert_image -from . import download - -log = logging.getLogger(__name__) - - -def find_image(configs: List[ImageConfig], name: str) -> ImageConfig: - for config in configs: - if config.name == name: - return config - raise ValueError(f"Image '{name}' is not defined") - - -def list_configs( - *, configs: List[ImageConfig], filter_type: Optional[str] = None -) -> None: - for config in configs: - if filter_type is None or config.output_format.ext() == filter_type: - print(config.name) - - -def list_files(*, configs: List[ImageConfig], output_dir: Path) -> None: - for config in configs: - print(output_dir / config.file_name()) - - -def list_dependencies(*, configs: List[ImageConfig], name: str) -> None: - image = find_image(configs, name) - for dep in image.dependencies(): - print(dep) - - -def show_artifact(*, artifacts: ArtifactManifest, item: str) -> None: - item = getattr(artifacts, item.replace("-", "_")) - if isinstance(item, str): - print(item) - elif isinstance(item, list): - for i in item: - print(i) - else: - raise ValueError(f"Unknown item type: {type(item)}") - - -def show_image( - *, - configs: List[ImageConfig], - name: str, - field_name: str, - devops_var: Optional[str] = None, -) -> None: - image = find_image(configs, name) - field = getattr(image, field_name.replace("-", "_"), None) - - out: str = None - - if field is None: - raise ValueError(f"Field '{field_name}' not found in image '{name}'") - if isinstance(field, str): - out = field - elif isinstance(field, list): - out = "\n".join([str(i) for i in field]) - elif hasattr(field, "__str__") and callable(field.__str__): - out = str(field) - else: - raise ValueError(f"Unknown field type: {type(field)}") - - if devops_var: - print(f"##vso[task.setvariable variable={devops_var}]{out}") - else: - print(out) - - -def build( - *, - artifacts: ArtifactManifest, - configs: List[ImageConfig], - name: str, - container_name: str, - output_dir: Path, - clones: int, - dry_run: bool, - force: bool, - image_architecture: Optional[str] = None, - download: bool = True, -) -> None: - image = find_image(configs, name) - log.info(f"Building image '{image.name}'") - - container_image: Optional[str] = container_name - if container_image is None: - log.error("Image Customizer container image is required") - exit(1) - - if image.image_customizer_convert: - # If 'convert' is requested, run Image Customizer convert subcommand - convert_image( - container_image, - image.id, - image.base_image.path, - image.output_format.ic_name(), - output_dir / image.file_name(), - image_architecture, - dry_run, - ) - else: - rpm_sources: List[Path] = [] - if image.requires_trident: - rpm_sources.append(RpmSources.TRIDENT.path()) - if image.requires_dhcp: - rpm_sources.append(RpmSources.DHCP.path()) - - rpm_overrides_path = RpmSources.RPM_OVERRIDES.path() - if rpm_overrides_path.exists(): - rpm_sources.append(rpm_overrides_path) - - if not image.base_image.path.exists(): - if download: - log.info( - f"Downloading base image to '{image.base_image.path}'" - " (use --no-download to skip this step)" - ) - - else: - log.error(f"Base image '{image.base_image.path}' does not exist.") - exit(1) - - build_image( - container_image=container_image, - image=image, - output_dir=output_dir, - artifacts=artifacts, - clones=clones, - rpm_sources=rpm_sources, - image_architecture=image_architecture, - dry_run=dry_run, - force=force, - ) - - -def download_base_image( - *, - artifacts: ArtifactManifest, - name: str, -) -> None: - image_manifest = next( - (img for img in artifacts.base_images if img.image.name == name), None - ) - if image_manifest is None: - raise ValueError(f"Image '{name}' not found in artifacts") - log.info(f"Downloading base image '{name}' to '{image_manifest.image.path}'") - download.download_base_image(image_manifest) - - -def generate_matrix( - *, - configs: List[ImageConfig], - arch: str, - indent: Optional[int] = None, -) -> None: - matrix = {} - for config in configs: - if config.architecture == arch: - matrix[config.name] = { - "image_name": config.name, - "base_image": config.base_image.name, - "img_file": str(config.base_image.path), - } - print(json.dumps(matrix, indent=indent)) diff --git a/tests/images/builder/sign.py b/tests/images/builder/sign.py deleted file mode 100644 index 0045feef8e..0000000000 --- a/tests/images/builder/sign.py +++ /dev/null @@ -1,474 +0,0 @@ -import logging -import os -import re -import subprocess -import yaml -import threading - -from pathlib import Path -from typing import Optional - -from builder.context_managers import temp_dir - - -logging.basicConfig(level=logging.DEBUG) -log = logging.getLogger(__name__) - -# Common name of CA (Certificate Authority) certificate -CA_CN = "Trident Testing CA" - -# Name of CA certificate -CA_NAME = "trident_ca" - -# Common name of signing key -KEY_CN = "Trident Testing Signing Key" - -# Name of signing key -KEY_NAME = "trident_signing_key" - -# Directory for the NSS key database -NSS_KEY_DB = "db" - -# Name of PKCS#12 archive file that contains private key and signing certificate -PKCS12_ARCH_FILE = "signer.p12" - -# IMAGE CUSTOMIZER ARTIFACT NAMES -IC_ARTIFACT_NAME_UKIS = "ukis" -IC_ARTIFACT_NAME_SHIM = "shim" -IC_ARTIFACT_NAME_SYSTEMD_BOOT = "systemd-boot" -IC_ARTIFACT_NAME_VERITY_HASH = "verity-hash" - -_KERNEL_FLAG_SUPPORTED = None -_PESIGN_CERT_ARG = None - - -def _efikeygen_supports_kernel_flag() -> bool: - global _KERNEL_FLAG_SUPPORTED - - if _KERNEL_FLAG_SUPPORTED is None: - result = subprocess.run( - ["efikeygen", "--help"], capture_output=True, text=True, check=False - ) - help_output = f"{result.stdout}\n{result.stderr}" - _KERNEL_FLAG_SUPPORTED = "--kernel" in help_output - log.debug(f"efikeygen --kernel support: {_KERNEL_FLAG_SUPPORTED}") - - return _KERNEL_FLAG_SUPPORTED - - -def _get_pesign_certificate_arg() -> str: - global _PESIGN_CERT_ARG - - if _PESIGN_CERT_ARG is None: - result = subprocess.run( - ["pesign", "--help"], capture_output=True, text=True, check=False - ) - help_output = f"{result.stdout}\n{result.stderr}" - _PESIGN_CERT_ARG = ( - "--certficate" if "--certficate" in help_output else "--certificate" - ) - log.debug(f"Using pesign certificate argument: {_PESIGN_CERT_ARG}") - - return _PESIGN_CERT_ARG - - -def generate_ca_certificate(tmp_dir: Path): - """ - Generates a single CA certificate and key that will be used to sign all leaf certificates. - This should be called once per build process. - - Args: - tmp_dir: Path to the temporary directory where the CA key is stored - - Returns: - ca_nss_key_db: Full path to the NSS key database - - Raises: - Exception: If certutil or efikeygen fails. - """ - ca_nss_key_db = tmp_dir / NSS_KEY_DB - os.makedirs(ca_nss_key_db, exist_ok=True) - log.debug(f"Initializing CA NSS key database in {ca_nss_key_db}") - - # Initialize a NSS key database for CA - subprocess.run( - ["certutil", "-N", "-d", str(ca_nss_key_db), "--empty-password"], check=True - ) - - # Generate CA certificate - subprocess.run( - [ - "efikeygen", - "-C", - "-S", - "-n", - CA_NAME, - "-c", - f"CN={CA_CN}", - "-d", - str(ca_nss_key_db), - ], - check=True, - ) - - log.info(f"Generated CA certificate {CA_NAME} at {ca_nss_key_db}") - return ca_nss_key_db - - -def generate_leaf_certificate(ca_nss_key_db: Path, id: str): - """ - Generates a leaf certificate signed by the CA for a specific image clone. - - Args: - ca_nss_key_db: Path to the CA's NSS key database - id: ID of the signing key for this image clone - - Returns: - leaf_key_name: Name of the leaf key that was generated - - Raises: - Exception: If certificate generation fails. - """ - # Generate unique leaf key name - leaf_key_name = f"{KEY_NAME}_{id}" - - cmd = [ - "efikeygen", - "-n", - leaf_key_name, - "-c", - f"CN={KEY_CN} {id}", - "--signer", - CA_NAME, - "-d", - str(ca_nss_key_db), - ] - - if _efikeygen_supports_kernel_flag(): - cmd.append("--kernel") - - # Generate signing key/cert, signed by CA in the shared DB - subprocess.run(cmd, check=True) - - log.debug( - f"Process with PID {threading.get_ident()} generated leaf key {leaf_key_name} in {ca_nss_key_db}" - ) - return leaf_key_name - - -def publish_ca_certificate(ca_nss_key_db: Path, output_dir: Path): - """ - Extract and publish the CA certificate that can validate all leaf certificates. - - Args: - ca_nss_key_db: Path to the CA's NSS key database - output_dir: Directory where the CA certificate will be published - - Raises: - Exception: If any shell command fails. - """ - # Export PKCS#12 from NSS DB - key_path = ca_nss_key_db / PKCS12_ARCH_FILE - subprocess.run( - [ - "pk12util", - "-d", - str(ca_nss_key_db), - "-n", - CA_NAME, - "-o", - str(key_path), - "-W", - "", - ], - check=True, - ) - - # Extract certificate from PKCS#12 file, no private key - ca_cert_path = output_dir / "ca_cert.pem" - subprocess.run( - [ - "openssl", - "pkcs12", - "-in", - str(key_path), - "-out", - str(ca_cert_path), - "-nokeys", - "-passin", - "pass:", - ], - check=True, - ) - - log.info(f"CA Certificate published to {ca_cert_path}") - - -def sign_boot_artifacts( - ca_nss_key_db: Path, - leaf_key_name: str, - inject_files_yaml_path: Path, - output_artifacts_dir: Path, -): - """ - Signs unsigned boot artifacts listed in inject-files.yaml and produces signed boot artifacts. - - Args: - ca_nss_key_db: Path to the NSS key database for the CA certificate - leaf_key_name: Name of the leaf certificate - inject_files_yaml_path: Full path to inject-files.yaml - output_artifacts_dir: Dir where artifacts are output by Image Customizer - """ - with open(inject_files_yaml_path, "r") as f: - data = f.read() - - log.debug(f"Contents of {inject_files_yaml_path}:\n{data}") - inject_files_config = yaml.safe_load(data) - - for entry in inject_files_config.get("injectFiles", []): - artifact_type = entry.get("type", "") - signed_path_str = entry.get("source") - unsigned_path_str = entry.get("unsignedSource", "") - - if not signed_path_str: - raise RuntimeError(f"Missing source in inject-files entry: {entry}") - - if not unsigned_path_str: - # MIC v1.1+ uses the same path for unsigned and signed files. - unsigned_path_str = signed_path_str - - signed_rel_path = ( - signed_path_str[2:] if signed_path_str.startswith("./") else signed_path_str - ) - unsigned_rel_path = ( - unsigned_path_str[2:] - if unsigned_path_str.startswith("./") - else unsigned_path_str - ) - signed_path = output_artifacts_dir.absolute() / signed_rel_path - unsigned_path = output_artifacts_dir.absolute() / unsigned_rel_path - - if not artifact_type: - artifact_type = get_artifact_type_from_name(unsigned_path.name) - - log.info( - f"Signing file of type '{artifact_type}' at {unsigned_path} to {signed_path}" - ) - if artifact_type == IC_ARTIFACT_NAME_VERITY_HASH: - sign_verity_hash( - ca_nss_key_db, - leaf_key_name, - unsigned_path, - signed_path, - ) - else: - sign_pe_artifact( - ca_nss_key_db, - leaf_key_name, - unsigned_path, - signed_path, - ) - - -def get_artifact_type_from_name(name: str) -> Optional[str]: - if re.match(r"vmlinuz.*\.efi", name): - return IC_ARTIFACT_NAME_UKIS - if re.match(r"bootx64\.efi", name): - return IC_ARTIFACT_NAME_SHIM - if re.match(r"systemd-bootx64\.efi", name): - return IC_ARTIFACT_NAME_SYSTEMD_BOOT - if re.match(r".*hash.*", name): - return IC_ARTIFACT_NAME_VERITY_HASH - - return None - - -def sign_verity_hash( - ca_nss_key_db: Path, - leaf_key_name: str, - unsigned_verity_hash_path: Path, - signed_verity_hash_path: Path, -): - """ - Sign the verity hash file using the signing key. - - Args: - ca_nss_key_db: Path to the NSS key database for the CA certificate - leaf_key_name: Name of the leaf certificate - unsigned_verity_hash_path: Path to the unsigned verity hash file - signed_verity_hash_path: Path to the signed verity hash file - - Raises: - Exception: If pesign fails. - """ - log.debug( - f"Process with PID {threading.get_ident()} is signing {unsigned_verity_hash_path}" - ) - - signed_verity_hash_path.parent.mkdir(parents=True, exist_ok=True) - signed_verity_hash_path.parent.chmod(0o700) - - with temp_dir(sudo=True) as tmpdir: - tmp_signed_artifact = ( - tmpdir - / f"{unsigned_verity_hash_path.stem}.signed{unsigned_verity_hash_path.suffix}" - ) - tmp_unsigned_artifact = ( - tmpdir - / f"{unsigned_verity_hash_path.stem}.unsigned{unsigned_verity_hash_path.suffix}" - ) - key_path = tmpdir / "key.p12" - key_crt_path = tmpdir / "key.crt" - - subprocess.run( - ["sudo", "cp", str(unsigned_verity_hash_path), str(tmp_unsigned_artifact)], - check=True, - ) - - log.debug(f"Exporting PKCS12 key to {key_path}") - subprocess.run( - [ - "pk12util", - "-d", - str(ca_nss_key_db), - "-n", - leaf_key_name, - "-o", - str(key_path), - "-W", - "", - ], - check=True, - ) - - log.debug(f"Extracting cert from PKCS12 key to {key_crt_path}") - subprocess.run( - [ - "openssl", - "pkcs12", - "-in", - str(key_path), - "-out", - str(key_crt_path), - "-clcerts", - "-nodes", - "-passin", - "pass:", - ], - check=True, - ) - - log.debug( - f"Signing verity hash file at {tmp_unsigned_artifact} using openssl smime" - ) - subprocess.run( - [ - "openssl", - "smime", - "-sign", - "-noattr", - "-binary", - "-in", - str(tmp_unsigned_artifact), - "-signer", - str(key_crt_path), - "-passin", - "pass:", - "-outform", - "der", - "-out", - str(tmp_signed_artifact), - ], - check=True, - ) - - try: - result = subprocess.run( - [ - "openssl", - "pkcs7", - "-inform", - "DER", - "-in", - str(tmp_signed_artifact), - "-print_certs", - "-text", - ], - check=True, - capture_output=True, - text=True, - ) - log.debug(f"Certs for {unsigned_verity_hash_path}:\n{result.stdout}") - except subprocess.CalledProcessError as e: - log.error(f"Failed to print certs for {unsigned_verity_hash_path}: {e}") - - subprocess.run( - ["sudo", "cp", str(tmp_signed_artifact), str(signed_verity_hash_path)], - check=True, - ) - - log.debug(f"Signed verity-hash artifact generated at {signed_verity_hash_path}") - - -def sign_pe_artifact( - ca_nss_key_db: Path, - leaf_key_name: str, - unsigned_artifact_path: Path, - signed_artifact_path: Path, -): - """ - Sign the artifact using the signing key. - - Args: - ca_nss_key_db: Path to the NSS key database for the CA certificate - leaf_key_name: Name of the leaf certificate - unsigned_artifact_path: Path to the unsigned artifact - signed_artifact_path: Path to the signed artifact - - Raises: - Exception: If pesign fails. - """ - log.debug( - f"Process with PID {threading.get_ident()} is signing {unsigned_artifact_path} to {signed_artifact_path}" - ) - - with temp_dir(sudo=True) as tmpdir: - tmp_signed_artifact = ( - tmpdir - / f"{unsigned_artifact_path.stem}.signed{unsigned_artifact_path.suffix}" - ) - tmp_unsigned_artifact = ( - tmpdir - / f"{unsigned_artifact_path.stem}.unsigned{unsigned_artifact_path.suffix}" - ) - - subprocess.run( - ["sudo", "cp", str(unsigned_artifact_path), str(tmp_unsigned_artifact)], - check=True, - ) - - cert_arg = _get_pesign_certificate_arg() - - subprocess.run( - [ - "pesign", - "--certdir", - str(ca_nss_key_db), - cert_arg, - leaf_key_name, - "--sign", - "--in", - str(tmp_unsigned_artifact), - "--out", - str(tmp_signed_artifact), - "--force", - ], - check=True, - ) - - subprocess.run( - ["sudo", "cp", str(tmp_signed_artifact), str(signed_artifact_path)], - check=True, - ) - - log.debug(f"Artifact signed to {signed_artifact_path}") diff --git a/tests/images/builder/utils.py b/tests/images/builder/utils.py deleted file mode 100644 index 33692361ee..0000000000 --- a/tests/images/builder/utils.py +++ /dev/null @@ -1,24 +0,0 @@ -from pathlib import Path - -BUILD_DIR = "/tmp" -HOST_PATH = Path("/host") - - -def build_path(path: Path) -> Path: - """ - Convert a host filesystem path to its corresponding path inside the Docker container. - - The Docker container mounts the host's root filesystem at /host, so this function - transforms absolute host paths like '/home/user/file.txt' to container paths - like '/host/home/user/file.txt'. - - Args: - path: Absolute or relative path on the host filesystem - - Returns: - Path that can be used inside the Docker container to access the same file - - Example: - build_path(Path("/home/user/config.yaml")) -> Path("/host/home/user/config.yaml") - """ - return HOST_PATH / path.absolute().relative_to(Path("/")) diff --git a/tests/images/foreign-direct-streaming-testimage/by-arch+source/arm64+ubuntu-2204.yaml b/tests/images/foreign-direct-streaming-testimage/by-arch+source/arm64+ubuntu-2204.yaml new file mode 100644 index 0000000000..3f1dcbc0e4 --- /dev/null +++ b/tests/images/foreign-direct-streaming-testimage/by-arch+source/arm64+ubuntu-2204.yaml @@ -0,0 +1,3 @@ +base: + $set: + ref: ubuntu_2204_arm64 diff --git a/tests/images/foreign-direct-streaming-testimage/by-arch+source/arm64+ubuntu-2404.yaml b/tests/images/foreign-direct-streaming-testimage/by-arch+source/arm64+ubuntu-2404.yaml new file mode 100644 index 0000000000..94f6b20459 --- /dev/null +++ b/tests/images/foreign-direct-streaming-testimage/by-arch+source/arm64+ubuntu-2404.yaml @@ -0,0 +1,3 @@ +base: + $set: + ref: ubuntu_2404_arm64 diff --git a/tests/images/foreign-direct-streaming-testimage/by-source/gb200-2404.yaml b/tests/images/foreign-direct-streaming-testimage/by-source/gb200-2404.yaml new file mode 100644 index 0000000000..6799181856 --- /dev/null +++ b/tests/images/foreign-direct-streaming-testimage/by-source/gb200-2404.yaml @@ -0,0 +1,3 @@ +base: + $set: + ref: gb200_2404_arm64 diff --git a/tests/images/foreign-direct-streaming-testimage/by-source/ubuntu-2404.yaml b/tests/images/foreign-direct-streaming-testimage/by-source/ubuntu-2404.yaml new file mode 100644 index 0000000000..1fc2ef81f2 --- /dev/null +++ b/tests/images/foreign-direct-streaming-testimage/by-source/ubuntu-2404.yaml @@ -0,0 +1,3 @@ +base: + $set: + ref: ubuntu_2404_amd64 diff --git a/tests/images/foreign-direct-streaming-testimage/image.yaml b/tests/images/foreign-direct-streaming-testimage/image.yaml new file mode 100644 index 0000000000..3d006bffb6 --- /dev/null +++ b/tests/images/foreign-direct-streaming-testimage/image.yaml @@ -0,0 +1,16 @@ +name: foreign-direct-streaming-testimage +operation: convert + +matrix: + arch: [amd64, arm64] + source: [ubuntu-2204, ubuntu-2404, gb200-2404] + +selectors: + exclude: + - { arch: amd64, source: gb200-2404 } + +base: + ref: ubuntu_2204_amd64 + +outputs: + - format: baremetal-image diff --git a/tests/images/legacy-map.json b/tests/images/legacy-map.json new file mode 100644 index 0000000000..b87725ff5c --- /dev/null +++ b/tests/images/legacy-map.json @@ -0,0 +1,263 @@ +{ + "trident-installer": { + "image": "trident-installer", + "selectors": { + "arch": "amd64", + "variant": "default" + }, + "baseImage": "baremetal", + "ext": "iso" + }, + "trident-split-installer": { + "image": "trident-installer", + "selectors": { + "arch": "amd64", + "variant": "split" + }, + "baseImage": "baremetal", + "ext": "iso" + }, + "trident-installer-arm64": { + "image": "trident-installer", + "selectors": { + "arch": "arm64", + "variant": "default" + }, + "baseImage": "core_arm64", + "ext": "iso" + }, + "trident-container-installer": { + "image": "trident-container-installer", + "selectors": {}, + "baseImage": "baremetal", + "ext": "iso" + }, + "trident-direct-streaming-installer-amd64": { + "image": "trident-installer", + "selectors": { + "arch": "amd64", + "variant": "direct-streaming" + }, + "baseImage": "baremetal", + "ext": "iso" + }, + "trident-direct-streaming-installer-arm64": { + "image": "trident-installer", + "selectors": { + "arch": "arm64", + "variant": "direct-streaming" + }, + "baseImage": "core_arm64", + "ext": "iso" + }, + "trident-functest": { + "image": "trident-functest", + "selectors": {}, + "baseImage": "baremetal", + "ext": "qcow2" + }, + "trident-testimage": { + "image": "trident-testimage", + "selectors": { + "arch": "amd64" + }, + "baseImage": "baremetal", + "ext": "cosi" + }, + "trident-testimage-arm64": { + "image": "trident-testimage", + "selectors": { + "arch": "arm64" + }, + "baseImage": "core_arm64", + "ext": "cosi" + }, + "trident-verity-testimage": { + "image": "trident-verity-testimage", + "selectors": { + "deployment": "host", + "mode": "root" + }, + "baseImage": "baremetal", + "ext": "cosi" + }, + "trident-usrverity-testimage": { + "image": "trident-verity-testimage", + "selectors": { + "deployment": "host", + "mode": "usr" + }, + "baseImage": "baremetal", + "ext": "cosi" + }, + "trident-container-verity-testimage": { + "image": "trident-verity-testimage", + "selectors": { + "deployment": "container", + "mode": "root" + }, + "baseImage": "baremetal", + "ext": "cosi" + }, + "trident-container-usrverity-testimage": { + "image": "trident-verity-testimage", + "selectors": { + "deployment": "container", + "mode": "usr" + }, + "baseImage": "baremetal", + "ext": "cosi" + }, + "trident-container-testimage": { + "image": "trident-container-testimage", + "selectors": {}, + "baseImage": "baremetal", + "ext": "cosi" + }, + "azurelinux-direct-streaming-testimage-amd64": { + "image": "azurelinux-direct-streaming-testimage", + "selectors": { + "arch": "amd64" + }, + "baseImage": "baremetal", + "ext": "cosi", + "tailorExt": "raw" + }, + "azurelinux-direct-streaming-testimage-arm64": { + "image": "azurelinux-direct-streaming-testimage", + "selectors": { + "arch": "arm64" + }, + "baseImage": "core_arm64", + "ext": "cosi", + "tailorExt": "raw" + }, + "azl-installer": { + "image": "azl-installer", + "selectors": {}, + "baseImage": "baremetal", + "ext": "iso" + }, + "trident-vm-grub-testimage": { + "image": "trident-vm-testimage", + "selectors": { + "arch": "amd64", + "scenario": "grub" + }, + "baseImage": "qemu_guest", + "ext": "cosi" + }, + "trident-vm-grub-verity-testimage": { + "image": "trident-vm-testimage", + "selectors": { + "arch": "amd64", + "scenario": "grub-verity" + }, + "baseImage": "qemu_guest", + "ext": "cosi" + }, + "trident-vm-root-verity-testimage": { + "image": "trident-vm-testimage", + "selectors": { + "arch": "amd64", + "scenario": "root-verity" + }, + "baseImage": "qemu_guest", + "ext": "cosi" + }, + "trident-vm-usr-verity-testimage": { + "image": "trident-vm-testimage", + "selectors": { + "arch": "amd64", + "scenario": "usr-verity" + }, + "baseImage": "qemu_guest", + "ext": "cosi" + }, + "trident-vm-acl-agent-update-testimage": { + "image": "trident-vm-testimage", + "selectors": { + "arch": "amd64", + "scenario": "acl-agent" + }, + "baseImage": "qemu_guest", + "ext": "cosi" + }, + "trident-vm-grub-verity-azure-testimage": { + "image": "trident-vm-testimage", + "selectors": { + "arch": "amd64", + "scenario": "grub-verity-azure" + }, + "baseImage": "core_selinux", + "ext": "cosi" + }, + "trident-vm-grub-testimage-arm64": { + "image": "trident-vm-testimage", + "selectors": { + "arch": "arm64", + "scenario": "grub" + }, + "baseImage": "core_arm64", + "ext": "cosi" + }, + "trident-vm-grub-verity-testimage-arm64": { + "image": "trident-vm-testimage", + "selectors": { + "arch": "arm64", + "scenario": "grub-verity" + }, + "baseImage": "core_arm64", + "ext": "cosi" + }, + "ubuntu-direct-streaming-testimage-2204-amd64": { + "image": "foreign-direct-streaming-testimage", + "selectors": { + "arch": "amd64", + "source": "ubuntu-2204" + }, + "baseImage": "ubuntu_2204_amd64", + "ext": "cosi", + "tailorExt": "raw" + }, + "ubuntu-direct-streaming-testimage-2204-arm64": { + "image": "foreign-direct-streaming-testimage", + "selectors": { + "arch": "arm64", + "source": "ubuntu-2204" + }, + "baseImage": "ubuntu_2204_arm64", + "ext": "cosi", + "tailorExt": "raw" + }, + "ubuntu-direct-streaming-testimage-2404-amd64": { + "image": "foreign-direct-streaming-testimage", + "selectors": { + "arch": "amd64", + "source": "ubuntu-2404" + }, + "baseImage": "ubuntu_2404_amd64", + "ext": "cosi", + "tailorExt": "raw" + }, + "ubuntu-direct-streaming-testimage-2404-arm64": { + "image": "foreign-direct-streaming-testimage", + "selectors": { + "arch": "arm64", + "source": "ubuntu-2404" + }, + "baseImage": "ubuntu_2404_arm64", + "ext": "cosi", + "tailorExt": "raw" + }, + "gb200-direct-streaming-testimage-2404-arm64": { + "image": "foreign-direct-streaming-testimage", + "selectors": { + "arch": "arm64", + "source": "gb200-2404" + }, + "baseImage": "gb200_2404_arm64", + "ext": "cosi", + "tailorExt": "raw" + } +} diff --git a/tests/images/legacy_targets.py b/tests/images/legacy_targets.py new file mode 100755 index 0000000000..5e7d7b620b --- /dev/null +++ b/tests/images/legacy_targets.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +import argparse +import json +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +WORKSPACE = REPO_ROOT / "tests" / "images" / "tailor.yaml" +LEGACY_MAP = REPO_ROOT / "tests" / "images" / "legacy-map.json" +SELF = Path(__file__).resolve() + + +def load_map(): + return json.loads(LEGACY_MAP.read_text()) + + +def cargo_cmd() -> str: + cargo = shutil.which("cargo") + if cargo: + return cargo + fallback = Path.home() / ".cargo" / "bin" / "cargo" + if fallback.exists(): + return str(fallback) + raise SystemExit( + "cargo not found: install a Rust toolchain (see " + "../common_tasks/rustup.yml in CI) or ensure cargo is on PATH" + ) + + +def default_container_full() -> str: + text = WORKSPACE.read_text() + needle = """ entries:\n - name: ic\n container: """ + idx = text.find(needle) + if idx == -1: + raise SystemExit("default toolchain not found in tests/images/tailor.yaml") + rest = text[idx + len(needle) :].splitlines() + container = rest[0].strip() + tag = None + for line in rest[1:6]: + s = line.strip() + if s.startswith("tag:"): + tag = s.split(":", 1)[1].strip() + break + if tag: + return f"{container}:{tag}" + return container + + +def tailor_manifest_for_container(container_ref: str | None) -> Path: + if not container_ref or container_ref == default_container_full(): + return WORKSPACE + text = WORKSPACE.read_text() + name = "ic-override" + if ":" in container_ref.rsplit("/", 1)[-1]: + container, tag = container_ref.rsplit(":", 1) + tag_block = f" tag: {tag}\n" + else: + container = container_ref + tag_block = "" + needle = """toolchains:\n default: ic\n entries:\n - name: ic\n container: mcr.microsoft.com/azurelinux/imagecustomizer\n tag: latest\n""" + replacement = ( + "toolchains:\n" + f" default: {name}\n" + " entries:\n" + " - name: ic\n" + " container: mcr.microsoft.com/azurelinux/imagecustomizer\n" + " tag: latest\n" + f" - name: {name}\n" + f" container: {container}\n" + f"{tag_block}" + " pull: never\n" + ) + rendered = text.replace(needle, replacement, 1) + tmp = REPO_ROOT / "tests" / "images" / ".tailor.make.yaml" + tmp.write_text(rendered) + return tmp + + +def selector_args(entry: dict) -> list[str]: + args: list[str] = [] + for key, value in entry["selectors"].items(): + args.extend(["-s", f"{key}={value}"]) + return args + + +def call_tailor(*args: str, capture: bool = False) -> subprocess.CompletedProcess: + cmd = [ + cargo_cmd(), + "run", + "--manifest-path", + "tools/tailor/Cargo.toml", + "--quiet", + "--", + *args, + ] + env = dict(os.environ) + env["PATH"] = f"{Path.home() / '.cargo' / 'bin'}:{env.get('PATH','')}" + return subprocess.run( + cmd, cwd=str(REPO_ROOT), check=True, text=True, capture_output=capture, env=env + ) + + +def legacy_list(filter_type: str): + for name, entry in load_map().items(): + if entry["ext"] == filter_type: + print(name) + + +def dependencies(name: str): + mapping = load_map() + entry = mapping[name] + image_dir = REPO_ROOT / "tests" / "images" / entry["image"] + deps: list[Path] = [WORKSPACE, LEGACY_MAP, SELF] + base_dep_map = { + "baremetal": REPO_ROOT / "artifacts" / "baremetal.vhdx", + "core_selinux": REPO_ROOT / "artifacts" / "core_selinux.vhdx", + "core_arm64": REPO_ROOT / "artifacts" / "core_arm64.vhdx", + "qemu_guest": REPO_ROOT / "artifacts" / "qemu_guest.vhdx", + "ubuntu_2204_amd64": REPO_ROOT / "artifacts" / "ubuntu_2204_amd64.vhdx", + "ubuntu_2204_arm64": REPO_ROOT / "artifacts" / "ubuntu_2204_arm64.vhdx", + "ubuntu_2404_amd64": REPO_ROOT / "artifacts" / "ubuntu_2404_amd64.vhdx", + "ubuntu_2404_arm64": REPO_ROOT / "artifacts" / "ubuntu_2404_arm64.vhdx", + "gb200_2404_arm64": REPO_ROOT / "artifacts" / "gb200_2404_arm64.vhdx", + } + base_dep = base_dep_map.get(entry.get("baseImage")) + if base_dep is not None: + deps.append(base_dep) + for path in sorted(image_dir.rglob("*")): + if ( + path.is_file() + and ".rendered" not in path.parts + and path.name != ".tailor.make.yaml" + ): + deps.append(path) + needs_rpms = name not in { + "trident-functest", + "trident-container-installer", + "trident-container-testimage", + "trident-container-verity-testimage", + "trident-container-usrverity-testimage", + "ubuntu-direct-streaming-testimage-2204-amd64", + "ubuntu-direct-streaming-testimage-2204-arm64", + "ubuntu-direct-streaming-testimage-2404-amd64", + "ubuntu-direct-streaming-testimage-2404-arm64", + "gb200-direct-streaming-testimage-2404-arm64", + } + if needs_rpms: + rpm_dir = REPO_ROOT / "bin" / "RPMS" + if rpm_dir.exists(): + deps.append(rpm_dir) + deps.extend(sorted(rpm_dir.rglob("*.rpm"))) + if name in { + "trident-direct-streaming-installer-amd64", + "trident-direct-streaming-installer-arm64", + }: + for extra in [ + REPO_ROOT / "artifacts" / "rcp-agent", + REPO_ROOT / "artifacts" / "rcp-agent.service", + ]: + deps.append(extra) + if entry["image"] == "trident-vm-testimage": + # IC configs under trident-vm-testimage/ reference base/files/id_rsa.pub + # directly; the Makefile's own rule (keyed off artifacts/id_rsa[.pub], + # staged by the pipeline or `make artifacts/id_rsa`) copies the staged + # key into place. Without this dependency, make never runs that rule + # and IC fails with "failed to find SSH public key file". + ssh_key = image_dir / "base" / "files" / "id_rsa.pub" + deps.append(ssh_key) + if name == "azl-installer": + for extra in [ + REPO_ROOT + / "tests" + / "images" + / "azl-installer" + / "iso" + / "bin" + / "liveinstaller", + REPO_ROOT + / "tests" + / "images" + / "azl-installer" + / "iso" + / "images" + / "trident-testimage.cosi", + ]: + if extra.exists(): + deps.append(extra) + seen = set() + for dep in deps: + try: + s = str(dep.relative_to(REPO_ROOT)) + except ValueError: + s = str(dep) + if s not in seen: + seen.add(s) + print(s) + + +def resolve_slug(entry: dict, manifest: Path) -> str: + proc = call_tailor( + "--manifest", + str(manifest), + "matrix", + entry["image"], + "--format", + "json", + *selector_args(entry), + capture=True, + ) + cells = json.loads(proc.stdout) + if len(cells) != 1: + raise SystemExit( + f"expected exactly one cell for {entry['image']}, got {len(cells)}" + ) + return cells[0]["slug"] + + +def build(name: str, output_path: str, container: str | None): + mapping = load_map() + entry = mapping[name] + target = Path(output_path) + target.parent.mkdir(parents=True, exist_ok=True) + manifest = tailor_manifest_for_container(container) + try: + slug = resolve_slug(entry, manifest) + cert_path = REPO_ROOT / "tests" / "images" / "artifacts" / "ca_cert.pem" + cert_path.unlink(missing_ok=True) + call_tailor( + "--manifest", + str(manifest), + "build", + entry["image"], + *selector_args(entry), + "--output-dir", + str(target.parent), + ) + source_ext = entry.get("tailorExt", entry["ext"]) + built = target.parent / f"{slug}.{source_ext}" + if not built.exists(): + raise SystemExit(f"expected built artifact not found: {built}") + built.replace(target) + if cert_path.exists(): + shutil.copy2(cert_path, target.parent / "ca_cert.pem") + finally: + if manifest.name == ".tailor.make.yaml": + manifest.unlink(missing_ok=True) + + +def oras_download(name: str, out_path: Path): + ref = f"mcr.microsoft.com/azurelinux/3.0/image/{name}:latest" + with tempfile.TemporaryDirectory() as td: + subprocess.run( + ["oras", "pull", ref, "--output", td, "--platform", "linux/amd64"], + check=True, + ) + files = list(Path(td).glob("*.vhdx")) + if len(files) != 1: + raise SystemExit(f"expected one .vhdx from {ref}, got {len(files)}") + out_path.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(files[0]), str(out_path)) + + +def download_image(name: str): + target_map = { + "baremetal": REPO_ROOT / "artifacts" / "baremetal.vhdx", + "core_selinux": REPO_ROOT / "artifacts" / "core_selinux.vhdx", + "minimal": REPO_ROOT / "artifacts" / "minimal.vhdx", + "minimal_aarch64": REPO_ROOT / "artifacts" / "minimal_aarch64.vhdx", + } + if name not in target_map: + raise SystemExit(f"unsupported base image: {name}") + oras_download(name, target_map[name]) + + +def main(): + ap = argparse.ArgumentParser() + sub = ap.add_subparsers(dest="cmd", required=True) + + p = sub.add_parser("list") + p.add_argument("--filter-type", required=True) + + p = sub.add_parser("dependencies") + p.add_argument("name") + + p = sub.add_parser("build") + p.add_argument("name") + p.add_argument("--output-path", required=True) + p.add_argument("--container") + + p = sub.add_parser("show-artifact") + p.add_argument("item") + + p = sub.add_parser("download-image") + p.add_argument("name") + + args = ap.parse_args() + if args.cmd == "list": + legacy_list(args.filter_type) + elif args.cmd == "dependencies": + dependencies(args.name) + elif args.cmd == "build": + build(args.name, args.output_path, args.container) + elif args.cmd == "show-artifact": + if args.item != "customizer-container-full": + raise SystemExit(f"unsupported artifact item: {args.item}") + print(default_container_full()) + elif args.cmd == "download-image": + download_image(args.name) + + +if __name__ == "__main__": + import os + + main() diff --git a/tests/images/tailor.yaml b/tests/images/tailor.yaml new file mode 100644 index 0000000000..2d7b1d5a75 --- /dev/null +++ b/tests/images/tailor.yaml @@ -0,0 +1,76 @@ +schemaVersion: 1 + +previewFeatures: + - signing + +toolchains: + default: ic + entries: + - name: ic + container: mcr.microsoft.com/azurelinux/imagecustomizer + tag: latest + +signing: + default: test-ca + profiles: + test-ca: + backend: local-test-ca + publishCaCert: ./artifacts/ca_cert.pem + +baseImages: + - name: baremetal + path: ../../artifacts/baremetal.vhdx + arch: amd64 + source: + oci: + uri: mcr.microsoft.com/azurelinux/3.0/image/baremetal:latest + platform: linux/amd64 + - name: core_arm64 + path: ../../artifacts/core_arm64.vhdx + arch: arm64 + source: + oci: + uri: mcr.microsoft.com/azurelinux/3.0/image/core_arm64:latest + platform: linux/arm64 + - name: core_selinux + path: ../../artifacts/core_selinux.vhdx + arch: amd64 + source: + oci: + uri: mcr.microsoft.com/azurelinux/3.0/image/core_selinux:latest + platform: linux/amd64 + - name: qemu_guest + path: ../../artifacts/qemu_guest.vhdx + arch: amd64 + source: + oci: + uri: mcr.microsoft.com/azurelinux/3.0/image/qemu_guest:latest + platform: linux/amd64 + - name: ubuntu_2204_amd64 + path: ../../artifacts/ubuntu_2204_amd64.vhdx + arch: amd64 + - name: ubuntu_2204_arm64 + path: ../../artifacts/ubuntu_2204_arm64.vhdx + arch: arm64 + - name: ubuntu_2404_amd64 + path: ../../artifacts/ubuntu_2404_amd64.vhdx + arch: amd64 + - name: ubuntu_2404_arm64 + path: ../../artifacts/ubuntu_2404_arm64.vhdx + arch: arm64 + - name: gb200_2404_arm64 + path: ../../artifacts/gb200_2404_arm64.vhdx + arch: arm64 + +images: + members: + - azl-installer + - azurelinux-direct-streaming-testimage + - foreign-direct-streaming-testimage + - trident-container-installer + - trident-container-testimage + - trident-functest + - trident-installer + - trident-testimage + - trident-verity-testimage + - trident-vm-testimage diff --git a/tests/images/testimages.py b/tests/images/testimages.py deleted file mode 100755 index a5137a2114..0000000000 --- a/tests/images/testimages.py +++ /dev/null @@ -1,267 +0,0 @@ -#!/usr/bin/env python3 - -from pathlib import Path -from typing import List - -from builder import ( - ArtifactManifest, - BaseImage, - BaseImageManifest, - ImageConfig, - OutputFormat, - SystemArchitecture, - cli, -) - -# # # # # # # # # # # # # # # # # # # -# DEFAULT CUSTOMIZER VERSION # -# # -# The version of Image Customizer # -# to use by default when building # -# images. # -# # # # # # # # # # # # # # # # # # # -DEFAULT_IMAGE_CUSTOMIZER_VERSION = "latest" - - -DEFINED_IMAGES: List[ImageConfig] = [ - # Installer images - ImageConfig( - "trident-installer", - config="trident-installer", - output_format=OutputFormat.ISO, - ), - ImageConfig( - "trident-split-installer", - config="trident-installer", - config_file="base/baseimg-split.yaml", - output_format=OutputFormat.ISO, - ), - ImageConfig( - "trident-installer-arm64", - config="trident-installer", - output_format=OutputFormat.ISO, - base_image=BaseImage.CORE_ARM64, - architecture=SystemArchitecture.ARM64, - ), - ImageConfig( - "trident-container-installer", - config="trident-container-installer", - output_format=OutputFormat.ISO, - requires_trident=False, - ), - ImageConfig( - "trident-direct-streaming-installer-amd64", - config="trident-installer", - config_file="base/baseimg-direct-streaming.yaml", - output_format=OutputFormat.ISO, - ), - ImageConfig( - "trident-direct-streaming-installer-arm64", - config="trident-installer", - config_file="base/baseimg-direct-streaming.yaml", - output_format=OutputFormat.ISO, - base_image=BaseImage.CORE_ARM64, - architecture=SystemArchitecture.ARM64, - ), - # Test images - ImageConfig( - "trident-functest", - output_format=OutputFormat.QCOW2, - requires_trident=False, - ), - ImageConfig("trident-testimage"), - ImageConfig( - "trident-testimage-arm64", - config="trident-testimage", - base_image=BaseImage.CORE_ARM64, - architecture=SystemArchitecture.ARM64, - ), - ImageConfig("trident-verity-testimage"), - ImageConfig( - "trident-usrverity-testimage", - config="trident-verity-testimage", - config_file="usr/host.yaml", - requires_ukify=True, - ), - ImageConfig( - "trident-container-verity-testimage", - config="trident-verity-testimage", - config_file="base/baseimg-container.yaml", - requires_trident=False, - ), - ImageConfig( - "trident-container-usrverity-testimage", - config="trident-verity-testimage", - config_file="usr/container.yaml", - requires_ukify=True, - requires_trident=False, - ), - ImageConfig( - "trident-container-testimage", - requires_trident=False, - ), - # Direct streaming images - ImageConfig( - "azurelinux-direct-streaming-testimage-amd64", - config="azurelinux-direct-streaming-testimage", - output_format=OutputFormat.BAREMETAL_IMAGE, - ), - ImageConfig( - "azurelinux-direct-streaming-testimage-arm64", - config="azurelinux-direct-streaming-testimage", - output_format=OutputFormat.BAREMETAL_IMAGE, - base_image=BaseImage.CORE_ARM64, - architecture=SystemArchitecture.ARM64, - ), - # AZL installer - ImageConfig( - "azl-installer", - config_file=Path("installer-iso.yaml"), - output_format=OutputFormat.ISO, - requires_trident=True, - extra_dependencies=[ - Path("tests/images/azl-installer/iso/bin/liveinstaller"), - Path("tests/images/azl-installer/iso/images/trident-testimage.cosi"), - ], - ), - # VM test images - ImageConfig( - "trident-vm-grub-testimage", - base_image=BaseImage.QEMU_GUEST, - config="trident-vm-testimage", - config_file="base/updateimg-grub.yaml", - ssh_key="files/id_rsa.pub", - ), - ImageConfig( - "trident-vm-grub-verity-testimage", - base_image=BaseImage.QEMU_GUEST, - config="trident-vm-testimage", - config_file="base/updateimg-grub-verity.yaml", - ssh_key="files/id_rsa.pub", - ), - ImageConfig( - "trident-vm-root-verity-testimage", - base_image=BaseImage.QEMU_GUEST, - config="trident-vm-testimage", - config_file="base/baseimg-root-verity.yaml", - requires_ukify=True, - ssh_key="files/id_rsa.pub", - ), - ImageConfig( - "trident-vm-usr-verity-testimage", - base_image=BaseImage.QEMU_GUEST, - config="trident-vm-testimage", - config_file="base/baseimg-usr-verity.yaml", - requires_ukify=True, - ssh_key="files/id_rsa.pub", - ), - ImageConfig( - "trident-vm-acl-agent-update-testimage", - base_image=BaseImage.QEMU_GUEST, - config="trident-vm-testimage", - config_file="base/updateimg-acl-agent.yaml", - requires_ukify=True, - ssh_key="files/id_rsa.pub", - ), - ImageConfig( - "trident-vm-grub-verity-azure-testimage", - base_image=BaseImage.CORE_SELINUX, - config="trident-vm-testimage", - config_file="base/updateimg-grub-verity-azure.yaml", - ), - ImageConfig( - "trident-vm-grub-testimage-arm64", - base_image=BaseImage.CORE_ARM64, - config="trident-vm-testimage", - config_file="base/updateimg-grub.yaml", - ssh_key="files/id_rsa.pub", - architecture=SystemArchitecture.ARM64, - ), - ImageConfig( - "trident-vm-grub-verity-testimage-arm64", - base_image=BaseImage.CORE_ARM64, - config="trident-vm-testimage", - config_file="base/updateimg-grub-verity.yaml", - ssh_key="files/id_rsa.pub", - architecture=SystemArchitecture.ARM64, - ), - ImageConfig( - "ubuntu-direct-streaming-testimage-2204-amd64", - base_image=BaseImage.UBUNTU_2204_AMD64, - output_format=OutputFormat.BAREMETAL_IMAGE, - image_customizer_convert=True, - requires_trident=False, - ), - ImageConfig( - "ubuntu-direct-streaming-testimage-2204-arm64", - base_image=BaseImage.UBUNTU_2204_ARM64, - output_format=OutputFormat.BAREMETAL_IMAGE, - architecture=SystemArchitecture.ARM64, - image_customizer_convert=True, - requires_trident=False, - ), - ImageConfig( - "ubuntu-direct-streaming-testimage-2404-amd64", - base_image=BaseImage.UBUNTU_2404_AMD64, - output_format=OutputFormat.BAREMETAL_IMAGE, - image_customizer_convert=True, - requires_trident=False, - ), - ImageConfig( - "ubuntu-direct-streaming-testimage-2404-arm64", - base_image=BaseImage.UBUNTU_2404_ARM64, - output_format=OutputFormat.BAREMETAL_IMAGE, - architecture=SystemArchitecture.ARM64, - image_customizer_convert=True, - requires_trident=False, - ), - ImageConfig( - "gb200-direct-streaming-testimage-2404-arm64", - base_image=BaseImage.GB200_2404_ARM64, - output_format=OutputFormat.BAREMETAL_IMAGE, - architecture=SystemArchitecture.ARM64, - image_customizer_convert=True, - requires_trident=False, - ), -] - -ARTIFACTS = ArtifactManifest( - customizer_version=DEFAULT_IMAGE_CUSTOMIZER_VERSION, - customizer_container="mcr.microsoft.com/azurelinux/imagecustomizer", - base_images=[ - BaseImageManifest( - image=BaseImage.BAREMETAL, - package_name="baremetal_vhdx-3.0-stable", - version="*", - ), - BaseImageManifest( - image=BaseImage.CORE_ARM64, - package_name="core_vhdx-arm64-3.0-stable", - version="*", - ), - BaseImageManifest( - image=BaseImage.CORE_SELINUX, - package_name="core_selinux_vhdx-3.0-stable", - version="*", - ), - BaseImageManifest( - image=BaseImage.QEMU_GUEST, - package_name="qemu_guest_vhdx-3.0-stable", - version="*", - ), - BaseImageManifest( - image=BaseImage.MINIMAL, - package_name="minimal_vhdx-3.0-stable", - version="*", - ), - ], -) - -if __name__ == "__main__": - import os - - # Change to the base directory in the trident repo - os.chdir(Path(__file__).parent.parent.parent) - - # Run the CLI - cli.init(DEFINED_IMAGES, ARTIFACTS) diff --git a/tests/images/trident-container-installer/README.md b/tests/images/trident-container-installer/README.md index 19ebcd492c..5fadc6fee4 100644 --- a/tests/images/trident-container-installer/README.md +++ b/tests/images/trident-container-installer/README.md @@ -13,7 +13,7 @@ container at boot. From the repo root, run: ```bash -python3 tests/images/testimages.py build trident-container-installer +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml build trident-container-installer --output-dir ./artifacts ``` Output is written to `artifacts/trident-container-installer.iso` by default. diff --git a/tests/images/trident-container-installer/image.yaml b/tests/images/trident-container-installer/image.yaml new file mode 100644 index 0000000000..9fca7d3935 --- /dev/null +++ b/tests/images/trident-container-installer/image.yaml @@ -0,0 +1,13 @@ +name: trident-container-installer + +base: + ref: baremetal + +outputs: + - format: iso + +extraDependencies: + - ./base + +config: + $include: ./tailor/baseimg.yaml diff --git a/tests/images/trident-container-installer/tailor/baseimg.yaml b/tests/images/trident-container-installer/tailor/baseimg.yaml new file mode 100644 index 0000000000..055f6b5e7b --- /dev/null +++ b/tests/images/trident-container-installer/tailor/baseimg.yaml @@ -0,0 +1,71 @@ +storage: + bootType: efi + disks: + - partitionTableType: gpt + maxSize: 4G + partitions: + - id: esp + type: esp + size: 8M + - id: rootfs + size: grow + filesystems: + - deviceId: esp + type: fat32 + mountPoint: + path: /boot/efi + options: umask=0077 + - deviceId: rootfs + type: ext4 + mountPoint: + path: / +os: + bootloader: + resetType: hard-reset + hostname: trident-container-mos-testimg + packages: + install: + - curl + - device-mapper + - dnf + - docker-cli + - iproute + - iptables + - mdadm + - moby-engine + - openssh-server + - selinux-policy + - squashfs-tools + - tar + - vim + additionalFiles: + - source: base/files/getty@.service + destination: /usr/lib/systemd/system/getty@.service + - source: base/files/serial-getty@.service + destination: /usr/lib/systemd/system/serial-getty@.service + - source: base/files/root.profile + destination: /root/.profile + - source: base/files/trident-container.service + destination: /usr/lib/systemd/system/trident-container.service + - source: base/files/containerd-mmap-fix.cil + destination: /usr/share/selinux/packages/containerd-mmap-fix.cil + services: + enable: + - trident-container + - sshd +scripts: + postCustomization: + - path: base/scripts/post-install.sh + - path: base/scripts/load-containerd-selinux-fix.sh +iso: + additionalFiles: + - source: base/files/config-placeholder.yaml + destination: /trident-config.yaml + - source: base/files/override-placeholder.conf + destination: /trident-override.conf + kernelCommandLine: + extraCommandLine: + - enforcing=0 + - console=tty0 + - console=ttyS0 + - rd.luks=0 diff --git a/tests/images/trident-container-testimage/README.md b/tests/images/trident-container-testimage/README.md index e8feada25c..004e8cb243 100644 --- a/tests/images/trident-container-testimage/README.md +++ b/tests/images/trident-container-testimage/README.md @@ -12,7 +12,7 @@ loaded at runtime. From the repo root, run: ```bash -python3 tests/images/testimages.py build trident-container-testimage +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml build trident-container-testimage --output-dir ./artifacts ``` Output is written to `artifacts/trident-container-testimage.cosi` by default. diff --git a/tests/images/trident-container-testimage/image.yaml b/tests/images/trident-container-testimage/image.yaml new file mode 100644 index 0000000000..69785cda6e --- /dev/null +++ b/tests/images/trident-container-testimage/image.yaml @@ -0,0 +1,13 @@ +name: trident-container-testimage + +base: + ref: baremetal + +outputs: + - format: cosi + +extraDependencies: + - ./base + +config: + $include: ./tailor/baseimg.yaml diff --git a/tests/images/trident-container-testimage/tailor/baseimg.yaml b/tests/images/trident-container-testimage/tailor/baseimg.yaml new file mode 100644 index 0000000000..98cef436fb --- /dev/null +++ b/tests/images/trident-container-testimage/tailor/baseimg.yaml @@ -0,0 +1,71 @@ +storage: + bootType: efi + disks: + - partitionTableType: gpt + maxSize: 4G + partitions: + - id: esp + type: esp + size: 8M + - id: rootfs + size: grow + filesystems: + - deviceId: esp + type: fat32 + mountPoint: + path: /boot/efi + options: umask=0077 + - deviceId: rootfs + type: ext4 + mountPoint: + path: / +os: + bootloader: + resetType: hard-reset + hostname: trident-container-testimg + kernelCommandLine: + extraCommandLine: + - console=tty0 + - console=ttyS0 + - rd.info + - log_buf_len=1M + packages: + remove: + - grub2-efi-binary + install: + - grub2-efi-binary-noprefix + - curl + - dnf + - efibootmgr + - iproute + - iptables + - lsof + - mdadm + - netplan + - openssh-server + - tpm2-tools + - vim + - device-mapper + - dosfstools + - lvm2 + - veritysetup + - ntfs-3g + - ntfsprogs + - docker-cli + - moby-engine + - selinux-policy + - squashfs-tools + - tar + additionalFiles: + - source: base/files/trident-container.service + destination: /usr/lib/systemd/system/trident-container.service + - source: base/files/containerd-mmap-fix.cil + destination: /usr/share/selinux/packages/containerd-mmap-fix.cil + services: + enable: + - trident-container + - sshd +scripts: + postCustomization: + - path: base/scripts/post-install.sh + - path: base/scripts/load-containerd-selinux-fix.sh diff --git a/tests/images/trident-functest/image.yaml b/tests/images/trident-functest/image.yaml new file mode 100644 index 0000000000..4f9d4ff305 --- /dev/null +++ b/tests/images/trident-functest/image.yaml @@ -0,0 +1,13 @@ +name: trident-functest + +base: + ref: baremetal + +outputs: + - format: qcow2 + +extraDependencies: + - ./base + +config: + $include: ./tailor/baseimg.yaml diff --git a/tests/images/trident-functest/tailor/baseimg.yaml b/tests/images/trident-functest/tailor/baseimg.yaml new file mode 100644 index 0000000000..c087bee4c6 --- /dev/null +++ b/tests/images/trident-functest/tailor/baseimg.yaml @@ -0,0 +1,75 @@ +storage: + bootType: efi + disks: + - partitionTableType: gpt + maxSize: 15G + partitions: + - id: esp + type: esp + size: 50M + label: esp + - id: root-a + size: 4G + type: root + label: root-a + - id: root-b + size: 4G + type: root + label: root-b + - id: swap + size: 2G + type: swap + label: swap + - id: trident + size: 100M + type: linux-generic + label: trident + filesystems: + - deviceId: esp + type: fat32 + mountPoint: + path: /boot/efi + options: umask=0077 + - deviceId: root-a + type: ext4 + mountPoint: + path: / + - deviceId: trident + type: ext4 + mountPoint: + path: /var/lib/trident +os: + bootloader: + resetType: hard-reset + hostname: trident-functest + selinux: + mode: permissive + kernelCommandLine: + extraCommandLine: + - console=tty0 + - console=ttyS0 + - rd.info + - log_buf_len=1M + packages: + install: + - curl + - dnf + - efibootmgr + - iproute + - iptables + - lsof + - mdadm + - netplan + - openssh-server + - tpm2-tools + - vim + - audit + - device-mapper + - dosfstools + - lvm2 + - veritysetup + - ntfs-3g + - ntfsprogs + services: + enable: + - sshd diff --git a/tests/images/trident-installer/README.md b/tests/images/trident-installer/README.md index b3f1e2d7c8..7226026ce5 100644 --- a/tests/images/trident-installer/README.md +++ b/tests/images/trident-installer/README.md @@ -1,4 +1,19 @@ # Trident Installer ISO Test Image This image is used by the Trident test pipelines. It contains a copy of Trident that launches at -startup and reads a configuration that can be patched into the ISO. \ No newline at end of file +startup and reads a configuration that can be patched into the ISO. + +## Building + +From the repo root, run: + +```bash +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml build trident-installer -s arch=amd64 -s variant=default --output-dir ./artifacts +``` + +Other legacy variants map to selectors on the same family: + +- `trident-split-installer` → `-s arch=amd64 -s variant=split` +- `trident-direct-streaming-installer-amd64` → `-s arch=amd64 -s variant=direct-streaming` +- `trident-installer-arm64` → `-s arch=arm64 -s variant=default` +- `trident-direct-streaming-installer-arm64` → `-s arch=arm64 -s variant=direct-streaming` diff --git a/tests/images/trident-installer/base/baseimg-direct-streaming.yaml b/tests/images/trident-installer/base/baseimg-direct-streaming.yaml index 3ace14fbda..2e728714dc 100644 --- a/tests/images/trident-installer/base/baseimg-direct-streaming.yaml +++ b/tests/images/trident-installer/base/baseimg-direct-streaming.yaml @@ -82,9 +82,9 @@ os: destination: /root/.profile # Netlaunch RCP files - - source: ../../../../bin/rcp-agent + - source: ../../../artifacts/rcp-agent destination: /usr/bin/rcp-agent - - source: ../../../../tools/cmd/rcp-agent/rcp-agent.service + - source: ../../../artifacts/rcp-agent.service destination: /usr/lib/systemd/system/rcp-agent.service selinux: diff --git a/tests/images/trident-installer/by-arch/arm64.yaml b/tests/images/trident-installer/by-arch/arm64.yaml new file mode 100644 index 0000000000..6fe6f6f38d --- /dev/null +++ b/tests/images/trident-installer/by-arch/arm64.yaml @@ -0,0 +1,3 @@ +base: + $set: + ref: core_arm64 diff --git a/tests/images/trident-installer/by-variant/direct-streaming.yaml b/tests/images/trident-installer/by-variant/direct-streaming.yaml new file mode 100644 index 0000000000..9469a76c0c --- /dev/null +++ b/tests/images/trident-installer/by-variant/direct-streaming.yaml @@ -0,0 +1,52 @@ +config: + os: + hostname: + $set: trident-direct-streaming-mos-testimage + packages: + update: + - systemd + install: + $remove: + - trident-static-pcrlock-files + - selinux-policy + - audit + $append: + - setools-console + - trident-selinux + - trident-install-service + services: + enable: + $remove: + - trident-install + $append: + - rcp-agent + additionalFiles: + $remove: + - source: base/files/trident-install.service + destination: /usr/lib/systemd/system/trident-install.service + $append: + - source: ../../../artifacts/rcp-agent + destination: /usr/bin/rcp-agent + - source: ../../../artifacts/rcp-agent.service + destination: /usr/lib/systemd/system/rcp-agent.service + scripts: + postCustomization: + $replace: + - path: base/post-install.sh + arguments: + - stream-image-test + iso: + additionalFiles: + $replace: + - source: base/files/pre-trident-script.sh + destination: /pre-trident-script.sh + - source: base/files/rcp-agent.yaml + destination: /rcp-agent.yaml + kernelCommandLine: + extraCommandLine: + $append: + - selinux=1 + +extraDependencies: + - ../../../artifacts/rcp-agent + - ../../../artifacts/rcp-agent.service diff --git a/tests/images/trident-installer/by-variant/split.yaml b/tests/images/trident-installer/by-variant/split.yaml new file mode 100644 index 0000000000..328eeb6654 --- /dev/null +++ b/tests/images/trident-installer/by-variant/split.yaml @@ -0,0 +1,22 @@ +config: + os: + packages: + install: + $remove: + - binutils + - trident-static-pcrlock-files + - selinux-policy + - selinux-policy-devel + - audit + additionalFiles: + $remove: + - source: base/files/trident-install.service + destination: /usr/lib/systemd/system/trident-install.service + $append: + - source: base/files/trident-split-install.service + destination: /usr/lib/systemd/system/trident-install.service + iso: + additionalFiles: + $remove: + - source: base/files/override-placeholder.conf + destination: /trident-override.conf diff --git a/tests/images/trident-installer/image.yaml b/tests/images/trident-installer/image.yaml new file mode 100644 index 0000000000..6b187585e7 --- /dev/null +++ b/tests/images/trident-installer/image.yaml @@ -0,0 +1,60 @@ +name: trident-installer + +matrix: + arch: [amd64, arm64] + variant: [default, split, direct-streaming] + +selectors: + exclude: + - { arch: arm64, variant: split } + +base: + ref: baremetal + +outputs: + - format: iso + +rpmSources: + - ../../../bin/RPMS + +extraDependencies: + - ./base + +config: + storage: + $include: ./include/storage.yaml + os: + bootloader: + resetType: hard-reset + hostname: trident-mos-testimage + packages: + install: + - $include: ./include/common-packages.yaml + - binutils + - trident-static-pcrlock-files + - selinux-policy + - selinux-policy-devel + - audit + services: + enable: + - sshd + - trident-install + - tridentd.socket + additionalFiles: + - $include: ./include/common-files.yaml + - source: base/files/trident-install.service + destination: /usr/lib/systemd/system/trident-install.service + selinux: + mode: enforcing + scripts: + postCustomization: + - path: base/post-install.sh + iso: + additionalFiles: + - source: base/files/config-placeholder.yaml + destination: /trident-config.yaml + - source: base/files/override-placeholder.conf + destination: /trident-override.conf + kernelCommandLine: + extraCommandLine: + - $include: ./include/kernel-args.yaml diff --git a/tests/images/trident-installer/include/common-files.yaml b/tests/images/trident-installer/include/common-files.yaml new file mode 100644 index 0000000000..ff0048588d --- /dev/null +++ b/tests/images/trident-installer/include/common-files.yaml @@ -0,0 +1,6 @@ +- source: base/files/getty@.service + destination: /usr/lib/systemd/system/getty@.service +- source: base/files/serial-getty@.service + destination: /usr/lib/systemd/system/serial-getty@.service +- source: base/files/root.profile + destination: /root/.profile diff --git a/tests/images/trident-installer/include/common-packages.yaml b/tests/images/trident-installer/include/common-packages.yaml new file mode 100644 index 0000000000..82b2d430e0 --- /dev/null +++ b/tests/images/trident-installer/include/common-packages.yaml @@ -0,0 +1,19 @@ +- curl +- device-mapper +- dnf +- dosfstools +- efibootmgr +- iproute +- iptables +- lsof +- mdadm +- netplan +- ntfs-3g +- ntfsprogs +- openssh-server +- squashfs-tools +- tar +- tpm2-tools +- trident +- vim +- veritysetup diff --git a/tests/images/trident-installer/include/kernel-args.yaml b/tests/images/trident-installer/include/kernel-args.yaml new file mode 100644 index 0000000000..9b4f24cc61 --- /dev/null +++ b/tests/images/trident-installer/include/kernel-args.yaml @@ -0,0 +1,3 @@ +- console=tty0 +- console=ttyS0 +- rd.luks=0 diff --git a/tests/images/trident-installer/include/storage.yaml b/tests/images/trident-installer/include/storage.yaml new file mode 100644 index 0000000000..28c45f83e1 --- /dev/null +++ b/tests/images/trident-installer/include/storage.yaml @@ -0,0 +1,20 @@ +bootType: efi +disks: + - partitionTableType: gpt + maxSize: 4G + partitions: + - id: esp + type: esp + size: 8M + - id: rootfs + size: grow +filesystems: + - deviceId: esp + type: fat32 + mountPoint: + path: /boot/efi + options: umask=0077 + - deviceId: rootfs + type: ext4 + mountPoint: + path: / diff --git a/tests/images/trident-testimage/README.md b/tests/images/trident-testimage/README.md index 108ab80e11..fb1f2174a2 100644 --- a/tests/images/trident-testimage/README.md +++ b/tests/images/trident-testimage/README.md @@ -17,8 +17,10 @@ It also includes openssh-server to allow for remote access. From the repo root, run: ```bash -python3 tests/images/testimages.py build trident-testimage +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml build trident-testimage -s arch=amd64 --output-dir ./artifacts ``` +Use `-s arch=arm64` for the ARM64 variant. + Output is written to `artifacts/trident-testimage.cosi` by default. Use `--output-dir ` to change the output location. diff --git a/tests/images/trident-testimage/by-arch/arm64.yaml b/tests/images/trident-testimage/by-arch/arm64.yaml new file mode 100644 index 0000000000..6fe6f6f38d --- /dev/null +++ b/tests/images/trident-testimage/by-arch/arm64.yaml @@ -0,0 +1,3 @@ +base: + $set: + ref: core_arm64 diff --git a/tests/images/trident-testimage/image.yaml b/tests/images/trident-testimage/image.yaml new file mode 100644 index 0000000000..06bd41af16 --- /dev/null +++ b/tests/images/trident-testimage/image.yaml @@ -0,0 +1,19 @@ +name: trident-testimage + +matrix: + arch: [amd64, arm64] + +base: + ref: baremetal + +outputs: + - format: cosi + +rpmSources: + - ../../../bin/RPMS + +extraDependencies: + - ./base + +config: + $include: ./tailor/baseimg.yaml diff --git a/tests/images/trident-testimage/tailor/baseimg.yaml b/tests/images/trident-testimage/tailor/baseimg.yaml new file mode 100644 index 0000000000..89cbfd7bd9 --- /dev/null +++ b/tests/images/trident-testimage/tailor/baseimg.yaml @@ -0,0 +1,66 @@ +storage: + bootType: efi + disks: + - partitionTableType: gpt + maxSize: 4G + partitions: + - id: esp + type: esp + size: 8M + - id: rootfs + size: grow + filesystems: + - deviceId: esp + type: fat32 + mountPoint: + path: /boot/efi + options: umask=0077 + - deviceId: rootfs + type: ext4 + mountPoint: + path: / +os: + bootloader: + resetType: hard-reset + hostname: trident-testimg + kernelCommandLine: + extraCommandLine: + - console=tty0 + - console=ttyS0 + - rd.info + - log_buf_len=1M + packages: + remove: + - grub2-efi-binary + install: + - grub2-efi-binary-noprefix + - curl + - dnf + - efibootmgr + - iproute + - iptables + - lsof + - mdadm + - netplan + - openssh-server + - tpm2-tools + - trident-service + - vim + - audit + - device-mapper + - dosfstools + - lvm2 + - veritysetup + - ntfs-3g + - ntfsprogs + services: + enable: + - sshd + - trident + - tridentd.socket + additionalFiles: + - source: base/files/use-grpc-client-commit.conf + destination: /etc/systemd/system/trident.service.d/override.conf +scripts: + postCustomization: + - path: base/post-install.sh diff --git a/tests/images/trident-verity-testimage/README.md b/tests/images/trident-verity-testimage/README.md index 0c8dd782a7..af74e5886b 100644 --- a/tests/images/trident-verity-testimage/README.md +++ b/tests/images/trident-verity-testimage/README.md @@ -18,8 +18,14 @@ dm-verity. From the repo root, run: ```bash -python3 tests/images/testimages.py build trident-verity-testimage +cargo run --manifest-path tools/tailor/Cargo.toml -- --manifest tests/images/tailor.yaml build trident-verity-testimage -s deployment=host -s mode=root --output-dir ./artifacts ``` -Output is written to `artifacts/trident-verity-testimage.cosi` by default. Use +Other legacy variants map to selectors on the same family: + +- `trident-usrverity-testimage` → `-s deployment=host -s mode=usr` +- `trident-container-verity-testimage` → `-s deployment=container -s mode=root` +- `trident-container-usrverity-testimage` → `-s deployment=container -s mode=usr` + +Output is written to `artifacts/` using the tailor cell slug by default. Use `--output-dir ` to change the output location. diff --git a/tests/images/trident-verity-testimage/by-deployment+mode/container+root.yaml b/tests/images/trident-verity-testimage/by-deployment+mode/container+root.yaml new file mode 100644 index 0000000000..8d3e5de076 --- /dev/null +++ b/tests/images/trident-verity-testimage/by-deployment+mode/container+root.yaml @@ -0,0 +1,19 @@ +config: + os: + hostname: trident-container-verity-testimg + packages: + install: + - dnf + - docker-cli + - moby-engine + - squashfs-tools + - tar + additionalFiles: + - source: base/files/trident-container.service + destination: /usr/lib/systemd/system/trident-container.service + services: + enable: + - trident-container + scripts: + postCustomization: + - path: base/create-web-dir.sh diff --git a/tests/images/trident-verity-testimage/by-deployment+mode/container+usr.yaml b/tests/images/trident-verity-testimage/by-deployment+mode/container+usr.yaml new file mode 100644 index 0000000000..c7eed50273 --- /dev/null +++ b/tests/images/trident-verity-testimage/by-deployment+mode/container+usr.yaml @@ -0,0 +1,15 @@ +config: + os: + hostname: trident-container-usrverity-testimg + packages: + install: + - docker-cli + - moby-engine + - squashfs-tools + - tar + additionalFiles: + - source: usr/files/trident-container.service + destination: /usr/lib/systemd/system/trident-container.service + services: + enable: + - trident-container diff --git a/tests/images/trident-verity-testimage/by-deployment+mode/host+root.yaml b/tests/images/trident-verity-testimage/by-deployment+mode/host+root.yaml new file mode 100644 index 0000000000..e75de1bfb6 --- /dev/null +++ b/tests/images/trident-verity-testimage/by-deployment+mode/host+root.yaml @@ -0,0 +1,17 @@ +config: + os: + hostname: trident-verity-testimg + packages: + install: + - trident-service + additionalFiles: + - source: base/files/use-grpc-client-commit.conf + destination: /etc/systemd/system/trident.service.d/override.conf + services: + enable: + - trident + - tridentd.socket + scripts: + postCustomization: + - path: base/trident-debug-log.sh + - path: base/create-web-dir.sh diff --git a/tests/images/trident-verity-testimage/by-deployment+mode/host+usr.yaml b/tests/images/trident-verity-testimage/by-deployment+mode/host+usr.yaml new file mode 100644 index 0000000000..567cc110db --- /dev/null +++ b/tests/images/trident-verity-testimage/by-deployment+mode/host+usr.yaml @@ -0,0 +1,17 @@ +config: + os: + hostname: trident-usrverity-testimg + packages: + install: + - binutils + - trident-service + - trident-static-pcrlock-files + - audit + - selinux-policy-devel + additionalFiles: + - source: usr/files/use-grpc-client-commit.conf + destination: /etc/systemd/system/trident.service.d/override.conf + services: + enable: + - trident + - tridentd.socket diff --git a/tests/images/trident-verity-testimage/by-deployment/container.yaml b/tests/images/trident-verity-testimage/by-deployment/container.yaml new file mode 100644 index 0000000000..de2ca21c3f --- /dev/null +++ b/tests/images/trident-verity-testimage/by-deployment/container.yaml @@ -0,0 +1 @@ +features: [] diff --git a/tests/images/trident-verity-testimage/by-deployment/host.yaml b/tests/images/trident-verity-testimage/by-deployment/host.yaml new file mode 100644 index 0000000000..7ef528ff76 --- /dev/null +++ b/tests/images/trident-verity-testimage/by-deployment/host.yaml @@ -0,0 +1,2 @@ +rpmSources: + - ../../../bin/RPMS diff --git a/tests/images/trident-verity-testimage/by-mode/root.yaml b/tests/images/trident-verity-testimage/by-mode/root.yaml new file mode 100644 index 0000000000..04d59a7568 --- /dev/null +++ b/tests/images/trident-verity-testimage/by-mode/root.yaml @@ -0,0 +1,5 @@ +config: + os: + $include: ./include/root-os-common.yaml + storage: + $include: ./include/root-storage.yaml diff --git a/tests/images/trident-verity-testimage/by-mode/usr.yaml b/tests/images/trident-verity-testimage/by-mode/usr.yaml new file mode 100644 index 0000000000..8a5445367a --- /dev/null +++ b/tests/images/trident-verity-testimage/by-mode/usr.yaml @@ -0,0 +1,14 @@ +signing: true +config: + os: + $include: ./include/usr-os-common.yaml + storage: + $include: ./include/usr-storage.yaml + output: + artifacts: + items: + - ukis + path: ./output + previewFeatures: + - uki + - output-artifacts diff --git a/tests/images/trident-verity-testimage/image.yaml b/tests/images/trident-verity-testimage/image.yaml new file mode 100644 index 0000000000..47417c6b27 --- /dev/null +++ b/tests/images/trident-verity-testimage/image.yaml @@ -0,0 +1,17 @@ +name: trident-verity-testimage + +matrix: + deployment: [host, container] + mode: [root, usr] + +base: + ref: baremetal + +outputs: + - format: cosi + +extraDependencies: + - ./base + - ./usr + +config: {} diff --git a/tests/images/trident-verity-testimage/include/root-os-common.yaml b/tests/images/trident-verity-testimage/include/root-os-common.yaml new file mode 100644 index 0000000000..16faa0d324 --- /dev/null +++ b/tests/images/trident-verity-testimage/include/root-os-common.yaml @@ -0,0 +1,42 @@ +bootloader: + resetType: hard-reset +selinux: + mode: disabled +kernelCommandLine: + extraCommandLine: + - console=tty0 + - console=ttyS0 + - rd.info + - log_buf_len=1M +packages: + install: + - grub2-efi-binary-noprefix + - curl + - device-mapper + - dosfstools + - dracut-overlayfs + - efibootmgr + - iproute + - iptables + - lsof + - lvm2 + - mdadm + - netplan + - openssh-server + - systemd-udev + - tpm2-tools + - veritysetup + - vim + remove: + - grub2-efi-binary +additionalFiles: + - source: base/etc-mount.service + destination: /etc/systemd/system/etc-mount.service + - source: base/etc-mount.sh + destination: /usr/local/bin/etc-mount.sh + - source: base/sudoers-wheel + destination: /etc/sudoers.d/wheel +services: + enable: + - etc-mount + - sshd diff --git a/tests/images/trident-verity-testimage/include/root-storage.yaml b/tests/images/trident-verity-testimage/include/root-storage.yaml new file mode 100644 index 0000000000..cb38c27f81 --- /dev/null +++ b/tests/images/trident-verity-testimage/include/root-storage.yaml @@ -0,0 +1,56 @@ +disks: + - partitionTableType: gpt + maxSize: 5G + partitions: + - id: esp + type: esp + size: 8M + - id: boot + size: 1G + - label: root + id: root + size: 2G + - label: root-hash + id: verityhash + size: 128M + - id: sysexts + size: 1G + - id: confexts + size: 200M + - id: var + size: grow +bootType: efi +verity: + - id: rootverity + name: root + dataDeviceId: root + hashDeviceId: verityhash + dataDeviceMountIdType: part-label + hashDeviceMountIdType: part-label +filesystems: + - deviceId: esp + type: fat32 + mountPoint: + path: /boot/efi + options: umask=0077 + - deviceId: boot + type: ext4 + mountPoint: + path: /boot + - deviceId: rootverity + type: ext4 + mountPoint: + path: / + options: defaults,ro + - deviceId: sysexts + type: ext4 + mountPoint: + path: /var/lib/extensions + - deviceId: confexts + type: ext4 + mountPoint: + path: /var/lib/confexts + - deviceId: var + type: ext4 + mountPoint: + path: /var diff --git a/tests/images/trident-verity-testimage/include/usr-os-common.yaml b/tests/images/trident-verity-testimage/include/usr-os-common.yaml new file mode 100644 index 0000000000..3817dbcc59 --- /dev/null +++ b/tests/images/trident-verity-testimage/include/usr-os-common.yaml @@ -0,0 +1,41 @@ +bootloader: + resetType: hard-reset +selinux: + mode: disabled +uki: + mode: create +kernelCommandLine: + extraCommandLine: + - console=tty0 + - console=ttyS0 + - rd.info + - log_buf_len=1M + - rd.hostonly=0 +packages: + install: + - curl + - device-mapper + - efibootmgr + - iproute + - iptables + - lvm2 + - mdadm + - netplan + - openssh-server + - systemd-udev + - tpm2-tools + - veritysetup + - vim + - systemd-ukify + - systemd-boot + - efibootmgr + remove: + - grub2-efi-binary +additionalFiles: + - source: usr/files/dracut-nohostonly.conf + destination: /usr/lib/dracut/dracut.conf.d/nohostonly.conf + - source: usr/files/sudoers-wheel + destination: /etc/sudoers.d/wheel +services: + enable: + - sshd diff --git a/tests/images/trident-verity-testimage/include/usr-storage.yaml b/tests/images/trident-verity-testimage/include/usr-storage.yaml new file mode 100644 index 0000000000..ce7337eaa0 --- /dev/null +++ b/tests/images/trident-verity-testimage/include/usr-storage.yaml @@ -0,0 +1,45 @@ +disks: + - partitionTableType: gpt + maxSize: 5G + partitions: + - id: esp + type: esp + size: 500M + label: esp + - id: boot + size: 150M + - id: root + size: 2G + - id: usr-data + label: usr + size: 1G + - id: usr-hash + label: usr-hash + size: 128M +bootType: efi +verity: + - id: usr + name: usr + dataDeviceId: usr-data + hashDeviceId: usr-hash + dataDeviceMountIdType: uuid + hashDeviceMountIdType: uuid +filesystems: + - deviceId: esp + type: fat32 + mountPoint: + path: /boot/efi + options: umask=0077 + - deviceId: boot + type: ext4 + mountPoint: + path: /boot + - deviceId: root + type: ext4 + mountPoint: + path: / + - deviceId: usr + type: ext4 + mountPoint: + path: /usr + options: defaults,ro diff --git a/tests/images/trident-vm-testimage/by-arch/arm64.yaml b/tests/images/trident-vm-testimage/by-arch/arm64.yaml new file mode 100644 index 0000000000..6fe6f6f38d --- /dev/null +++ b/tests/images/trident-vm-testimage/by-arch/arm64.yaml @@ -0,0 +1,3 @@ +base: + $set: + ref: core_arm64 diff --git a/tests/images/trident-vm-testimage/by-scenario/acl-agent.yaml b/tests/images/trident-vm-testimage/by-scenario/acl-agent.yaml new file mode 100644 index 0000000000..a2210cc7f8 --- /dev/null +++ b/tests/images/trident-vm-testimage/by-scenario/acl-agent.yaml @@ -0,0 +1,24 @@ +config: + storage: + $include: ./include/storage-acl-agent.yaml + os: + hostname: trident-acl-agent-testimg + packages: + install: + $append: + - trident-acl + additionalFiles: + - source: base/files/99-dhcp-eth0.network + destination: /etc/systemd/network/99-dhcp-eth0.network + - source: base/files/sudoers-wheel + destination: /etc/sudoers.d/wheel + - source: base/files/trident-acl-agent-override.conf + destination: /etc/systemd/system/trident-acl-agent.service.d/override.conf + - source: base/files/trident-acl-agent-cert-install.service + destination: /etc/systemd/system/trident-acl-agent-cert-install.service + services: + enable: + - kdump + - tridentd.socket + - trident-acl-agent-cert-install.service + - trident-acl-agent.service diff --git a/tests/images/trident-vm-testimage/by-scenario/grub-verity+grub-verity-azure.yaml b/tests/images/trident-vm-testimage/by-scenario/grub-verity+grub-verity-azure.yaml new file mode 100644 index 0000000000..ad0608c844 --- /dev/null +++ b/tests/images/trident-vm-testimage/by-scenario/grub-verity+grub-verity-azure.yaml @@ -0,0 +1,6 @@ +config: + os: + $include: ./include/grub-verity-os-common.yaml + scripts: + postCustomization: + - $include: ./include/grub-verity-scripts.yaml diff --git a/tests/images/trident-vm-testimage/by-scenario/grub-verity-azure.yaml b/tests/images/trident-vm-testimage/by-scenario/grub-verity-azure.yaml new file mode 100644 index 0000000000..d2b2c5464e --- /dev/null +++ b/tests/images/trident-vm-testimage/by-scenario/grub-verity-azure.yaml @@ -0,0 +1,23 @@ +base: + $set: + ref: core_selinux +config: + storage: + $include: ./include/storage-grub-verity.yaml + os: + hostname: trident-vm-testimg + selinux: + mode: + $set: permissive + packages: + install: + $remove: + - netplan + $append: + - selinux-policy + - WALinuxAgent + users: + $replace: + - name: testuser + secondaryGroups: + - wheel diff --git a/tests/images/trident-vm-testimage/by-scenario/grub-verity.yaml b/tests/images/trident-vm-testimage/by-scenario/grub-verity.yaml new file mode 100644 index 0000000000..413e42b822 --- /dev/null +++ b/tests/images/trident-vm-testimage/by-scenario/grub-verity.yaml @@ -0,0 +1,5 @@ +config: + storage: + $include: ./include/storage-grub-verity.yaml + os: + hostname: trident-vm-verity-testimg diff --git a/tests/images/trident-vm-testimage/by-scenario/grub.yaml b/tests/images/trident-vm-testimage/by-scenario/grub.yaml new file mode 100644 index 0000000000..d9bfa313ae --- /dev/null +++ b/tests/images/trident-vm-testimage/by-scenario/grub.yaml @@ -0,0 +1,55 @@ +config: + storage: + $include: ./include/storage-grub.yaml + os: + bootloader: + resetType: hard-reset + hostname: trident-vm-testimg + selinux: + mode: disabled + kernelCommandLine: + extraCommandLine: + - console=tty0 + - console=tty1 + - console=ttyS0 + - rd.debug + - loglevel=6 + - log_buf_len=1M + - systemd.journald.forward_to_console=1 + packages: + install: + - curl + - dnf + - efibootmgr + - grub2-efi-binary-noprefix + - iproute + - iptables + - jq + - lsof + - netplan + - openssh-server + - trident-service + - vim + - netplan + remove: + - grub2-efi-binary + services: + enable: + - trident + - tridentd.socket + users: + - name: testuser + sshPublicKeyPaths: + - base/files/id_rsa.pub + secondaryGroups: + - wheel + additionalFiles: + - source: base/files/sshd-keygen.service + destination: /usr/lib/systemd/system/sshd-keygen.service + - source: base/files/99-dhcp-eth0.network + destination: /etc/systemd/network/99-dhcp-eth0.network + - source: base/files/sudoers-wheel + destination: /etc/sudoers.d/wheel + scripts: + postCustomization: + - $include: ./include/grub-scripts.yaml diff --git a/tests/images/trident-vm-testimage/by-scenario/root-verity+usr-verity+acl-agent.yaml b/tests/images/trident-vm-testimage/by-scenario/root-verity+usr-verity+acl-agent.yaml new file mode 100644 index 0000000000..ac986e4ae7 --- /dev/null +++ b/tests/images/trident-vm-testimage/by-scenario/root-verity+usr-verity+acl-agent.yaml @@ -0,0 +1,8 @@ +config: + os: + $include: ./include/uki-verity-os-common.yaml + scripts: + postCustomization: + - $include: ./include/uki-verity-scripts.yaml + previewFeatures: + - uki diff --git a/tests/images/trident-vm-testimage/by-scenario/root-verity.yaml b/tests/images/trident-vm-testimage/by-scenario/root-verity.yaml new file mode 100644 index 0000000000..a19ba87c3e --- /dev/null +++ b/tests/images/trident-vm-testimage/by-scenario/root-verity.yaml @@ -0,0 +1,24 @@ +config: + storage: + $include: ./include/storage-root-verity.yaml + os: + hostname: trident-vm-root-verity-testimg + additionalFiles: + - source: base/files/99-dhcp-eth0.network + destination: /etc/systemd/network/99-dhcp-eth0.network + - source: base/files/sudoers-wheel + destination: /etc/sudoers.d/wheel + overlays: + - mountPoint: /etc + lowerDirs: + - /etc + upperDir: /var/lib/overlays/etc/upper + workDir: /var/lib/overlays/etc/work + mountDependencies: + - /var + isInitrdOverlay: true + services: + enable: + - kdump + - trident + - tridentd.socket diff --git a/tests/images/trident-vm-testimage/by-scenario/usr-verity.yaml b/tests/images/trident-vm-testimage/by-scenario/usr-verity.yaml new file mode 100644 index 0000000000..766620fea4 --- /dev/null +++ b/tests/images/trident-vm-testimage/by-scenario/usr-verity.yaml @@ -0,0 +1,15 @@ +config: + storage: + $include: ./include/storage-usr-verity.yaml + os: + hostname: trident-vm-usr-verity-testimg + additionalFiles: + - source: base/files/99-dhcp-eth0.network + destination: /etc/systemd/network/99-dhcp-eth0.network + - source: base/files/sudoers-wheel + destination: /etc/sudoers.d/wheel + services: + enable: + - kdump + - trident + - tridentd.socket diff --git a/tests/images/trident-vm-testimage/image.yaml b/tests/images/trident-vm-testimage/image.yaml new file mode 100644 index 0000000000..ce0f3c26a3 --- /dev/null +++ b/tests/images/trident-vm-testimage/image.yaml @@ -0,0 +1,23 @@ +name: trident-vm-testimage + +matrix: + arch: [amd64, arm64] + scenario: [grub, grub-verity, root-verity, usr-verity, acl-agent, grub-verity-azure] + +selectors: + exclude: + - { arch: arm64, scenario: [root-verity, usr-verity, acl-agent, grub-verity-azure] } + +base: + ref: qemu_guest + +outputs: + - format: cosi + +rpmSources: + - ../../../bin/RPMS + +extraDependencies: + - ./base + +config: {} diff --git a/tests/images/trident-vm-testimage/include/grub-os-common.yaml b/tests/images/trident-vm-testimage/include/grub-os-common.yaml new file mode 100644 index 0000000000..08805a784d --- /dev/null +++ b/tests/images/trident-vm-testimage/include/grub-os-common.yaml @@ -0,0 +1,22 @@ +bootloader: + resetType: hard-reset +selinux: + mode: disabled +kernelCommandLine: + extraCommandLine: + - console=tty0 + - console=tty1 + - console=ttyS0 + - rd.debug + - loglevel=6 + - log_buf_len=1M + - systemd.journald.forward_to_console=1 +packages: + remove: + - grub2-efi-binary +users: + - name: testuser + sshPublicKeyPaths: + - base/files/id_rsa.pub + secondaryGroups: + - wheel diff --git a/tests/images/trident-vm-testimage/include/grub-scripts.yaml b/tests/images/trident-vm-testimage/include/grub-scripts.yaml new file mode 100644 index 0000000000..cefd80ff51 --- /dev/null +++ b/tests/images/trident-vm-testimage/include/grub-scripts.yaml @@ -0,0 +1,4 @@ +- path: base/scripts/post-install.sh +- path: base/scripts/ssh-move-host-keys.sh +- path: base/scripts/duid-type-to-link-layer.sh +- path: base/scripts/update-os-release.sh diff --git a/tests/images/trident-vm-testimage/include/grub-verity-os-common.yaml b/tests/images/trident-vm-testimage/include/grub-verity-os-common.yaml new file mode 100644 index 0000000000..6432937bc3 --- /dev/null +++ b/tests/images/trident-vm-testimage/include/grub-verity-os-common.yaml @@ -0,0 +1,59 @@ +bootloader: + resetType: hard-reset +selinux: + mode: disabled +kernelCommandLine: + extraCommandLine: + - console=tty0 + - console=tty1 + - console=ttyS0 + - rd.debug + - loglevel=6 + - log_buf_len=1M + - systemd.journald.forward_to_console=1 +packages: + install: + - curl + - device-mapper + - dnf + - dracut-overlayfs + - efibootmgr + - grub2-efi-binary-noprefix + - iproute + - iptables + - jq + - kexec-tools + - lsof + - lvm2 + - netplan + - openssh-server + - systemd-udev + - trident-service + - veritysetup + - vim + - netplan + remove: + - grub2-efi-binary +additionalFiles: + - source: base/files/etc-mount.service + destination: /etc/systemd/system/etc-mount.service + - source: base/files/etc-mount.sh + destination: /usr/local/bin/etc-mount.sh + - source: base/files/sudoers-wheel + destination: /etc/sudoers.d/wheel + - source: base/files/99-dhcp-eth0.network + destination: /etc/systemd/network/99-dhcp-eth0.network + - source: base/files/use-grpc-client-commit.conf + destination: /etc/systemd/system/trident.service.d/override.conf +services: + enable: + - etc-mount + - kdump + - trident + - tridentd.socket +users: + - name: testuser + sshPublicKeyPaths: + - base/files/id_rsa.pub + secondaryGroups: + - wheel diff --git a/tests/images/trident-vm-testimage/include/grub-verity-scripts.yaml b/tests/images/trident-vm-testimage/include/grub-verity-scripts.yaml new file mode 100644 index 0000000000..db0dca3e4c --- /dev/null +++ b/tests/images/trident-vm-testimage/include/grub-verity-scripts.yaml @@ -0,0 +1,4 @@ +- path: base/scripts/post-install.sh +- path: base/scripts/ssh-move-host-keys.sh +- path: base/scripts/update-os-release.sh +- path: base/scripts/duid-type-to-link-layer.sh diff --git a/tests/images/trident-vm-testimage/include/storage-acl-agent.yaml b/tests/images/trident-vm-testimage/include/storage-acl-agent.yaml new file mode 100644 index 0000000000..82632328f0 --- /dev/null +++ b/tests/images/trident-vm-testimage/include/storage-acl-agent.yaml @@ -0,0 +1,101 @@ +bootType: efi +disks: + - partitionTableType: gpt + partitions: + - id: esp + type: esp + label: esp + size: 512M + - id: boot-a + size: 256M + - id: boot-b + size: 256M + - id: root-a + size: 4G + - id: root-b + size: 4G + - id: usr-a + size: 1G + - id: usr-b + size: 1G + - id: usr-hash-a + size: 128M + - id: usr-hash-b + size: 128M + - id: trident + label: trident + size: 512M + - id: trident-acl-agent + label: trident-acl-agent + size: 128M + - id: kubelet + label: kubelet + size: 128M + - id: etc-trident + label: etc-trident + size: 128M + - id: home + label: home + size: 1G + - id: srv + label: srv + size: 128M +verity: + - id: usrverity + name: usr + dataDeviceId: usr-a + hashDeviceId: usr-hash-a + dataDeviceMountIdType: uuid + hashDeviceMountIdType: uuid +filesystems: + - deviceId: esp + type: fat32 + mountPoint: + idType: part-label + path: /boot/efi + options: umask=0077 + - deviceId: boot-a + type: ext4 + mountPoint: + idType: uuid + path: /boot + - deviceId: usrverity + type: ext4 + mountPoint: + path: /usr + options: defaults,ro + - deviceId: root-a + type: ext4 + mountPoint: + idType: uuid + path: / + - deviceId: trident + type: ext4 + mountPoint: + idType: part-label + path: /var/lib/trident + - deviceId: trident-acl-agent + type: ext4 + mountPoint: + idType: part-label + path: /var/lib/trident-acl-agent + - deviceId: kubelet + type: ext4 + mountPoint: + idType: part-label + path: /var/lib/kubelet + - deviceId: etc-trident + type: ext4 + mountPoint: + idType: part-label + path: /etc/trident + - deviceId: home + type: ext4 + mountPoint: + idType: part-label + path: /home + - deviceId: srv + type: ext4 + mountPoint: + idType: part-label + path: /srv diff --git a/tests/images/trident-vm-testimage/include/storage-grub-verity.yaml b/tests/images/trident-vm-testimage/include/storage-grub-verity.yaml new file mode 100644 index 0000000000..fbe1d145ea --- /dev/null +++ b/tests/images/trident-vm-testimage/include/storage-grub-verity.yaml @@ -0,0 +1,54 @@ +bootType: efi +disks: + - partitionTableType: gpt + partitions: + - id: esp + type: esp + size: 8M + - id: boot + size: 256M + - id: root + size: 4G + - id: root-hash + size: 128M + - id: var + size: 1G + - id: trident-overlay + size: 32M + - id: srv + size: 1G + - id: home + size: 32M +verity: + - id: rootverity + name: root + dataDeviceId: root + hashDeviceId: root-hash + dataDeviceMountIdType: uuid + hashDeviceMountIdType: uuid +filesystems: + - deviceId: esp + type: fat32 + mountPoint: + path: /boot/efi + options: umask=0077 + - deviceId: boot + type: ext4 + mountPoint: /boot + - deviceId: rootverity + type: ext4 + mountPoint: + path: / + options: defaults,ro + - deviceId: var + type: ext4 + mountPoint: /var + - deviceId: trident-overlay + type: ext4 + mountPoint: /var/lib/trident-overlay + - deviceId: srv + type: ext4 + mountPoint: /srv + - deviceId: home + type: ext4 + mountPoint: /home diff --git a/tests/images/trident-vm-testimage/include/storage-grub.yaml b/tests/images/trident-vm-testimage/include/storage-grub.yaml new file mode 100644 index 0000000000..f36e7979e8 --- /dev/null +++ b/tests/images/trident-vm-testimage/include/storage-grub.yaml @@ -0,0 +1,31 @@ +bootType: efi +disks: + - partitionTableType: gpt + maxSize: 10G + partitions: + - id: esp + type: esp + size: 16M + - id: root-a + size: 4G + - id: root-b + size: 4G + - id: trident + size: 1G + - id: srv + size: grow +filesystems: + - deviceId: esp + type: fat32 + mountPoint: + path: /boot/efi + options: umask=0077 + - deviceId: root-a + type: ext4 + mountPoint: / + - deviceId: trident + type: ext4 + mountPoint: /var/lib/trident + - deviceId: srv + type: ext4 + mountPoint: /srv diff --git a/tests/images/trident-vm-testimage/include/storage-root-verity.yaml b/tests/images/trident-vm-testimage/include/storage-root-verity.yaml new file mode 100644 index 0000000000..e5e3cbd74d --- /dev/null +++ b/tests/images/trident-vm-testimage/include/storage-root-verity.yaml @@ -0,0 +1,78 @@ +bootType: efi +disks: + - partitionTableType: gpt + partitions: + - id: esp + type: esp + label: esp + size: 512M + - id: boot-a + size: 256M + - id: boot-b + size: 256M + - id: root-a + size: 4G + - id: root-b + size: 4G + - id: root-hash-a + size: 128M + - id: root-hash-b + size: 128M + - id: var-a + size: 1G + - id: var-b + size: 1G + - id: trident + label: trident + size: 512M + - id: home + label: home + size: 1G + - id: srv + label: srv + size: 128M +verity: + - id: rootverity + name: root + dataDeviceId: root-a + hashDeviceId: root-hash-a + dataDeviceMountIdType: uuid + hashDeviceMountIdType: uuid +filesystems: + - deviceId: esp + type: fat32 + mountPoint: + idType: part-label + path: /boot/efi + options: umask=0077 + - deviceId: boot-a + type: ext4 + mountPoint: + idType: uuid + path: /boot + - deviceId: rootverity + type: ext4 + mountPoint: + path: / + options: defaults,ro + - deviceId: var-a + type: ext4 + mountPoint: + idType: uuid + path: /var + options: defaults,x-initrd.mount + - deviceId: trident + type: ext4 + mountPoint: + idType: part-label + path: /var/lib/trident + - deviceId: home + type: ext4 + mountPoint: + idType: part-label + path: /home + - deviceId: srv + type: ext4 + mountPoint: + idType: part-label + path: /srv diff --git a/tests/images/trident-vm-testimage/include/storage-usr-verity.yaml b/tests/images/trident-vm-testimage/include/storage-usr-verity.yaml new file mode 100644 index 0000000000..a380c8eb59 --- /dev/null +++ b/tests/images/trident-vm-testimage/include/storage-usr-verity.yaml @@ -0,0 +1,77 @@ +bootType: efi +disks: + - partitionTableType: gpt + partitions: + - id: esp + type: esp + label: esp + size: 512M + - id: boot-a + size: 256M + - id: boot-b + size: 256M + - id: root-a + size: 4G + - id: root-b + size: 4G + - id: usr-a + size: 1G + - id: usr-b + size: 1G + - id: usr-hash-a + size: 128M + - id: usr-hash-b + size: 128M + - id: trident + label: trident + size: 512M + - id: home + label: home + size: 1G + - id: srv + label: srv + size: 128M +verity: + - id: usrverity + name: usr + dataDeviceId: usr-a + hashDeviceId: usr-hash-a + dataDeviceMountIdType: uuid + hashDeviceMountIdType: uuid +filesystems: + - deviceId: esp + type: fat32 + mountPoint: + idType: part-label + path: /boot/efi + options: umask=0077 + - deviceId: boot-a + type: ext4 + mountPoint: + idType: uuid + path: /boot + - deviceId: usrverity + type: ext4 + mountPoint: + path: /usr + options: defaults,ro + - deviceId: root-a + type: ext4 + mountPoint: + idType: uuid + path: / + - deviceId: trident + type: ext4 + mountPoint: + idType: part-label + path: /var/lib/trident + - deviceId: home + type: ext4 + mountPoint: + idType: part-label + path: /home + - deviceId: srv + type: ext4 + mountPoint: + idType: part-label + path: /srv diff --git a/tests/images/trident-vm-testimage/include/uki-verity-os-common.yaml b/tests/images/trident-vm-testimage/include/uki-verity-os-common.yaml new file mode 100644 index 0000000000..6f277178bd --- /dev/null +++ b/tests/images/trident-vm-testimage/include/uki-verity-os-common.yaml @@ -0,0 +1,41 @@ +bootloader: + resetType: hard-reset +selinux: + mode: disabled +uki: + mode: create +kernelCommandLine: + extraCommandLine: + - console=tty0 + - console=tty1 + - console=ttyS0 + - rd.debug + - loglevel=6 + - log_buf_len=1M + - systemd.journald.forward_to_console=1 + - rd.hostonly=0 +packages: + install: + - device-mapper + - dnf + - efibootmgr + - iproute + - iptables + - jq + - kexec-tools + - lvm2 + - openssh-server + - systemd-boot + - systemd-udev + - trident-service + - veritysetup + - vim + - netplan + remove: + - grub2-efi-binary +users: + - name: testuser + sshPublicKeyPaths: + - base/files/id_rsa.pub + secondaryGroups: + - wheel diff --git a/tests/images/trident-vm-testimage/include/uki-verity-scripts.yaml b/tests/images/trident-vm-testimage/include/uki-verity-scripts.yaml new file mode 100644 index 0000000000..89bf812cc3 --- /dev/null +++ b/tests/images/trident-vm-testimage/include/uki-verity-scripts.yaml @@ -0,0 +1,6 @@ +- path: base/scripts/post-install.sh +- path: base/scripts/update-host-status.sh +- path: base/scripts/prepare-update-config-verity.sh + arguments: + - uki +- path: base/scripts/duid-type-to-link-layer.sh diff --git a/tools/tailor/crates/tailor-config/src/fragment.rs b/tools/tailor/crates/tailor-config/src/fragment.rs index 1fdc993f56..e899e91254 100644 --- a/tools/tailor/crates/tailor-config/src/fragment.rs +++ b/tools/tailor/crates/tailor-config/src/fragment.rs @@ -18,7 +18,7 @@ use serde_yaml_ng::Value; use crate::{ error::ConfigError, - schema::{AxisValues, ExtraParam}, + schema::{AxisValues, ExtraParam, SigningRef}, types::ParamValue, }; @@ -45,6 +45,9 @@ pub(crate) struct Fragment { /// (base → most-specific), mirroring `rpmSources`. #[serde(default)] pub(crate) extra_params: Vec, + /// Opt a matched cell into a workspace signing profile. + #[serde(default)] + pub(crate) signing: Option, /// Drop cells this fragment applies to from bulk selection unless the run pins the fragment's /// value or names the cell (`meta/docs/2026-07-22-fragment-skip.md`). Mergeable last-wins; a /// more-specific fragment may set `false` to un-skip. diff --git a/tools/tailor/crates/tailor-config/src/render.rs b/tools/tailor/crates/tailor-config/src/render.rs index 2295fb4776..18029fc749 100644 --- a/tools/tailor/crates/tailor-config/src/render.rs +++ b/tools/tailor/crates/tailor-config/src/render.rs @@ -18,7 +18,7 @@ use crate::{ include, interpolate, matrix, matrix::AxisTuple, merge, - schema::{BaseSource, ExtraParam, ImageDefinition, OutputSpec}, + schema::{BaseSource, ExtraParam, ImageDefinition, OutputSpec, SigningRef}, types::{OutputFormat, ParamValue}, }; @@ -65,6 +65,8 @@ pub struct RenderedCell { pub rpm_sources: Vec, /// Extra IC command-line flags appended verbatim, concatenated across matched fragments. pub extra_params: Vec, + /// The resolved per-cell `signing:` opt-in after fragment merging. + pub signing: Option, /// Resolved `skip` for this cell (merged from fragment `skip:` fields, last-wins). When `true`, /// the cell is dropped from bulk selection unless specifically requested /// (`meta/docs/2026-07-22-fragment-skip.md`). @@ -203,6 +205,7 @@ fn render_cell( .iter() .flat_map(|f| f.doc.extra_params.clone()) .collect(); + let signing = resolve_signing_ref(image, &tuple, &matched)?; // Resolve `skip` last-wins over the matched fragments (base → most-specific). When the winning // value is `true`, remember that fragment's predicate coordinates as the pins that can override @@ -223,6 +226,7 @@ fn render_cell( outputs, rpm_sources, extra_params, + signing, skip, skip_pins, }) @@ -323,6 +327,20 @@ fn resolve_outputs( } } +fn resolve_signing_ref( + _image: &ImageDefinition, + _tuple: &AxisTuple, + matched: &[&LoadedFragment], +) -> Result, ConfigError> { + let mut signing: Option = None; + for fragment in matched { + if let Some(value) = fragment.doc.signing.clone() { + signing = Some(value); + } + } + Ok(signing) +} + /// Reject `compression:` on formats where it makes no sense: `cosi` (IC already compresses it), /// `iso` (compressing the image breaks bootability), and the `pxe-*` outputs (a directory / an /// already-gzipped tar). Compression is a tailor post-step over a single raw disk image. diff --git a/tools/tailor/crates/tailor-core/src/domain.rs b/tools/tailor/crates/tailor-core/src/domain.rs index 56835740b9..f80a015432 100644 --- a/tools/tailor/crates/tailor-core/src/domain.rs +++ b/tools/tailor/crates/tailor-core/src/domain.rs @@ -6,7 +6,7 @@ use std::{collections::BTreeMap, fmt, path::PathBuf, sync::Arc}; use serde_yaml_ng::Value; use tailor_config::{ Arch, BaseImageCatalogue, BaseSource, ExtraParam, ImageDefinition, OutputArtifactsPolicy, - OutputSpec, ToolsDirSource, ToolsDirSourceInline, + OutputSpec, SigningRef, ToolsDirSource, ToolsDirSourceInline, }; /// A resolved image — the catalogue/authoring unit, after config load and defaults are applied. @@ -62,6 +62,8 @@ pub struct Cell { pub rpm_sources: Vec, /// Extra IC command-line flags appended verbatim after every tailor-managed flag. pub extra_params: Vec, + /// The resolved per-cell `signing:` opt-in after fragment merging. + pub signing: Option, /// Resolved `${inputs.}` producer artifact paths substituted into `ic_config`, in declared /// order. Content-hashed into the fingerprint (like `extraDependencies`) so a producer rebuild /// invalidates this cell (`meta/docs/2026-09-09-inter-image-dependencies.md`). Live in the (already diff --git a/tools/tailor/crates/tailor-core/src/orchestrator.rs b/tools/tailor/crates/tailor-core/src/orchestrator.rs index d43617f131..fcc1a31cbf 100644 --- a/tools/tailor/crates/tailor-core/src/orchestrator.rs +++ b/tools/tailor/crates/tailor-core/src/orchestrator.rs @@ -556,6 +556,7 @@ pub fn cells(target: &Arc) -> Result, CoreError> { base_image: base_image.clone(), rpm_sources: rc.rpm_sources.clone(), extra_params: rc.extra_params.clone(), + signing: rc.signing.clone(), input_deps: Vec::new(), tools_dir, skip: rc.skip, diff --git a/tools/tailor/crates/tailor-core/src/ports.rs b/tools/tailor/crates/tailor-core/src/ports.rs index fa7b5ab0f0..4782053835 100644 --- a/tools/tailor/crates/tailor-core/src/ports.rs +++ b/tools/tailor/crates/tailor-core/src/ports.rs @@ -85,10 +85,10 @@ pub struct ToolsDirPlan { /// The host-side signing port: sign the boot artifacts IC emits from a `customize` pass, in place, /// so IC's `inject-files` pass can re-inject them (`meta/docs/2026-06-29-signing.md` §6). Object-safe and /// **synchronous** — held as `dyn Signer` in [`ExecutionContext`]; the executor invokes it on a -/// blocking thread so the async runtime is never blocked on `openssl`/`sbsign`. +/// blocking thread so the async runtime is never blocked on `openssl`/`sbsign`/`pesign`. pub trait Signer: Send + Sync + fmt::Debug { /// Cheap, side-effect-free check that this backend can sign: required host binaries present - /// (`openssl`, and `sbsign` when PE artifacts are expected) and key material resolvable. Called + /// (`openssl`, plus either `sbsign` or the `pesign` toolchain when PE artifacts are expected) and key material resolvable. Called /// once per build, before any IC run (`meta/docs/2026-06-29-signing.md` §5.1). fn preflight(&self) -> Result<(), SignError>; diff --git a/tools/tailor/crates/tailor-core/src/signing.rs b/tools/tailor/crates/tailor-core/src/signing.rs index d86fa2428f..8fd0f93a39 100644 --- a/tools/tailor/crates/tailor-core/src/signing.rs +++ b/tools/tailor/crates/tailor-core/src/signing.rs @@ -6,7 +6,7 @@ //! is cheap and side-effect-free. //! //! This module is the **foundation** (config + preflight). The signing *execution* — cert minting via -//! `rcgen`, PE signing via `sbsign`, and the `inject-files` IC pass — is a later milestone +//! `rcgen`, PE signing via `sbsign`/`pesign`, and the `inject-files` IC pass — is a later milestone //! (`meta/docs/2026-06-29-signing.md` §11, S1-remainder). Until it lands, `tailor` refuses a signed build rather //! than silently emit an unsigned image. @@ -42,7 +42,7 @@ pub enum SignError { /// Preflight found unmet prerequisites — a missing tool binary, an unreadable key, etc. Reported /// in aggregate, before any (slow, privileged) IC run. Preflight { missing: Vec }, - /// A signing step failed during execution (openssl/sbsign, `inject-files.yaml` handling, or IO). + /// A signing step failed during execution (openssl/sbsign/pesign, `inject-files.yaml` handling, or IO). Execution { detail: String }, } diff --git a/tools/tailor/crates/tailor-exec/src/arg_builder.rs b/tools/tailor/crates/tailor-exec/src/arg_builder.rs index 2c093df1e6..c975b5e0c4 100644 --- a/tools/tailor/crates/tailor-exec/src/arg_builder.rs +++ b/tools/tailor/crates/tailor-exec/src/arg_builder.rs @@ -124,11 +124,16 @@ pub fn build_ic_args(cell: &Cell, context: &ExecutionContext) -> Result raw intermediate ({intermediate})\n{customize}\n\n\ - # pass 2/3: host-side sign the staged boot artifacts (openssl + sbsign); publish CA -> {ca}\n\ + # pass 2/3: host-side sign the staged boot artifacts (openssl + sbsign/pesign); publish CA -> {ca}\n\ # pass 3/3: inject-files -> final {fmt} ({final})", slug = cell.slug.as_ref(), intermediate = intermediate.display(), @@ -827,6 +832,10 @@ mod tests { assert!(!args.iter().any(|arg| arg == FLAG_CONFIG_FILE)); assert!(!args.iter().any(|arg| arg == FLAG_RPM_SOURCE)); + // IC's `convert` subcommand has no --image-cache-dir flag (only `customize` downloads/caches + // OCI/azureLinux bases) and rejects unknown flags outright, so tailor must never emit it here + // even though sample_context() configures a cache dir (as a real workspace would). + assert!(!args.iter().any(|arg| arg == FLAG_IMAGE_CACHE_DIR)); assert_eq!(args[0], SUBCOMMAND_CONVERT); assert!( args.windows(2) diff --git a/tools/tailor/crates/tailor-sign/Cargo.toml b/tools/tailor/crates/tailor-sign/Cargo.toml index b07e96cfd9..938937b506 100644 --- a/tools/tailor/crates/tailor-sign/Cargo.toml +++ b/tools/tailor/crates/tailor-sign/Cargo.toml @@ -15,6 +15,7 @@ serde = { workspace = true } serde_yaml_ng = { workspace = true } tracing = { workspace = true } tempfile = { workspace = true } +xxhash-rust = { workspace = true } [dev-dependencies] diff --git a/tools/tailor/crates/tailor-sign/src/lib.rs b/tools/tailor/crates/tailor-sign/src/lib.rs index 63f2e8c046..9870466684 100644 --- a/tools/tailor/crates/tailor-sign/src/lib.rs +++ b/tools/tailor/crates/tailor-sign/src/lib.rs @@ -2,8 +2,9 @@ //! (`meta/docs/2026-06-29-signing.md` §6). //! //! External **host** tools only, no bundled crypto: `openssl` mints the per-cell CA + code-signing -//! leaf and signs verity-hash artifacts (detached CMS/DER), and `sbsign` signs PE/Authenticode boot -//! artifacts (UKIs, shim, systemd-boot). The signer reads the `inject-files.yaml` IC emits from a +//! leaf and signs verity-hash artifacts (detached CMS/DER), while PE/Authenticode boot artifacts +//! (UKIs, shim, systemd-boot) are signed with either `sbsign` or `pesign` depending on which host +//! toolchain is available. The signer reads the `inject-files.yaml` IC emits from a //! `customize` pass and signs each listed artifact **in place**, so IC's `inject-files` pass can //! re-inject the now-signed files. @@ -18,6 +19,7 @@ use std::{ use serde::Deserialize; use tempfile::TempDir; use tracing::debug; +use xxhash_rust::xxh3::xxh3_64; use tailor_config::{SigningBackend, SigningProfile}; use tailor_core::{MissingPrerequisite, SignError, Signer, SigningPlan, SigningResult}; @@ -26,6 +28,13 @@ use tailor_core::{MissingPrerequisite, SignError, Signer, SigningPlan, SigningRe const OPENSSL: &str = "openssl"; /// The `sbsign` binary — PE/Authenticode signing. const SBSIGN: &str = "sbsign"; +/// The `pesign` binary — alternate PE/Authenticode signing backend used on AZL/Mariner hosts. +const PESIGN: &str = "pesign"; +/// NSS DB init/import tools used by the `pesign` fallback. +const CERTUTIL: &str = "certutil"; +const PK12UTIL: &str = "pk12util"; +/// Friendly name used when importing a generated PEM keypair into a temporary NSS DB for `pesign`. +const PESIGN_NICKNAME: &str = "tailor-signing-leaf"; /// Validity of the minted local test CA and leaf. Long enough that CI never trips over expiry; this /// is explicitly **not** a production trust root (`meta/docs/2026-06-29-signing.md` §6). const CERT_VALIDITY_DAYS: &str = "3650"; @@ -88,7 +97,7 @@ struct Ca { cert: PathBuf, } -/// A host-tool signer (`openssl` + `sbsign`) parameterized by its [`KeySource`]. +/// A host-tool signer (`openssl` + `sbsign`/`pesign`) parameterized by its [`KeySource`]. #[derive(Debug)] struct HostSigner { profile_id: String, @@ -108,19 +117,16 @@ impl Signer for HostSigner { }); }; - // Both tools are checked up front (fail-fast, `meta/docs/2026-06-29-signing.md` §5.1): `openssl` mints and - // signs verity-hash, `sbsign` signs PE artifacts. We cannot yet know which artifact types IC - // will emit, so require both whenever a profile signs — a missing tool fails at build start - // rather than minutes into a customize. + // `openssl` is always required (CA/leaf minting + verity CMS). PE artifacts are signed with + // either `sbsign` or the AZL/Mariner-native `pesign` toolchain, so fail fast only when + // neither path is available. if !tool_on_path(OPENSSL) { note(format!( "`{OPENSSL}` not found on PATH (required to mint/sign)" )); } - if !tool_on_path(SBSIGN) { - note(format!( - "`{SBSIGN}` not found on PATH (required to sign PE/UKI boot artifacts)" - )); + if let Err(detail) = check_pe_signer_tools() { + note(detail); } match &self.key_source { KeySource::LocalTestCa { .. } => {} @@ -297,7 +303,7 @@ enum ArtifactType { /// How an artifact is signed. #[derive(Debug, PartialEq, Eq)] enum ArtifactKind { - /// PE/Authenticode via `sbsign` (UKIs, shim, systemd-boot `.efi`). + /// PE/Authenticode via `sbsign` or `pesign` (UKIs, shim, systemd-boot `.efi`). Pe, /// Detached CMS/DER via `openssl smime` (dm-verity root hash). Verity, @@ -326,7 +332,7 @@ fn infer_kind(source: &Path) -> ArtifactKind { } } -// ───────────────────────────── openssl / sbsign orchestration ───────────────────────────── +// ───────────────────────────── openssl / sbsign / pesign orchestration ───────────────────────────── /// Mint a self-signed CA (RSA-2048) with `openssl req -x509`. fn mint_ca(ca_key: &Path, ca_cert: &Path) -> Result<(), SignError> { @@ -350,8 +356,31 @@ fn mint_ca(ca_key: &Path, ca_cert: &Path) -> Result<(), SignError> { ) } +/// X.509 limits a `CommonName` attribute to 64 characters (RFC 5280 `ub-common-name`); OpenSSL's +/// `ASN1_mbstring_ncopy` enforces this and aborts the whole `req` with "string too long" if we hand +/// it anything longer. Cell slugs (image + every matrix axis + format, optionally `_cloneN`) easily +/// exceed that once `leaf_id` is embedded verbatim in `/CN=tailor leaf {leaf_id}` (12 literal chars +/// leave only 52 for `leaf_id`), so bound the CN to a safe length and disambiguate any truncation +/// with a short content hash - the full `leaf_id` is still the one used for on-disk file naming +/// elsewhere, this only affects the certificate's CN attribute. +const CN_MAX_LEN: usize = 64; +const CN_PREFIX: &str = "tailor leaf "; + +fn leaf_common_name(leaf_id: &str) -> String { + let budget = CN_MAX_LEN - CN_PREFIX.len(); + if leaf_id.len() <= budget { + return format!("{CN_PREFIX}{leaf_id}"); + } + // Keep a readable prefix and append a short hash of the *full* leaf_id so two different + // leaf_ids that share a truncated prefix never collide on CN alone. + let hash = format!("{:016x}", xxh3_64(leaf_id.as_bytes())); + let keep = budget.saturating_sub(hash.len() + 1); // +1 for the separator + format!("{CN_PREFIX}{}-{hash}", &leaf_id[..keep]) +} + /// Mint a code-signing leaf (`extendedKeyUsage=codeSigning`) signed by the CA. The leaf CN embeds the -/// per-cell `leaf_id` so parallel cells never share a leaf. +/// per-cell `leaf_id` (bounded to fit X.509's 64-character CN limit, see [`leaf_common_name`]) so +/// parallel cells never share a leaf. fn mint_leaf( ca_key: &Path, ca_cert: &Path, @@ -373,7 +402,7 @@ fn mint_leaf( "-out", &lossy(&csr), "-subj", - &format!("/CN=tailor leaf {leaf_id}"), + &format!("/CN={}", leaf_common_name(leaf_id)), ], )?; // codeSigning EKU via an ext file (process substitution isn't available through `Command`). @@ -402,8 +431,21 @@ fn mint_leaf( ) } -/// PE/Authenticode-sign `file` in place with `sbsign` (writes to a temp output, then renames over). +/// PE/Authenticode-sign `file` in place, preferring `sbsign` and falling back to `pesign` when the +/// host only has the AZL/Mariner-native toolchain. fn sign_pe(file: &Path, key: &Path, cert: &Path) -> Result<(), SignError> { + match pe_signing_tool() { + Some(PeSigningTool::Sbsign) => sign_pe_with_sbsign(file, key, cert), + Some(PeSigningTool::Pesign) => sign_pe_with_pesign(file, key, cert), + None => Err(SignError::Execution { + detail: format!( + "neither `{SBSIGN}` nor the `pesign` toolchain (`{PESIGN}`, `{CERTUTIL}`, `{PK12UTIL}`) is available on PATH" + ), + }), + } +} + +fn sign_pe_with_sbsign(file: &Path, key: &Path, cert: &Path) -> Result<(), SignError> { let signed = with_suffix(file, ".signed"); run( SBSIGN, @@ -420,6 +462,63 @@ fn sign_pe(file: &Path, key: &Path, cert: &Path) -> Result<(), SignError> { fs::rename(&signed, file).map_err(|e| io_err("replace with signed PE", file, &e)) } +fn sign_pe_with_pesign(file: &Path, key: &Path, cert: &Path) -> Result<(), SignError> { + let temp = TempDir::new().map_err(|e| SignError::Execution { + detail: format!("create pesign temp dir: {e}"), + })?; + let nssdb = temp.path().join("nssdb"); + fs::create_dir_all(&nssdb).map_err(|e| io_err("create pesign NSS DB dir", &nssdb, &e))?; + run(CERTUTIL, &["-N", "-d", &lossy(&nssdb), "--empty-password"])?; + + let pkcs12 = temp.path().join("leaf.p12"); + run( + OPENSSL, + &[ + "pkcs12", + "-export", + "-out", + &lossy(&pkcs12), + "-inkey", + &lossy(key), + "-in", + &lossy(cert), + "-passout", + "pass:", + "-name", + PESIGN_NICKNAME, + ], + )?; + run( + PK12UTIL, + &[ + "-i", + &lossy(&pkcs12), + "-d", + &lossy(&nssdb), + "-W", + "", + ], + )?; + + let cert_arg = pesign_certificate_arg()?; + let signed = with_suffix(file, ".signed"); + let args = vec![ + "--certdir".to_owned(), + lossy(&nssdb), + cert_arg, + PESIGN_NICKNAME.to_owned(), + "--sign".to_owned(), + "--in".to_owned(), + lossy(file), + "--out".to_owned(), + lossy(&signed), + "--force".to_owned(), + ]; + let arg_refs: Vec<&str> = args.iter().map(String::as_str).collect(); + run(PESIGN, &arg_refs)?; + fs::rename(&signed, file).map_err(|e| io_err("replace with signed PE", file, &e)) +} + /// Sign a dm-verity root hash with a detached CMS/DER signature (`openssl smime`), replacing the /// artifact in place — mirroring Trident's `sign.py`. /// @@ -449,6 +548,58 @@ fn sign_verity(file: &Path, key: &Path, cert: &Path) -> Result<(), SignError> { fs::rename(&sig, file).map_err(|e| io_err("replace with verity signature", file, &e)) } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PeSigningTool { + Sbsign, + Pesign, +} + +fn pe_signing_tool() -> Option { + if tool_on_path(SBSIGN) { + Some(PeSigningTool::Sbsign) + } else if tool_on_path(PESIGN) && tool_on_path(CERTUTIL) && tool_on_path(PK12UTIL) { + Some(PeSigningTool::Pesign) + } else { + None + } +} + +fn check_pe_signer_tools() -> Result<(), String> { + if pe_signing_tool().is_some() { + return Ok(()); + } + let mut missing = Vec::new(); + for tool in [PESIGN, CERTUTIL, PK12UTIL] { + if !tool_on_path(tool) { + missing.push(format!("`{tool}`")); + } + } + Err(format!( + "neither `{SBSIGN}` nor the `pesign` toolchain is available on PATH (need `{SBSIGN}` or `{PESIGN}` + `{CERTUTIL}` + `{PK12UTIL}` to sign PE/UKI boot artifacts; missing: {})", + missing.join(", ") + )) +} + +fn pesign_certificate_arg() -> Result { + let output = Command::new(PESIGN) + .arg("--help") + .output() + .map_err(|e| SignError::Execution { + detail: format!("failed to run `{PESIGN} --help`: {e}"), + })?; + let help = format!( + "{} +{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + Ok(if help.contains("--certficate") { + "--certficate".to_owned() + } else { + "--certificate".to_owned() + }) +} + // ───────────────────────────── small helpers ───────────────────────────── /// Is `name` an executable file on `PATH`? A cheap, side-effect-free presence probe for preflight. @@ -532,6 +683,35 @@ mod tests { tool_on_path(OPENSSL) } + #[test] + fn leaf_common_name_fits_within_64_chars_for_short_ids() { + let cn = leaf_common_name("solo_amd64_cosi"); + assert_eq!(cn, "tailor leaf solo_amd64_cosi"); + assert!(cn.len() <= 64); + } + + #[test] + fn leaf_common_name_truncates_and_hashes_long_ids() { + // Reproduces the exact real-world overflow: a `trident-verity-testimage` container/usr + // clone0 slug is 69 chars once prefixed with "tailor leaf ", which OpenSSL's + // ASN1_mbstring_ncopy (ub-common-name = 64) rejects outright. + let long_id = "trident-verity-testimage_container_usr_amd64_cosi_clone0"; + assert!(format!("tailor leaf {long_id}").len() > 64); + + let cn = leaf_common_name(long_id); + assert!(cn.len() <= 64, "CN still too long: {cn:?} ({} chars)", cn.len()); + assert!(cn.starts_with("tailor leaf ")); + + // Two different long ids sharing the same truncated prefix must not collide. + let other_id = "trident-verity-testimage_container_usr_amd64_cosi_clone1"; + let other_cn = leaf_common_name(other_id); + assert_ne!(cn, other_cn); + assert!(other_cn.len() <= 64); + + // Deterministic: same input always yields the same CN. + assert_eq!(cn, leaf_common_name(long_id)); + } + #[test] fn artifact_type_dispatch_covers_every_kind() { let pe = |t: ArtifactType| InjectArtifact { @@ -590,7 +770,8 @@ mod tests { let SignError::Preflight { missing } = signer.preflight().unwrap_err() else { panic!("expected a preflight error"); }; - // openssl/sbsign presence varies by host, so assert on the key/cert prerequisites we control. + // PE signer availability varies by host (`sbsign` vs `pesign`), so assert only on the + // key/cert prerequisites we control here. assert!(missing.iter().any(|m| m.detail.contains("`key`"))); assert!(missing.iter().any(|m| m.detail.contains("`cert`"))); } @@ -614,7 +795,7 @@ mod tests { let dir = tempdir().unwrap(); let artifacts = dir.path().join("artifacts"); fs::create_dir_all(&artifacts).unwrap(); - // A verity-hash artifact (openssl can sign arbitrary bytes; sbsign needs a real PE, so this + // A verity-hash artifact (openssl can sign arbitrary bytes; PE signing needs a real EFI // test exercises the openssl path end to end without a PE fixture). fs::write(artifacts.join("root.hash"), b"deadbeef root hash bytes").unwrap(); fs::write( diff --git a/tools/tailor/crates/tailor/src/run.rs b/tools/tailor/crates/tailor/src/run.rs index fff51e9559..e0df368563 100644 --- a/tools/tailor/crates/tailor/src/run.rs +++ b/tools/tailor/crates/tailor/src/run.rs @@ -338,7 +338,29 @@ fn validate(workspace: &Workspace, names: &[String], selector: &Selector) -> Res } // Surface signing prerequisites non-fatally (meta/docs/2026-06-29-signing.md §5.1) so they are discoverable // without starting a real build. - let signing = signing_requirements(&targets, tool.signing.as_ref())?; + let mut signing: Vec> = Vec::new(); + for target in &targets { + for cell in cells_selected(target, selector)? { + let Some((profile_id, profile)) = + tailor_config::resolve_signing(cell.signing.as_ref(), tool.signing.as_ref())? + else { + continue; + }; + let slug = cell.slug.as_ref().to_owned(); + if let Some(existing) = signing + .iter_mut() + .find(|requirement| requirement.profile_id == profile_id) + { + existing.images.push(slug); + } else { + signing.push(SigningRequirement { + profile_id, + profile, + images: vec![slug], + }); + } + } + } ensure_signing_preview(&tool, &signing)?; report_signing(&build_signers(&signing, &workspace.root)); Ok(()) @@ -1424,6 +1446,7 @@ fn convert_cell(input: &Path, dir: &Path, arch: Arch, format: OutputFormat, slug base_image: None, rpm_sources: Vec::new(), extra_params: Vec::new(), + signing: None, input_deps: Vec::new(), tools_dir: None, skip: false, @@ -1501,21 +1524,22 @@ async fn build( .get_or_insert_with(Runtime::default) .build_dir_base = Some(output_dir.join(TAILOR_STATE_DIR).join(BUILD_DIR)); } - let signing = signing_requirements(&targets, tool.signing.as_ref())?; + let signing = signing_requirements(&ordered, &build_selection, tool.signing.as_ref())?; ensure_signing_preview(&tool, &signing)?; // Build one signer per required profile (a shared CA per build); `signer_for` resolves a cell to - // its signer via the image name (meta/docs/2026-06-29-signing.md §6). Empty when nothing signs. + // its signer via the rendered cell slug, so only the cells whose merged fragments opted into + // `signing:` take the signed three-pass path. let signers = build_signers(&signing, &workspace.root); - let image_signer: HashMap<&str, Arc> = signers + let slug_signer: HashMap<&str, Arc> = signers .iter() .flat_map(|(requirement, signer)| { requirement .images .iter() - .map(move |image| (image.as_str(), Arc::clone(signer))) + .map(move |slug| (slug.as_str(), Arc::clone(signer))) }) .collect(); - let signer_for = |cell: &Cell| image_signer.get(cell.target.name()).cloned(); + let signer_for = |cell: &Cell| slug_signer.get(cell.slug.as_ref()).cloned(); // Dry-run prints each selected cell's container invocation (the signed 3-pass for signed cells) // without resolving digests, running, or contacting any engine — daemon-free via a no-op runtime. @@ -1542,7 +1566,7 @@ async fn build( return Ok(()); } - // Fail fast on every signing prerequisite — including the `openssl`/`sbsign` binaries — before any + // Fail fast on every signing prerequisite — including the `openssl`/PE-signing binaries — before any // (slow, privileged) IC run (meta/docs/2026-06-29-signing.md §5.1). preflight_signers(&signers)?; @@ -1677,14 +1701,11 @@ async fn clean( for target in &targets { // A signed image also drops an enrollable CA cert beside each cell's image (§6); remove it // too. Lenient on config: `validate` surfaces signing errors; cleanup should not fail on them. - let signed = tailor_config::resolve_signing( - target.definition.signing.as_ref(), - tool.signing.as_ref(), - ) - .ok() - .flatten() - .is_some(); for cell in cells_selected(target, selector)? { + let signed = tailor_config::resolve_signing(cell.signing.as_ref(), tool.signing.as_ref()) + .ok() + .flatten() + .is_some(); paths.push(output_dir.join(tailor_core::artifact_name( cell.slug.as_ref(), cell.output.format, @@ -1978,25 +1999,27 @@ fn ensure_signing_preview( /// each (`meta/docs/2026-06-29-signing.md` §5.1). Resolution also validates each referenced profile. fn signing_requirements<'a>( targets: &[Arc], + selector: &BuildSelection<'_>, workspace_signing: Option<&'a tailor_config::SigningConfig>, ) -> Result>, AppError> { let mut requirements: Vec> = Vec::new(); for target in targets { - let resolved = - tailor_config::resolve_signing(target.definition.signing.as_ref(), workspace_signing)?; - if let Some((profile_id, profile)) = resolved { - let image = target.definition.name.clone(); - if let Some(existing) = requirements - .iter_mut() - .find(|requirement| requirement.profile_id == profile_id) - { - existing.images.push(image); - } else { - requirements.push(SigningRequirement { - profile_id, - profile, - images: vec![image], - }); + for cell in select_node_cells(target, selector.selector, selector.requested.contains(target.name()), selector.required.get(target.name()))? { + let resolved = tailor_config::resolve_signing(cell.signing.as_ref(), workspace_signing)?; + if let Some((profile_id, profile)) = resolved { + let slug = cell.slug.as_ref().to_owned(); + if let Some(existing) = requirements + .iter_mut() + .find(|requirement| requirement.profile_id == profile_id) + { + existing.images.push(slug); + } else { + requirements.push(SigningRequirement { + profile_id, + profile, + images: vec![slug], + }); + } } } } @@ -2053,7 +2076,7 @@ fn preflight_signers(signers: &[ProfileSigner<'_>]) -> Result<(), AppError> { } /// Report signing-preflight status **without failing** — for `validate` and `--dry-run` -/// (`meta/docs/2026-06-29-signing.md` §5.1). Runs each signer's preflight so missing `openssl`/`sbsign` binaries +/// (`meta/docs/2026-06-29-signing.md` §5.1). Runs each signer's preflight so missing `openssl`/PE-signing binaries /// surface too. fn report_signing(signers: &[ProfileSigner<'_>]) { for (requirement, signer) in signers { diff --git a/tools/tailor/docs/explanation/crate-architecture.md b/tools/tailor/docs/explanation/crate-architecture.md index 578c8c38ee..c63b96532d 100644 --- a/tools/tailor/docs/explanation/crate-architecture.md +++ b/tools/tailor/docs/explanation/crate-architecture.md @@ -21,7 +21,7 @@ graph TD | `tailor-core` | Domain model, build plans, lockfile/stamp logic, orchestration, and port traits. | | `tailor-resolve` | Resolve toolchain and base image digests/hashes. | | `tailor-exec` | Docker/Bollard execution adapter, IC arg construction, path translation, cleanup. | -| `tailor-sign` | Host-side signing backends for the `Signer` port (openssl/`sbsign`); preview-gated. | +| `tailor-sign` | Host-side signing backends for the `Signer` port (openssl/`sbsign`/`pesign`); preview-gated. | | `tailor` | CLI parsing, command dispatch, output formatting, and composition root. | The config-to-render path is deterministic and synchronous. Container execution is the async boundary because Docker/Bollard and log streaming are async.