Migrate tests/images to tailor

.pipelines/templates/stages/build_image/build-image-template.yml

@@ -62,6 +62,10 @@ parameters:
 steps:
   - template: ../common_tasks/avoid-pypi-usage.yml
 
+  - template: ../common_tasks/cargo-auth.yml
+    parameters:
+      cargoConfigPath: $(TRIDENT_SOURCE_DIR)/.cargo/config.toml
+
   - template: common/sfi-enforce-isolation-with-etc-hosts.yaml@platform-pipelines
 
   - script: |
@@ -82,8 +86,17 @@ steps:
     retryCountOnTaskFailure: 3
 
   - bash: |
-      ./tests/images/testimages.py show-image ${{ parameters.imageName }} base-image --devops-var baseImageType
-    displayName: "Get Image Info"
+      set -euo pipefail
+      python3 - <<'PY' "${{ parameters.imageName }}"
+      import json
+      import sys
+      from pathlib import Path
+
+      legacy = json.loads(Path("tests/images/legacy-map.json").read_text())
+      entry = legacy[sys.argv[1]]
+      print(f"##vso[task.setvariable variable=baseImageType]{entry['baseImage']}")
+      PY
+    displayName: "Resolve image metadata"
     workingDirectory: ${{ parameters.tridentSourceDirectory }}
 
   - bash: |
@@ -117,10 +130,27 @@ steps:
       azureLinuxVersion: ${{ parameters.azureLinuxVersion }}
       runtimeBuildType: ${{ parameters.baseimgBuildType }}
 
+  - ${{ if eq(parameters.architecture, 'arm64') }}:
+    - task: DownloadPipelineArtifact@2
+      displayName: "Download go-tools for direct-streaming installer"
+      condition: or(eq('${{ parameters.imageName }}', 'trident-direct-streaming-installer-amd64'), eq('${{ parameters.imageName }}', 'trident-direct-streaming-installer-arm64'))
+      inputs:
+        buildType: current
+        artifactName: go-tools-arm64
+        targetPath: "$(Build.ArtifactStagingDirectory)/go-tools"
+  - ${{ if ne(parameters.architecture, 'arm64') }}:
+    - task: DownloadPipelineArtifact@2
+      displayName: "Download go-tools for direct-streaming installer"
+      condition: or(eq('${{ parameters.imageName }}', 'trident-direct-streaming-installer-amd64'), eq('${{ parameters.imageName }}', 'trident-direct-streaming-installer-arm64'))
+      inputs:
+        buildType: current
+        artifactName: go-tools
+        targetPath: "$(Build.ArtifactStagingDirectory)/go-tools"
+
   - bash: |
       set -ex
 
-      # Move base VHDX to artifacts/ (builder expects artifacts/*.vhdx)
+      # Move base VHDX to artifacts/ (tailor baseImages catalogue points at artifacts/*.vhdx)
       mkdir -p artifacts
       if ls "$(Build.ArtifactStagingDirectory)/images" | grep -q ".*\.vhdx$"; then
         mv $(Build.ArtifactStagingDirectory)/images/*.vhdx artifacts/
@@ -144,28 +174,114 @@ steps:
         DISTRO=azl4
       fi
 
-      # Move Trident RPMs to bin/RPMS/ (builder expects bin/RPMS/*.rpm)
+      # Move Trident RPMs to bin/RPMS/ (tailor rpmSources point at bin/RPMS/)
       if [ -d "$(Build.ArtifactStagingDirectory)/trident" ]; then
         mkdir -p bin/RPMS
         find "$(Build.ArtifactStagingDirectory)/trident" -name "*${DISTRO}*.rpm" -exec mv {} bin/RPMS/ \;
         rm -rf "$(Build.ArtifactStagingDirectory)/trident"
       fi
+
+      if [[ "${{ parameters.imageName }}" == trident-direct-streaming-installer-* ]]; then
+        if [ -f "$(Build.ArtifactStagingDirectory)/go-tools/rcp-agent" ]; then
+          cp "$(Build.ArtifactStagingDirectory)/go-tools/rcp-agent" artifacts/rcp-agent
+        elif [ -f bin/rcp-agent ]; then
+          cp bin/rcp-agent artifacts/rcp-agent
+        else
+          echo "Missing rcp-agent for direct-streaming installer"
+          exit 1
+        fi
+        chmod +x artifacts/rcp-agent
+        cp tools/cmd/rcp-agent/rcp-agent.service artifacts/rcp-agent.service
+      fi
     displayName: "Prepare and move requirements"
     workingDirectory: ${{ parameters.tridentSourceDirectory }}
 
   - bash: |
-      set -ex
+      set -euo pipefail
+
+      legacy_name='${{ parameters.imageName }}'
+      export PATH="$HOME/.cargo/bin:$PATH"
+      export CLICOLOR_FORCE=1
 
-      EXTRA_ARGS=""
+      readarray -t selector_lines < <(python3 - <<'PY' "$legacy_name"
+      import json
+      import sys
+      from pathlib import Path
 
+      entry = json.loads(Path("tests/images/legacy-map.json").read_text())[sys.argv[1]]
+      print(entry["image"])
+      print(entry["ext"])
+      print(entry.get("tailorExt", entry["ext"]))
+      for key, value in entry["selectors"].items():
+          print(f"{key}={value}")
+      PY
+      )
+
+      tailor_image="${selector_lines[0]}"
+      artifact_ext="${selector_lines[1]}"
+      source_ext="${selector_lines[2]}"
+      selector_args=()
+      if (( ${#selector_lines[@]} > 3 )); then
+        for selector in "${selector_lines[@]:3}"; do
+          selector_args+=("-s" "$selector")
+        done
+      fi
+
+      manifest_path=tests/images/tailor.yaml
       if [[ "${{ parameters.micBuildType }}" == "dev" ]]; then
-        EXTRA_ARGS+=" --container imagecustomizer:dev"
+        manifest_path=tests/images/.tailor.pipeline.yaml
+        python3 - <<'PY'
+      from pathlib import Path
+      text = Path("tests/images/tailor.yaml").read_text()
+      needle = """toolchains:
+        default: ic
+        entries:
+          - name: ic
+            container: mcr.microsoft.com/azurelinux/imagecustomizer
+            tag: latest
+      """
+      replacement = """toolchains:
+        default: ic-dev
+        entries:
+          - name: ic
+            container: mcr.microsoft.com/azurelinux/imagecustomizer
+            tag: latest
+          - name: ic-dev
+            container: imagecustomizer
+            tag: dev
+            pull: never
+      """
+      Path("tests/images/.tailor.pipeline.yaml").write_text(text.replace(needle, replacement, 1))
+      PY
+      fi
+
+      matrix_json=$(cargo run --manifest-path tools/tailor/Cargo.toml --quiet --         --manifest "$manifest_path"         matrix "$tailor_image" --format json "${selector_args[@]}")
+
+      slug=$(python3 - <<'PY' "$matrix_json"
+      import json
+      import sys
+      cells = json.loads(sys.argv[1])
+      if len(cells) != 1:
+          raise SystemExit(f"expected exactly one cell, got {len(cells)}")
+      print(cells[0]["slug"])
+      PY
+      )
+
+      rm -f tests/images/artifacts/ca_cert.pem
+      cargo run --manifest-path tools/tailor/Cargo.toml --quiet --         --manifest "$manifest_path"         build "$tailor_image"         "${selector_args[@]}"         --output-dir $(ob_outputDirectory)         --clones ${{ parameters.clones }}
+
+      if [[ ${{ parameters.clones }} -eq 1 ]]; then
+        mv "$(ob_outputDirectory)/${slug}.${source_ext}" "$(ob_outputDirectory)/${legacy_name}.${artifact_ext}"
+      else
+        for ((i=0; i<${{ parameters.clones }}; i++)); do
+          mv "$(ob_outputDirectory)/${slug}_clone${i}.${source_ext}"              "$(ob_outputDirectory)/${legacy_name}_${i}.${artifact_ext}"
+        done
+      fi
+
+      if [ -f tests/images/artifacts/ca_cert.pem ]; then
+        cp tests/images/artifacts/ca_cert.pem "$(ob_outputDirectory)/ca_cert.pem"
       fi
 
-      python3 ./tests/images/testimages.py build \
-          "${{ parameters.imageName }}" $EXTRA_ARGS \
-          --output-dir $(ob_outputDirectory) \
-          --no-download \
-          --clones ${{ parameters.clones }}
+      rm -f tests/images/.tailor.pipeline.yaml
     displayName: "Build ${{ parameters.imageName }}"
     workingDirectory: ${{ parameters.tridentSourceDirectory }}

.pipelines/templates/stages/trident_images/build-image-arm64.yml

@@ -52,6 +52,15 @@ jobs:
       - template: ../common_tasks/checkout_trident.yml
       - template: ../common_tasks/avoid-pypi-usage.yml
 
+      # The Makefile's tailor-backed legacy image targets invoke
+      # `cargo run --manifest-path tools/tailor/Cargo.toml`, so a Rust
+      # toolchain and cargo registry auth are required here even though this
+      # job previously never needed cargo.
+      - template: ../common_tasks/rustup.yml
+      - template: ../common_tasks/cargo-auth.yml
+        parameters:
+          cargoConfigPath: $(TRIDENT_SOURCE_DIR)/.cargo/config.toml
+
       - task: DownloadPipelineArtifact@2
         inputs:
           buildType: current

.pipelines/templates/stages/trident_images/build-image.yml

@@ -101,6 +101,15 @@ jobs:
       - template: ../common_tasks/checkout_trident.yml
       - template: ../common_tasks/avoid-pypi-usage.yml
 
+      # The Makefile's tailor-backed legacy image targets invoke
+      # `cargo run --manifest-path tools/tailor/Cargo.toml`, so a Rust
+      # toolchain and cargo registry auth are required here even though this
+      # job previously never needed cargo.
+      - template: ../common_tasks/rustup.yml
+      - template: ../common_tasks/cargo-auth.yml
+        parameters:
+          cargoConfigPath: $(TRIDENT_SOURCE_DIR)/.cargo/config.toml
+
       - ${{ if parameters.downloadGoTools }}:
         - task: DownloadPipelineArtifact@2
           displayName: "Download $(goToolsArtifactName)"

.pipelines/templates/stages/trident_images/trident-testimg-template.yml

@@ -135,7 +135,7 @@ steps:
         displayName: Copy SSH Keys
 
   - script: |
-      echo "##[warning]THE PIPELINE TEMPLATE trident-testimg-template.yaml IS DEPRECATED. PLEASE SWITCH TO USING testimages.py TO BUILD TEST IMAGES."
+      echo "##[warning]THE PIPELINE TEMPLATE trident-testimg-template.yaml IS DEPRECATED. PLEASE SWITCH TO USING the tailor workspace under tests/images."
       cat /etc/os-release
     displayName: "Report Host Info"
 
@@ -264,6 +264,18 @@ steps:
         rm -rf "$(Build.ArtifactStagingDirectory)/trident"
       fi
 
+      if [[ "${{ parameters.target }}" == artifacts/trident-direct-streaming-installer-* ]]; then
+        mkdir -p '${{ parameters.tridentSourceDirectory }}/artifacts'
+        if [ -f "${{ parameters.tridentSourceDirectory }}/bin/rcp-agent" ]; then
+          cp "${{ parameters.tridentSourceDirectory }}/bin/rcp-agent" '${{ parameters.tridentSourceDirectory }}/artifacts/rcp-agent'
+          chmod +x '${{ parameters.tridentSourceDirectory }}/artifacts/rcp-agent'
+        else
+          echo "Missing rcp-agent in bin/ for direct-streaming installer"
+          exit 1
+        fi
+        cp '${{ parameters.tridentSourceDirectory }}/tools/cmd/rcp-agent/rcp-agent.service' '${{ parameters.tridentSourceDirectory }}/artifacts/rcp-agent.service'
+      fi
+
     workingDirectory: ${{ parameters.tridentSourceDirectory }}
     displayName: "Prepare and move requirements"
 
@@ -280,13 +292,12 @@ steps:
         export MIC_CONTAINER_IMAGE="imagecustomizer:dev"
       fi
 
-      # Allow cross-platform (i.e. amd64 pipeline creating arm64 images)
-      if [[ "${{ parameters.micArchitecture }}" != "${{ parameters.pipelineArchitecture }}" ]]; then
-        # Export variable to tell Makefile and testimages to use `docker --platform`
-        export MIC_ARCHITECTURE="linux/${{ parameters.micArchitecture }}"
-      fi
+      # The Makefile's tailor-backed legacy target helper resolves the cell arch
+      # from the historical image name, so no extra MIC_ARCHITECTURE export is
+      # needed here for cross-arch direct-streaming builds.
 
       # Compress the full image & delete the uncompressed image
+      rm -f tests/images/artifacts/ca_cert.pem
       make ${{ parameters.target }}
     workingDirectory: ${{ parameters.tridentSourceDirectory }}
     displayName: "Build Test Image"
@@ -302,5 +313,9 @@ steps:
         # Everything else is a file
         sudo mv -v "${{ parameters.target }}" "${{ parameters.outputDirectory }}/"
       fi
+
+      if [ -f "tests/images/artifacts/ca_cert.pem" ]; then
+        sudo cp -v "tests/images/artifacts/ca_cert.pem" "${{ parameters.outputDirectory }}/ca_cert.pem"
+      fi
     workingDirectory: ${{ parameters.tridentSourceDirectory }}
     displayName: "Copy artifacts to output directory"