CNTRLPLANE-4025: Test HyperShift Azure Managed HSM on AKS

ci-operator/config/openshift/hypershift/openshift-hypershift-release-5.0__periodics.yaml

@@ -340,6 +340,20 @@ tests:
       ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true"
       HYPERSHIFT_AZURE_LOCATION: centralus
     workflow: hypershift-azure-e2e-v2-self-managed
+- as: e2e-aks-managed-hsm
+  cron: 0 6 * * *
+  steps:
+    cluster_profile: hypershift-aks
+    env:
+      AUTH_THROUGH_CERTS: "true"
+      CI_TESTS_RUN: ^TestCreateClusterCustomConfig$
+      HYPERSHIFT_AZURE_MANAGED_HSM: "true"
+      HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION: eastus
+      HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_OFFER: ""
+      HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_PUBLISHER: ""
+      HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_SKU: ""
+      HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_VERSION: ""
+    workflow: hypershift-azure-aks-e2e
 - as: e2e-azure-kubevirt-ovn
   cron: 0 4 * * *
   steps:

ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-5.0-periodics.yaml

@@ -166,6 +166,89 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build07
+  cron: 0 6 * * *
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: release-5.0
+    org: openshift
+    repo: hypershift
+  labels:
+    ci-operator.openshift.io/cloud: hypershift-aks
+    ci-operator.openshift.io/cloud-cluster-profile: hypershift-aks
+    ci-operator.openshift.io/variant: periodics
+    ci.openshift.io/generator: prowgen
+    job-release: "5.0"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-openshift-hypershift-release-5.0-periodics-e2e-aks-managed-hsm
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --target=e2e-aks-managed-hsm
+      - --variant=periodics
+      command:
+      - ci-operator
+      env:
+      - name: HTTP_SERVER_IP
+        valueFrom:
+          fieldRef:
+            fieldPath: status.podIP
+      image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+      imagePullPolicy: Always
+      name: ""
+      ports:
+      - containerPort: 8080
+        name: http
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build07
   cron: 0 2 * * *

ci-operator/step-registry/hypershift/azure/aks/e2e/hypershift-azure-aks-e2e-workflow.yaml

@@ -17,17 +17,19 @@ workflow:
 
     Learn more about HyperShift here: https://github.com/openshift/hypershift
 
-    Track HyperShift's development here: https://issues.redhat.com/projects/CNTRLPLANE/summary 
+    Track HyperShift's development here: https://issues.redhat.com/projects/CNTRLPLANE/summary
   steps:
     pre:
     - chain: cucushift-installer-rehearse-azure-aks-provision
     - ref: hypershift-azure-aks-attach-kv
+    - ref: hypershift-azure-aks-managed-hsm-provision
     - ref: hypershift-install
     - ref: cucushift-hypershift-extended-k8s-mgmt-get-guest-annotations
     - ref: hypershift-resolve-nodepool-releases
     test:
     - ref: hypershift-azure-run-e2e
     post:
+    - ref: hypershift-azure-aks-managed-hsm-deprovision
     - chain: cucushift-installer-rehearse-azure-aks-deprovision
     env:
       AKS_NODE_COUNT: "6"
@@ -48,6 +50,8 @@ workflow:
       AKS_CERT_ROTATION_POLL_INTERVAL: "1m"
       AKS_USE_HYPERSHIFT_MI: "true"
       ENABLE_SIZE_TAGGING: "true"
+      HYPERSHIFT_AZURE_MANAGED_HSM: "false"
+      HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION: "eastus"
       HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_OFFER: "aro4"
       HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_PUBLISHER: "azureopenshift"
       HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_VERSION: "419.6.20250523"

ci-operator/step-registry/hypershift/azure/aks/managed-hsm/OWNERS

@@ -0,0 +1,9 @@
+approvers:
+- csrwng
+- enxebre
+- bryan-cox
+options: {}
+reviewers:
+- csrwng
+- enxebre
+- bryan-cox

ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/OWNERS

@@ -0,0 +1,9 @@
+approvers:
+- csrwng
+- enxebre
+- bryan-cox
+options: {}
+reviewers:
+- csrwng
+- enxebre
+- bryan-cox

ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-commands.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+
+set -euo pipefail
+
+retry() {
+  local attempts="$1"
+  local delay="$2"
+  shift 2
+
+  local attempt
+  for ((attempt = 1; attempt <= attempts; attempt++)); do
+    if "$@"; then
+      return 0
+    fi
+    if ((attempt == attempts)); then
+      echo "Command failed after ${attempts} attempts: $*" >&2
+      return 1
+    fi
+    echo "Attempt ${attempt}/${attempts} failed; retrying in ${delay} seconds"
+    sleep "${delay}"
+  done
+}
+
+deleted_hsm_exists() {
+  local count
+  count="$(az keyvault list-deleted \
+    --resource-type hsm \
+    --query "[?name=='${HSM_NAME}'] | length(@)" \
+    -o tsv)"
+  [[ "${count}" == "1" ]]
+}
+
+deleted_hsm_absent() {
+  local count
+  count="$(az keyvault list-deleted \
+    --resource-type hsm \
+    --query "[?name=='${HSM_NAME}'] | length(@)" \
+    -o tsv)"
+  [[ "${count}" == "0" ]]
+}
+
+if [[ "${HYPERSHIFT_AZURE_MANAGED_HSM}" != "true" ]]; then
+  echo "Managed HSM deprovisioning is disabled"
+  exit 0
+fi
+
+HSM_NAME_FILE="${SHARED_DIR}/azure_managed_hsm_name"
+RESOURCE_GROUP_FILE="${SHARED_DIR}/azure_managed_hsm_resource_group"
+LOCATION_FILE="${SHARED_DIR}/azure_managed_hsm_location"
+if [[ ! -s "${HSM_NAME_FILE}" || ! -s "${RESOURCE_GROUP_FILE}" || ! -s "${LOCATION_FILE}" ]]; then
+  echo "Managed HSM cleanup cannot continue because resource information is missing" >&2
+  exit 1
+fi
+
+AZURE_AUTH_LOCATION="/etc/hypershift-ci-jobs-azurecreds/credentials.json"
+AZURE_AUTH_CLIENT_ID="$(jq -er .clientId "${AZURE_AUTH_LOCATION}")"
+AZURE_AUTH_CLIENT_SECRET="$(jq -er .clientSecret "${AZURE_AUTH_LOCATION}")"
+AZURE_AUTH_TENANT_ID="$(jq -er .tenantId "${AZURE_AUTH_LOCATION}")"
+AZURE_AUTH_SUBSCRIPTION_ID="$(jq -er .subscriptionId "${AZURE_AUTH_LOCATION}")"
+HSM_NAME="$(<"${HSM_NAME_FILE}")"
+RESOURCE_GROUP="$(<"${RESOURCE_GROUP_FILE}")"
+HSM_LOCATION="$(<"${LOCATION_FILE}")"
+
+az cloud set --name AzureCloud
+az login \
+  --service-principal \
+  --username "${AZURE_AUTH_CLIENT_ID}" \
+  --password "${AZURE_AUTH_CLIENT_SECRET}" \
+  --tenant "${AZURE_AUTH_TENANT_ID}" \
+  --output none
+az account set --subscription "${AZURE_AUTH_SUBSCRIPTION_ID}"
+
+ACTIVE_HSM_COUNT="$(az keyvault list \
+  --resource-type hsm \
+  --query "[?name=='${HSM_NAME}'] | length(@)" \
+  -o tsv)"
+DELETED_HSM_COUNT="$(az keyvault list-deleted \
+  --resource-type hsm \
+  --query "[?name=='${HSM_NAME}'] | length(@)" \
+  -o tsv)"
+
+if [[ "${ACTIVE_HSM_COUNT}" == "1" ]]; then
+  echo "Deleting Managed HSM ${HSM_NAME}"
+  az keyvault delete --hsm-name "${HSM_NAME}" --output none
+  retry 20 30 deleted_hsm_exists
+elif [[ "${ACTIVE_HSM_COUNT}" != "0" ]]; then
+  echo "Unexpected active Managed HSM count for ${HSM_NAME}: ${ACTIVE_HSM_COUNT}" >&2
+  exit 1
+elif [[ "${DELETED_HSM_COUNT}" == "1" ]]; then
+  echo "Managed HSM ${HSM_NAME} is already soft-deleted"
+elif [[ "${DELETED_HSM_COUNT}" != "0" ]]; then
+  echo "Unexpected deleted Managed HSM count for ${HSM_NAME}: ${DELETED_HSM_COUNT}" >&2
+  exit 1
+else
+  echo "Managed HSM ${HSM_NAME} was not created; nothing to purge"
+fi
+
+if [[ "${ACTIVE_HSM_COUNT}" == "1" || "${DELETED_HSM_COUNT}" == "1" ]]; then
+  echo "Purging Managed HSM ${HSM_NAME}"
+  az keyvault purge \
+    --hsm-name "${HSM_NAME}" \
+    --location "${HSM_LOCATION}" \
+    --output none
+  retry 20 15 deleted_hsm_absent
+fi
+
+RESOURCE_GROUP_EXISTS="$(az group exists --name "${RESOURCE_GROUP}")"
+if [[ "${RESOURCE_GROUP_EXISTS}" == "true" ]]; then
+  echo "Deleting resource group ${RESOURCE_GROUP}"
+  az group delete --name "${RESOURCE_GROUP}" --yes
+elif [[ "${RESOURCE_GROUP_EXISTS}" != "false" ]]; then
+  echo "Unexpected resource group existence result for ${RESOURCE_GROUP}: ${RESOURCE_GROUP_EXISTS}" >&2
+  exit 1
+fi

ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.metadata.json

@@ -0,0 +1,15 @@
+{
+	"path": "hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.yaml",
+	"owners": {
+		"approvers": [
+			"csrwng",
+			"enxebre",
+			"bryan-cox"
+		],
+		"reviewers": [
+			"csrwng",
+			"enxebre",
+			"bryan-cox"
+		]
+	}
+}

ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.yaml

@@ -0,0 +1,22 @@
+ref:
+  as: hypershift-azure-aks-managed-hsm-deprovision
+  cli: latest
+  commands: hypershift-azure-aks-managed-hsm-deprovision-commands.sh
+  credentials:
+  - mount_path: /etc/hypershift-ci-jobs-azurecreds
+    name: hypershift-ci-jobs-azurecreds
+    namespace: test-credentials
+  env:
+  - default: "false"
+    documentation: Whether an ephemeral Managed HSM was provisioned for the test.
+    name: HYPERSHIFT_AZURE_MANAGED_HSM
+  from: hypershift-tests
+  grace_period: 5m0s
+  resources:
+    requests:
+      cpu: 100m
+      memory: 100Mi
+  timeout: 45m0s
+  documentation: |-
+    Deletes and purges the ephemeral Managed HSM created for the HyperShift Azure
+    KMS test using the AKS workflow credentials, then removes its resource group.

ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/OWNERS

@@ -0,0 +1,9 @@
+approvers:
+- csrwng
+- enxebre
+- bryan-cox
+options: {}
+reviewers:
+- csrwng
+- enxebre
+- bryan-cox