From b2587cb80e4026c59a62a55591b026a1b725e3b3 Mon Sep 17 00:00:00 2001 From: Hilliary Lipsig Date: Wed, 5 Aug 2026 11:52:06 -0700 Subject: [PATCH 01/11] Add HyperShift Managed HSM periodic Provision an ephemeral Azure Managed HSM for the release-5.0 HyperShift v2 encryption test and clean it up after the daily periodic run. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...ift-hypershift-release-5.0__periodics.yaml | 12 ++ ...hift-hypershift-release-5.0-periodics.yaml | 83 +++++++++++ ...zure-create-selfmanaged-guests-commands.sh | 5 + ...ft-azure-e2e-v2-self-managed-workflow.yaml | 4 + .../hypershift/azure/managed-hsm/OWNERS | 11 ++ .../azure/managed-hsm/deprovision/OWNERS | 11 ++ ...-azure-managed-hsm-deprovision-commands.sh | 44 ++++++ ...-managed-hsm-deprovision-ref.metadata.json | 17 +++ ...ift-azure-managed-hsm-deprovision-ref.yaml | 23 +++ .../azure/managed-hsm/provision/OWNERS | 11 ++ ...ft-azure-managed-hsm-provision-commands.sh | 131 ++++++++++++++++++ ...re-managed-hsm-provision-ref.metadata.json | 17 +++ ...shift-azure-managed-hsm-provision-ref.yaml | 29 ++++ ...ft-azure-run-e2e-v2-selfmanaged-chain.yaml | 4 + 14 files changed, 402 insertions(+) create mode 100644 ci-operator/step-registry/hypershift/azure/managed-hsm/OWNERS create mode 100644 ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/OWNERS create mode 100644 ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-commands.sh create mode 100644 ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.metadata.json create mode 100644 ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.yaml create mode 100644 ci-operator/step-registry/hypershift/azure/managed-hsm/provision/OWNERS create mode 100644 ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-commands.sh create mode 100644 ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.metadata.json create mode 100644 ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.yaml diff --git a/ci-operator/config/openshift/hypershift/openshift-hypershift-release-5.0__periodics.yaml b/ci-operator/config/openshift/hypershift/openshift-hypershift-release-5.0__periodics.yaml index d395f51c24e2d..238dbb77b30d5 100644 --- a/ci-operator/config/openshift/hypershift/openshift-hypershift-release-5.0__periodics.yaml +++ b/ci-operator/config/openshift/hypershift/openshift-hypershift-release-5.0__periodics.yaml @@ -340,6 +340,18 @@ tests: ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true" HYPERSHIFT_AZURE_LOCATION: centralus workflow: hypershift-azure-e2e-v2-self-managed +- as: e2e-v2-azure-self-managed-managed-hsm + cron: 0 6 * * * + steps: + cluster_profile: hypershift-azure + dependencies: + OCP_IMAGE_N1: release:n1minor + env: + ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true" + HYPERSHIFT_AZURE_LOCATION: centralus + HYPERSHIFT_AZURE_MANAGED_HSM: "true" + HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION: eastus + workflow: hypershift-azure-e2e-v2-self-managed - as: e2e-azure-kubevirt-ovn cron: 0 4 * * * steps: diff --git a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-5.0-periodics.yaml b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-5.0-periodics.yaml index 8a9c856dae65c..aee1ab1b84843 100644 --- a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-5.0-periodics.yaml +++ b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-5.0-periodics.yaml @@ -3005,6 +3005,89 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build07 + cron: 0 6 * * * + decorate: true + decoration_config: + skip_cloning: true + extra_refs: + - base_ref: release-5.0 + org: openshift + repo: hypershift + labels: + ci-operator.openshift.io/cloud: hypershift-azure + ci-operator.openshift.io/cloud-cluster-profile: hypershift-azure + ci-operator.openshift.io/variant: periodics + ci.openshift.io/generator: prowgen + job-release: "5.0" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-hypershift-release-5.0-periodics-e2e-v2-azure-self-managed-managed-hsm + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=e2e-v2-azure-self-managed-managed-hsm + - --variant=periodics + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build01 cron: 37 9,21 * * * diff --git a/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh b/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh index 0341fb58a9d52..4861d06c56079 100644 --- a/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh +++ b/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh @@ -10,4 +10,9 @@ if [[ -f "${SHARED_DIR}/nodepool_release_images" ]]; then source "${SHARED_DIR}/nodepool_release_images" fi +if [[ -s "${SHARED_DIR}/azure_managed_hsm_key_id" ]]; then + export AZURE_ENCRYPTION_KEY_ID + AZURE_ENCRYPTION_KEY_ID="$(<"${SHARED_DIR}/azure_managed_hsm_key_id")" +fi + /hypershift/bin/create-guests diff --git a/ci-operator/step-registry/hypershift/azure/e2e/v2-self-managed/hypershift-azure-e2e-v2-self-managed-workflow.yaml b/ci-operator/step-registry/hypershift/azure/e2e/v2-self-managed/hypershift-azure-e2e-v2-self-managed-workflow.yaml index bad47d829cea0..eeac3c0dc4d58 100644 --- a/ci-operator/step-registry/hypershift/azure/e2e/v2-self-managed/hypershift-azure-e2e-v2-self-managed-workflow.yaml +++ b/ci-operator/step-registry/hypershift/azure/e2e/v2-self-managed/hypershift-azure-e2e-v2-self-managed-workflow.yaml @@ -25,18 +25,22 @@ workflow: - ref: hypershift-azure-setup-private-link - ref: hypershift-install - ref: hypershift-resolve-nodepool-releases + - ref: hypershift-azure-managed-hsm-provision - chain: hypershift-azure-create-selfmanaged-guests test: - chain: hypershift-azure-run-e2e-v2-selfmanaged post: - chain: hypershift-azure-dump-selfmanaged-guests - chain: hypershift-azure-destroy-selfmanaged-guests + - ref: hypershift-azure-managed-hsm-deprovision - chain: hypershift-destroy-nested-management-cluster env: CLOUD_PROVIDER: "Azure" HYPERSHIFT_PLATFORM: "azure" HYPERSHIFT_NODE_COUNT: "6" HYPERSHIFT_AZURE_LOCATION: "centralus" + HYPERSHIFT_AZURE_MANAGED_HSM: "false" + HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION: "eastus" AZURE_SELF_MANAGED: "true" HYPERSHIFT_EXTERNAL_DNS_DOMAIN: "sm.hcp-sm-azure.azure.devcluster.openshift.com" HYPERSHIFT_ETCD_STORAGE_CLASS: "managed-csi-premium-v2" diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/OWNERS b/ci-operator/step-registry/hypershift/azure/managed-hsm/OWNERS new file mode 100644 index 0000000000000..6c30e7a30d980 --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/managed-hsm/OWNERS @@ -0,0 +1,11 @@ +approvers: +- csrwng +- enxebre +- sjenning +- bryan-cox +options: {} +reviewers: +- csrwng +- enxebre +- sjenning +- bryan-cox diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/OWNERS b/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/OWNERS new file mode 100644 index 0000000000000..6c30e7a30d980 --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/OWNERS @@ -0,0 +1,11 @@ +approvers: +- csrwng +- enxebre +- sjenning +- bryan-cox +options: {} +reviewers: +- csrwng +- enxebre +- sjenning +- bryan-cox diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-commands.sh b/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-commands.sh new file mode 100644 index 0000000000000..526b99fd71367 --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-commands.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -euo pipefail + +if [[ "${HYPERSHIFT_AZURE_MANAGED_HSM}" != "true" ]]; then + echo "Managed HSM deprovisioning is disabled" + exit 0 +fi + +HSM_NAME_FILE="${SHARED_DIR}/azure_managed_hsm_name" +RESOURCE_GROUP_FILE="${SHARED_DIR}/azure_managed_hsm_resource_group" +if [[ ! -s "${HSM_NAME_FILE}" || ! -s "${RESOURCE_GROUP_FILE}" ]]; then + echo "Managed HSM resource information was not written; nothing to delete" + exit 0 +fi + +AZURE_AUTH_LOCATION="/etc/hypershift-ci-jobs-self-managed-azure/credentials.json" +AZURE_AUTH_CLIENT_ID="$(jq -er .clientId "${AZURE_AUTH_LOCATION}")" +AZURE_AUTH_CLIENT_SECRET="$(jq -er .clientSecret "${AZURE_AUTH_LOCATION}")" +AZURE_AUTH_TENANT_ID="$(jq -er .tenantId "${AZURE_AUTH_LOCATION}")" +AZURE_AUTH_SUBSCRIPTION_ID="$(jq -er .subscriptionId "${AZURE_AUTH_LOCATION}")" +HSM_NAME="$(<"${HSM_NAME_FILE}")" +RESOURCE_GROUP="$(<"${RESOURCE_GROUP_FILE}")" + +az cloud set --name AzureCloud +az login \ + --service-principal \ + --username "${AZURE_AUTH_CLIENT_ID}" \ + --password "${AZURE_AUTH_CLIENT_SECRET}" \ + --tenant "${AZURE_AUTH_TENANT_ID}" \ + --output none +az account set --subscription "${AZURE_AUTH_SUBSCRIPTION_ID}" + +if az keyvault show --hsm-name "${HSM_NAME}" --output none 2>/dev/null; then + echo "Deleting Managed HSM ${HSM_NAME}" + az keyvault delete --hsm-name "${HSM_NAME}" --output none +else + echo "Managed HSM ${HSM_NAME} does not exist or is already deleted" +fi + +if az group show --name "${RESOURCE_GROUP}" --output none 2>/dev/null; then + echo "Deleting resource group ${RESOURCE_GROUP}" + az group delete --name "${RESOURCE_GROUP}" --yes --no-wait +fi diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.metadata.json b/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.metadata.json new file mode 100644 index 0000000000000..4e43cd7bd2937 --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.metadata.json @@ -0,0 +1,17 @@ +{ + "path": "hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.yaml", + "owners": { + "approvers": [ + "csrwng", + "enxebre", + "sjenning", + "bryan-cox" + ], + "reviewers": [ + "csrwng", + "enxebre", + "sjenning", + "bryan-cox" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.yaml b/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.yaml new file mode 100644 index 0000000000000..b500ef4dab845 --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.yaml @@ -0,0 +1,23 @@ +ref: + as: hypershift-azure-managed-hsm-deprovision + best_effort: true + cli: latest + commands: hypershift-azure-managed-hsm-deprovision-commands.sh + credentials: + - mount_path: /etc/hypershift-ci-jobs-self-managed-azure + name: hypershift-ci-jobs-self-managed-azure + namespace: test-credentials + env: + - default: "false" + documentation: Whether an ephemeral Managed HSM was provisioned for the test. + name: HYPERSHIFT_AZURE_MANAGED_HSM + from: hypershift-tests + grace_period: 5m0s + resources: + requests: + cpu: 100m + memory: 100Mi + timeout: 15m0s + documentation: |- + Deletes the ephemeral Managed HSM created for the HyperShift Azure KMS test + and removes its resource group. Tenant-level cleanup purges the soft-deleted HSM. diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/OWNERS b/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/OWNERS new file mode 100644 index 0000000000000..6c30e7a30d980 --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/OWNERS @@ -0,0 +1,11 @@ +approvers: +- csrwng +- enxebre +- sjenning +- bryan-cox +options: {} +reviewers: +- csrwng +- enxebre +- sjenning +- bryan-cox diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-commands.sh b/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-commands.sh new file mode 100644 index 0000000000000..3b37bc600bd75 --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-commands.sh @@ -0,0 +1,131 @@ +#!/bin/bash + +set -euo pipefail + +if [[ "${HYPERSHIFT_AZURE_MANAGED_HSM}" != "true" ]]; then + echo "Managed HSM provisioning is disabled" + exit 0 +fi + +retry() { + local attempts="$1" + local delay="$2" + shift 2 + + local attempt + for ((attempt = 1; attempt <= attempts; attempt++)); do + if "$@"; then + return 0 + fi + if ((attempt == attempts)); then + echo "Command failed after ${attempts} attempts: $*" >&2 + return 1 + fi + echo "Attempt ${attempt}/${attempts} failed; retrying in ${delay} seconds" + sleep "${delay}" + done +} + +AZURE_AUTH_LOCATION="/etc/hypershift-ci-jobs-self-managed-azure/credentials.json" +WORKLOAD_IDENTITIES_FILE="/etc/hypershift-ci-jobs-self-managed-azure-e2e/workload-identities.json" + +AZURE_AUTH_CLIENT_ID="$(jq -er .clientId "${AZURE_AUTH_LOCATION}")" +AZURE_AUTH_CLIENT_SECRET="$(jq -er .clientSecret "${AZURE_AUTH_LOCATION}")" +AZURE_AUTH_TENANT_ID="$(jq -er .tenantId "${AZURE_AUTH_LOCATION}")" +AZURE_AUTH_SUBSCRIPTION_ID="$(jq -er .subscriptionId "${AZURE_AUTH_LOCATION}")" +KMS_CLIENT_ID="$(jq -er .kmsClientID "${WORKLOAD_IDENTITIES_FILE}")" + +az cloud set --name AzureCloud +az login \ + --service-principal \ + --username "${AZURE_AUTH_CLIENT_ID}" \ + --password "${AZURE_AUTH_CLIENT_SECRET}" \ + --tenant "${AZURE_AUTH_TENANT_ID}" \ + --output none +az account set --subscription "${AZURE_AUTH_SUBSCRIPTION_ID}" + +ADMIN_OBJECT_ID="$(az ad sp show --id "${AZURE_AUTH_CLIENT_ID}" --query id -o tsv)" +KMS_OBJECT_ID="$(az ad sp show --id "${KMS_CLIENT_ID}" --query id -o tsv)" + +UNIQUE_SUFFIX="$(printf '%s' "${PROW_JOB_ID}" | sha256sum | cut -c1-16)" +HSM_NAME="hshift-${UNIQUE_SUFFIX}" +RESOURCE_GROUP="${HSM_NAME}-rg" +KEY_NAME="etcd-encryption" + +# Write cleanup inputs before provisioning so post steps can remove partial resources. +printf '%s\n' "${HSM_NAME}" > "${SHARED_DIR}/azure_managed_hsm_name" +printf '%s\n' "${RESOURCE_GROUP}" > "${SHARED_DIR}/azure_managed_hsm_resource_group" + +az group create \ + --name "${RESOURCE_GROUP}" \ + --location "${HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION}" \ + --tags "prow-job-id=${PROW_JOB_ID}" \ + --output none + +az keyvault create \ + --hsm-name "${HSM_NAME}" \ + --resource-group "${RESOURCE_GROUP}" \ + --location "${HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION}" \ + --administrators "${ADMIN_OBJECT_ID}" \ + --retention-days 7 \ + --output none + +SECURITY_DOMAIN_DIR="$(mktemp -d)" +trap 'rm -rf "${SECURITY_DOMAIN_DIR}"' EXIT + +security_domain_args=() +for index in 0 1 2; do + openssl req \ + -newkey rsa:2048 \ + -nodes \ + -keyout "${SECURITY_DOMAIN_DIR}/key-${index}.pem" \ + -x509 \ + -days 1 \ + -out "${SECURITY_DOMAIN_DIR}/cert-${index}.pem" \ + -subj "/CN=HyperShiftCI${index}" \ + 2>/dev/null + security_domain_args+=(--sd-wrapping-keys "${SECURITY_DOMAIN_DIR}/cert-${index}.pem") +done + +az keyvault security-domain download \ + --hsm-name "${HSM_NAME}" \ + --security-domain-file "${SECURITY_DOMAIN_DIR}/security-domain.json" \ + "${security_domain_args[@]}" \ + --sd-quorum 2 \ + --output none + +retry 10 30 az keyvault role assignment create \ + --hsm-name "${HSM_NAME}" \ + --assignee "${ADMIN_OBJECT_ID}" \ + --role "Managed HSM Crypto Officer" \ + --scope "/" \ + --output none + +retry 10 30 az keyvault role assignment create \ + --hsm-name "${HSM_NAME}" \ + --assignee "${ADMIN_OBJECT_ID}" \ + --role "Managed HSM Crypto User" \ + --scope "/keys" \ + --output none + +retry 12 30 az keyvault key create \ + --hsm-name "${HSM_NAME}" \ + --name "${KEY_NAME}" \ + --kty RSA-HSM \ + --output none + +KEY_ID="$(az keyvault key show \ + --hsm-name "${HSM_NAME}" \ + --name "${KEY_NAME}" \ + --query key.kid \ + -o tsv)" + +retry 10 30 az keyvault role assignment create \ + --hsm-name "${HSM_NAME}" \ + --assignee "${KMS_OBJECT_ID}" \ + --role "Managed HSM Crypto User" \ + --scope "/keys/${KEY_NAME}" \ + --output none + +printf '%s\n' "${KEY_ID}" > "${SHARED_DIR}/azure_managed_hsm_key_id" +echo "Managed HSM key is ready for the HyperShift KMS test" diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.metadata.json b/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.metadata.json new file mode 100644 index 0000000000000..875f285e83425 --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.metadata.json @@ -0,0 +1,17 @@ +{ + "path": "hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.yaml", + "owners": { + "approvers": [ + "csrwng", + "enxebre", + "sjenning", + "bryan-cox" + ], + "reviewers": [ + "csrwng", + "enxebre", + "sjenning", + "bryan-cox" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.yaml b/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.yaml new file mode 100644 index 0000000000000..29b7152d8accd --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.yaml @@ -0,0 +1,29 @@ +ref: + as: hypershift-azure-managed-hsm-provision + cli: latest + commands: hypershift-azure-managed-hsm-provision-commands.sh + credentials: + - mount_path: /etc/hypershift-ci-jobs-self-managed-azure + name: hypershift-ci-jobs-self-managed-azure + namespace: test-credentials + - mount_path: /etc/hypershift-ci-jobs-self-managed-azure-e2e + name: hypershift-ci-jobs-self-managed-azure-e2e + namespace: test-credentials + env: + - default: "false" + documentation: Whether to provision an ephemeral Managed HSM for Azure KMS testing. + name: HYPERSHIFT_AZURE_MANAGED_HSM + - default: eastus + documentation: Azure location in which to provision the Managed HSM. + name: HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION + from: hypershift-tests + grace_period: 5m0s + resources: + requests: + cpu: 100m + memory: 100Mi + timeout: 45m0s + documentation: |- + Provisions and activates an ephemeral Azure Managed HSM, creates an RSA-HSM + key, grants the self-managed HyperShift KMS workload identity access to it, + and writes the key ID to SHARED_DIR. diff --git a/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml b/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml index 65a380fea1401..cb27acd784885 100644 --- a/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml +++ b/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml @@ -10,6 +10,10 @@ chain: commands: |- set -xuo pipefail export KUBECONFIG="${SHARED_DIR}/management_cluster_kubeconfig" + if [[ -s "${SHARED_DIR}/azure_managed_hsm_key_id" ]]; then + export AZURE_ENCRYPTION_KEY_ID + AZURE_ENCRYPTION_KEY_ID="$(<"${SHARED_DIR}/azure_managed_hsm_key_id")" + fi /hypershift/bin/run-tests timeout: 150m grace_period: 5m From 1235c6bc4fc7cba241b7e7d1d5c7c2862c2d3288 Mon Sep 17 00:00:00 2001 From: Hilliary Lipsig Date: Wed, 5 Aug 2026 12:33:22 -0700 Subject: [PATCH 02/11] Purge Managed HSM after periodic Wait for Azure to expose the soft-deleted HSM, purge it to stop billing, and retain partial-provision cleanup behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...-azure-managed-hsm-deprovision-commands.sh | 43 ++++++++++++++++++- ...ift-azure-managed-hsm-deprovision-ref.yaml | 7 ++- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-commands.sh b/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-commands.sh index 526b99fd71367..29224f38a2f87 100644 --- a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-commands.sh +++ b/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-commands.sh @@ -2,6 +2,34 @@ set -euo pipefail +retry() { + local attempts="$1" + local delay="$2" + shift 2 + + local attempt + for ((attempt = 1; attempt <= attempts; attempt++)); do + if "$@"; then + return 0 + fi + if ((attempt == attempts)); then + echo "Command failed after ${attempts} attempts: $*" >&2 + return 1 + fi + echo "Attempt ${attempt}/${attempts} failed; retrying in ${delay} seconds" + sleep "${delay}" + done +} + +deleted_hsm_exists() { + local count + count="$(az keyvault list-deleted \ + --resource-type hsm \ + --query "[?name=='${HSM_NAME}'] | length(@)" \ + -o tsv)" + [[ "${count}" == "1" ]] +} + if [[ "${HYPERSHIFT_AZURE_MANAGED_HSM}" != "true" ]]; then echo "Managed HSM deprovisioning is disabled" exit 0 @@ -31,11 +59,24 @@ az login \ --output none az account set --subscription "${AZURE_AUTH_SUBSCRIPTION_ID}" +PURGE_HSM=true if az keyvault show --hsm-name "${HSM_NAME}" --output none 2>/dev/null; then echo "Deleting Managed HSM ${HSM_NAME}" az keyvault delete --hsm-name "${HSM_NAME}" --output none + retry 20 30 deleted_hsm_exists +elif deleted_hsm_exists; then + echo "Managed HSM ${HSM_NAME} is already soft-deleted" else - echo "Managed HSM ${HSM_NAME} does not exist or is already deleted" + echo "Managed HSM ${HSM_NAME} was not created; nothing to purge" + PURGE_HSM=false +fi + +if [[ "${PURGE_HSM}" == "true" ]]; then + echo "Purging Managed HSM ${HSM_NAME}" + az keyvault purge \ + --hsm-name "${HSM_NAME}" \ + --location "${HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION}" \ + --output none fi if az group show --name "${RESOURCE_GROUP}" --output none 2>/dev/null; then diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.yaml b/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.yaml index b500ef4dab845..48f161e386a3f 100644 --- a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.yaml +++ b/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.yaml @@ -11,6 +11,9 @@ ref: - default: "false" documentation: Whether an ephemeral Managed HSM was provisioned for the test. name: HYPERSHIFT_AZURE_MANAGED_HSM + - default: eastus + documentation: Azure location containing the Managed HSM to purge. + name: HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION from: hypershift-tests grace_period: 5m0s resources: @@ -19,5 +22,5 @@ ref: memory: 100Mi timeout: 15m0s documentation: |- - Deletes the ephemeral Managed HSM created for the HyperShift Azure KMS test - and removes its resource group. Tenant-level cleanup purges the soft-deleted HSM. + Deletes and purges the ephemeral Managed HSM created for the HyperShift Azure + KMS test, then removes its resource group. From 6058ce42f0f7f9fedcd92e51c173bd1010ccc197 Mon Sep 17 00:00:00 2001 From: Hilliary Lipsig Date: Wed, 5 Aug 2026 13:22:20 -0700 Subject: [PATCH 03/11] Protect Managed HSM key ID from logs Disable shell tracing while reading and exporting the Managed HSM key URI in guest creation and v2 test execution. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../hypershift-azure-create-selfmanaged-guests-commands.sh | 2 ++ .../hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh b/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh index 4861d06c56079..ff2de1e48ddcb 100644 --- a/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh +++ b/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh @@ -11,8 +11,10 @@ if [[ -f "${SHARED_DIR}/nodepool_release_images" ]]; then fi if [[ -s "${SHARED_DIR}/azure_managed_hsm_key_id" ]]; then + set +x export AZURE_ENCRYPTION_KEY_ID AZURE_ENCRYPTION_KEY_ID="$(<"${SHARED_DIR}/azure_managed_hsm_key_id")" + set -x fi /hypershift/bin/create-guests diff --git a/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml b/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml index cb27acd784885..9700a9038a417 100644 --- a/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml +++ b/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml @@ -11,8 +11,10 @@ chain: set -xuo pipefail export KUBECONFIG="${SHARED_DIR}/management_cluster_kubeconfig" if [[ -s "${SHARED_DIR}/azure_managed_hsm_key_id" ]]; then + set +x export AZURE_ENCRYPTION_KEY_ID AZURE_ENCRYPTION_KEY_ID="$(<"${SHARED_DIR}/azure_managed_hsm_key_id")" + set -x fi /hypershift/bin/run-tests timeout: 150m From c8a9f8f09ceb0e7e621742b732833797f3160fec Mon Sep 17 00:00:00 2001 From: Hilliary Lipsig Date: Wed, 5 Aug 2026 15:03:31 -0700 Subject: [PATCH 04/11] Use cluster profile for Managed HSM Authenticate HSM provisioning and cleanup with the hypershift-azure cluster profile while retaining the existing KMS workload identity input. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../hypershift-azure-managed-hsm-deprovision-commands.sh | 2 +- .../hypershift-azure-managed-hsm-deprovision-ref.yaml | 6 +----- .../hypershift-azure-managed-hsm-provision-commands.sh | 2 +- .../hypershift-azure-managed-hsm-provision-ref.yaml | 7 ++----- 4 files changed, 5 insertions(+), 12 deletions(-) diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-commands.sh b/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-commands.sh index 29224f38a2f87..1a61061bc2903 100644 --- a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-commands.sh +++ b/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-commands.sh @@ -42,7 +42,7 @@ if [[ ! -s "${HSM_NAME_FILE}" || ! -s "${RESOURCE_GROUP_FILE}" ]]; then exit 0 fi -AZURE_AUTH_LOCATION="/etc/hypershift-ci-jobs-self-managed-azure/credentials.json" +AZURE_AUTH_LOCATION="${CLUSTER_PROFILE_DIR}/osServicePrincipal.json" AZURE_AUTH_CLIENT_ID="$(jq -er .clientId "${AZURE_AUTH_LOCATION}")" AZURE_AUTH_CLIENT_SECRET="$(jq -er .clientSecret "${AZURE_AUTH_LOCATION}")" AZURE_AUTH_TENANT_ID="$(jq -er .tenantId "${AZURE_AUTH_LOCATION}")" diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.yaml b/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.yaml index 48f161e386a3f..56282f409ab80 100644 --- a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.yaml +++ b/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.yaml @@ -3,10 +3,6 @@ ref: best_effort: true cli: latest commands: hypershift-azure-managed-hsm-deprovision-commands.sh - credentials: - - mount_path: /etc/hypershift-ci-jobs-self-managed-azure - name: hypershift-ci-jobs-self-managed-azure - namespace: test-credentials env: - default: "false" documentation: Whether an ephemeral Managed HSM was provisioned for the test. @@ -23,4 +19,4 @@ ref: timeout: 15m0s documentation: |- Deletes and purges the ephemeral Managed HSM created for the HyperShift Azure - KMS test, then removes its resource group. + KMS test using the cluster profile credentials, then removes its resource group. diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-commands.sh b/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-commands.sh index 3b37bc600bd75..1fdd0f3611f5e 100644 --- a/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-commands.sh +++ b/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-commands.sh @@ -26,7 +26,7 @@ retry() { done } -AZURE_AUTH_LOCATION="/etc/hypershift-ci-jobs-self-managed-azure/credentials.json" +AZURE_AUTH_LOCATION="${CLUSTER_PROFILE_DIR}/osServicePrincipal.json" WORKLOAD_IDENTITIES_FILE="/etc/hypershift-ci-jobs-self-managed-azure-e2e/workload-identities.json" AZURE_AUTH_CLIENT_ID="$(jq -er .clientId "${AZURE_AUTH_LOCATION}")" diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.yaml b/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.yaml index 29b7152d8accd..47b0e76253241 100644 --- a/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.yaml +++ b/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.yaml @@ -3,9 +3,6 @@ ref: cli: latest commands: hypershift-azure-managed-hsm-provision-commands.sh credentials: - - mount_path: /etc/hypershift-ci-jobs-self-managed-azure - name: hypershift-ci-jobs-self-managed-azure - namespace: test-credentials - mount_path: /etc/hypershift-ci-jobs-self-managed-azure-e2e name: hypershift-ci-jobs-self-managed-azure-e2e namespace: test-credentials @@ -25,5 +22,5 @@ ref: timeout: 45m0s documentation: |- Provisions and activates an ephemeral Azure Managed HSM, creates an RSA-HSM - key, grants the self-managed HyperShift KMS workload identity access to it, - and writes the key ID to SHARED_DIR. + key using the cluster profile credentials, grants the self-managed HyperShift + KMS workload identity access to it, and writes the key ID to SHARED_DIR. From d102a2149699acd9b4f5549e4fcc58daf9619af5 Mon Sep 17 00:00:00 2001 From: Hilliary Lipsig Date: Sun, 9 Aug 2026 16:51:50 -0700 Subject: [PATCH 05/11] Add Azure IPI resource garbage collection periodic Add periodic-ipi-deprovision-azure to clean up leaked Azure IPI cluster resources, matching the existing AWS and GCP GC patterns. Uses a 6-hour TTL to avoid disrupting active tests. Co-Authored-By: Claude Opus 4.6 --- .../jobs/infra-build-farm-periodics.yaml | 55 +++++++ .../ipi-deprovision/ipi-deprovision-bc.yaml | 4 +- core-services/ipi-deprovision/azure.sh | 137 ++++++++++++++++++ 3 files changed, 195 insertions(+), 1 deletion(-) create mode 100755 core-services/ipi-deprovision/azure.sh diff --git a/ci-operator/jobs/infra-build-farm-periodics.yaml b/ci-operator/jobs/infra-build-farm-periodics.yaml index 2dd1f4025e4c1..d35539ef74789 100644 --- a/ci-operator/jobs/infra-build-farm-periodics.yaml +++ b/ci-operator/jobs/infra-build-farm-periodics.yaml @@ -440,6 +440,61 @@ periodics: - key: gce.json path: gce.json secretName: cluster-secrets-gcp-openshift-gce-devel-ci-2 +- agent: kubernetes + cluster: build02 + decorate: true + decoration_config: + timeout: 8h0m0s + extra_refs: + - base_ref: main + org: openshift + repo: release + workdir: true + interval: 4h + labels: + ci.openshift.io/role: cloud-azure + pj-rehearse.openshift.io/can-be-rehearsed: "true" + max_concurrency: 1 + name: periodic-ipi-deprovision-azure + reporter_config: + slack: + channel: '#ops-testplatform' + job_states_to_report: + - failure + - error + report_template: '@azure-ci-admin Job *{{.Spec.Job}}* failed. <{{.Status.URL}}|View + logs>' + spec: + containers: + - command: + - ./core-services/ipi-deprovision/azure.sh + env: + - name: HOME + value: /tmp + - name: AZURE_AUTH_LOCATION + value: /azure/osServicePrincipal.json + - name: CLUSTER_TTL + value: 6 hours ago + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ipi-deprovision_latest + imagePullPolicy: Always + name: ipi-deprovision + resources: + requests: + cpu: "1" + memory: 600Mi + volumeMounts: + - mountPath: /azure + name: cluster-secrets-azure4 + nodeSelector: + beta.kubernetes.io/arch: amd64 + serviceAccountName: ipi-deprovisioner + volumes: + - name: cluster-secrets-azure4 + secret: + items: + - key: osServicePrincipal.json + path: osServicePrincipal.json + secretName: cluster-secrets-azure4 - agent: kubernetes cluster: build02 cron: '@daily' diff --git a/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml b/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml index 7a3bc8aebd254..8bb2297a73c81 100644 --- a/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml +++ b/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml @@ -47,7 +47,9 @@ spec: COPY ./openshift-install /usr/bin/openshift-install COPY ./hypershift /usr/bin/hypershift - RUN dnf install --nogpg -y google-cloud-sdk unzip jq less && \ + RUN rpm --import https://packages.microsoft.com/keys/microsoft.asc && \ + echo -e "[azure-cli]\nname=Azure CLI\nbaseurl=https://packages.microsoft.com/yumrepos/azure-cli\nenabled=1\ngpgcheck=1\ngpgkey=https://packages.microsoft.com/keys/microsoft.asc" > /etc/yum.repos.d/azure-cli.repo && \ + dnf install --nogpg -y google-cloud-sdk azure-cli unzip jq less && \ dnf clean all && \ curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ unzip awscliv2.zip && ./aws/install diff --git a/core-services/ipi-deprovision/azure.sh b/core-services/ipi-deprovision/azure.sh new file mode 100755 index 0000000000000..9b02486e0131a --- /dev/null +++ b/core-services/ipi-deprovision/azure.sh @@ -0,0 +1,137 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wait; fi' TERM + +function queue() { + local LIVE="$(jobs | wc -l)" + while [[ "${LIVE}" -ge 10 ]]; do + sleep 1 + LIVE="$(jobs | wc -l)" + done + echo "${@}" + "${@}" & +} + +function deprovision() { + WORKDIR="${1}" + timeout --signal=SIGTERM 30m openshift-install --dir "${WORKDIR}" --log-level error destroy cluster && touch "${WORKDIR}/success" || touch "${WORKDIR}/failure" +} + +logdir="${ARTIFACTS}/deprovision" +mkdir -p "${logdir}" + +AZURE_AUTH_LOCATION="${AZURE_AUTH_LOCATION:-/azure/osServicePrincipal.json}" +export AZURE_AUTH_LOCATION + +# Disable tracing due to credential handling +[[ $- == *x* ]] && WAS_TRACING=true || WAS_TRACING=false +set +x +AZURE_CLIENT_ID="$(jq -r .clientId "${AZURE_AUTH_LOCATION}")" +AZURE_CLIENT_SECRET="$(jq -r .clientSecret "${AZURE_AUTH_LOCATION}")" +AZURE_TENANT_ID="$(jq -r .tenantId "${AZURE_AUTH_LOCATION}")" +AZURE_SUBSCRIPTION_ID="$(jq -r .subscriptionId "${AZURE_AUTH_LOCATION}")" + +az login --service-principal \ + -u "${AZURE_CLIENT_ID}" \ + -p "${AZURE_CLIENT_SECRET}" \ + --tenant "${AZURE_TENANT_ID}" \ + --output none +$WAS_TRACING && set -x + +az account set --subscription "${AZURE_SUBSCRIPTION_ID}" +echo "Azure subscription: ${AZURE_SUBSCRIPTION_ID}" + +azure_rg_age_cutoff="$(date -u --date="${CLUSTER_TTL}" '+%Y-%m-%dT%H:%M:%SZ')" +echo "deprovisioning clusters with resource groups created before ${azure_rg_age_cutoff} ..." + +# List resource groups with creation time via ARM REST API ($expand=createdTime). +rg_json="[]" +next_url="/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourcegroups?api-version=2021-04-01&\$expand=createdTime" +while [[ -n "${next_url}" ]]; do + page="$(az rest --method get --url "${next_url}")" + rg_json="$(jq -s '.[0] + [.[1].value[] | {name: .name, location: .location, created: .createdTime, state: .properties.provisioningState}]' \ + <(echo "${rg_json}") <(echo "${page}"))" + next_url="$(echo "${page}" | jq -r '.nextLink // empty')" +done + +while IFS=$'\t' read -r rg_name rg_location rg_created rg_state; do + [[ -z "${rg_name}" ]] && continue + [[ "${rg_state}" == "Deleting" ]] && continue + + # Match CI-created resource groups: the IPI installer creates resource groups + # named -rg where the infraID starts with "ci-op-" (the CI namespace). + if [[ ! "${rg_name}" =~ ^ci-op- ]]; then + continue + fi + + if [[ "${rg_created}" > "${azure_rg_age_cutoff}" ]]; then + continue + fi + + # Derive the infraID from the resource group name by stripping the -rg suffix. + # If the RG doesn't end in -rg, use the full name as infraID. + if [[ "${rg_name}" =~ ^(.+)-rg$ ]]; then + infraID="${BASH_REMATCH[1]}" + else + infraID="${rg_name}" + fi + + workdir="${logdir}/${infraID}" + mkdir -p "${workdir}" + cat <"${workdir}/metadata.json" +{ + "infraID":"${infraID}", + "azure":{ + "region":"${rg_location}", + "resourceGroupName":"${rg_name}", + "cloudName":"AzurePublicCloud" + } +} +EOF + echo "will deprovision Azure cluster ${infraID} in ${rg_location} (rg: ${rg_name}, created: ${rg_created})" +done < <(echo "${rg_json}" | jq -r '.[] | [.name, .location, .created, .state] | @tsv') + +# log installer version for debugging purposes +openshift-install version + +clusters=$( find "${logdir}" -mindepth 1 -type d ) +for workdir in $(shuf <<< ${clusters}); do + queue deprovision "${workdir}" +done + +if ! wait; then + echo "At least one deprovision job failed or timed out." +fi + +# Force-delete resource groups that openshift-install failed to clean up +for workdir in $(find "${logdir}" -mindepth 1 -type d); do + if [[ -f "${workdir}/failure" ]]; then + rg_name="$(jq -r '.azure.resourceGroupName' "${workdir}/metadata.json")" + echo "openshift-install failed for ${rg_name}, force-deleting resource group ..." + if az group delete --name "${rg_name}" --yes --no-wait; then + echo "Initiated force-deletion of resource group ${rg_name}" + rm "${workdir}/failure" + touch "${workdir}/warning" + else + echo "Failed to force-delete resource group ${rg_name}" + fi + fi +done + +WARNINGS="$(find ${clusters} -name warning -printf '%H\n' | sort)" +if [[ -n "${WARNINGS}" ]]; then + echo "The following clusters required force-deletion of their resource groups:" + xargs --max-args 1 basename <<< $WARNINGS +fi + +FAILED="$(find ${clusters} -name failure -printf '%H\n' | sort)" +if [[ -n "${FAILED}" ]]; then + echo "Deprovision failed on the following clusters:" + xargs --max-args 1 basename <<< $FAILED + exit 1 +fi + +echo "Deprovision finished successfully" From f85a9b3846f3402011c72526d94a68fbd8032816 Mon Sep 17 00:00:00 2001 From: Hilliary Lipsig Date: Mon, 10 Aug 2026 10:27:39 -0700 Subject: [PATCH 06/11] Revert "Add Azure IPI resource garbage collection periodic" This reverts commit d102a2149699acd9b4f5549e4fcc58daf9619af5. --- .../jobs/infra-build-farm-periodics.yaml | 55 ------- .../ipi-deprovision/ipi-deprovision-bc.yaml | 4 +- core-services/ipi-deprovision/azure.sh | 137 ------------------ 3 files changed, 1 insertion(+), 195 deletions(-) delete mode 100755 core-services/ipi-deprovision/azure.sh diff --git a/ci-operator/jobs/infra-build-farm-periodics.yaml b/ci-operator/jobs/infra-build-farm-periodics.yaml index d35539ef74789..2dd1f4025e4c1 100644 --- a/ci-operator/jobs/infra-build-farm-periodics.yaml +++ b/ci-operator/jobs/infra-build-farm-periodics.yaml @@ -440,61 +440,6 @@ periodics: - key: gce.json path: gce.json secretName: cluster-secrets-gcp-openshift-gce-devel-ci-2 -- agent: kubernetes - cluster: build02 - decorate: true - decoration_config: - timeout: 8h0m0s - extra_refs: - - base_ref: main - org: openshift - repo: release - workdir: true - interval: 4h - labels: - ci.openshift.io/role: cloud-azure - pj-rehearse.openshift.io/can-be-rehearsed: "true" - max_concurrency: 1 - name: periodic-ipi-deprovision-azure - reporter_config: - slack: - channel: '#ops-testplatform' - job_states_to_report: - - failure - - error - report_template: '@azure-ci-admin Job *{{.Spec.Job}}* failed. <{{.Status.URL}}|View - logs>' - spec: - containers: - - command: - - ./core-services/ipi-deprovision/azure.sh - env: - - name: HOME - value: /tmp - - name: AZURE_AUTH_LOCATION - value: /azure/osServicePrincipal.json - - name: CLUSTER_TTL - value: 6 hours ago - image: quay-proxy.ci.openshift.org/openshift/ci:ci_ipi-deprovision_latest - imagePullPolicy: Always - name: ipi-deprovision - resources: - requests: - cpu: "1" - memory: 600Mi - volumeMounts: - - mountPath: /azure - name: cluster-secrets-azure4 - nodeSelector: - beta.kubernetes.io/arch: amd64 - serviceAccountName: ipi-deprovisioner - volumes: - - name: cluster-secrets-azure4 - secret: - items: - - key: osServicePrincipal.json - path: osServicePrincipal.json - secretName: cluster-secrets-azure4 - agent: kubernetes cluster: build02 cron: '@daily' diff --git a/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml b/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml index 8bb2297a73c81..7a3bc8aebd254 100644 --- a/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml +++ b/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml @@ -47,9 +47,7 @@ spec: COPY ./openshift-install /usr/bin/openshift-install COPY ./hypershift /usr/bin/hypershift - RUN rpm --import https://packages.microsoft.com/keys/microsoft.asc && \ - echo -e "[azure-cli]\nname=Azure CLI\nbaseurl=https://packages.microsoft.com/yumrepos/azure-cli\nenabled=1\ngpgcheck=1\ngpgkey=https://packages.microsoft.com/keys/microsoft.asc" > /etc/yum.repos.d/azure-cli.repo && \ - dnf install --nogpg -y google-cloud-sdk azure-cli unzip jq less && \ + RUN dnf install --nogpg -y google-cloud-sdk unzip jq less && \ dnf clean all && \ curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ unzip awscliv2.zip && ./aws/install diff --git a/core-services/ipi-deprovision/azure.sh b/core-services/ipi-deprovision/azure.sh deleted file mode 100755 index 9b02486e0131a..0000000000000 --- a/core-services/ipi-deprovision/azure.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/bash -set -o errexit -set -o nounset -set -o pipefail - -trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wait; fi' TERM - -function queue() { - local LIVE="$(jobs | wc -l)" - while [[ "${LIVE}" -ge 10 ]]; do - sleep 1 - LIVE="$(jobs | wc -l)" - done - echo "${@}" - "${@}" & -} - -function deprovision() { - WORKDIR="${1}" - timeout --signal=SIGTERM 30m openshift-install --dir "${WORKDIR}" --log-level error destroy cluster && touch "${WORKDIR}/success" || touch "${WORKDIR}/failure" -} - -logdir="${ARTIFACTS}/deprovision" -mkdir -p "${logdir}" - -AZURE_AUTH_LOCATION="${AZURE_AUTH_LOCATION:-/azure/osServicePrincipal.json}" -export AZURE_AUTH_LOCATION - -# Disable tracing due to credential handling -[[ $- == *x* ]] && WAS_TRACING=true || WAS_TRACING=false -set +x -AZURE_CLIENT_ID="$(jq -r .clientId "${AZURE_AUTH_LOCATION}")" -AZURE_CLIENT_SECRET="$(jq -r .clientSecret "${AZURE_AUTH_LOCATION}")" -AZURE_TENANT_ID="$(jq -r .tenantId "${AZURE_AUTH_LOCATION}")" -AZURE_SUBSCRIPTION_ID="$(jq -r .subscriptionId "${AZURE_AUTH_LOCATION}")" - -az login --service-principal \ - -u "${AZURE_CLIENT_ID}" \ - -p "${AZURE_CLIENT_SECRET}" \ - --tenant "${AZURE_TENANT_ID}" \ - --output none -$WAS_TRACING && set -x - -az account set --subscription "${AZURE_SUBSCRIPTION_ID}" -echo "Azure subscription: ${AZURE_SUBSCRIPTION_ID}" - -azure_rg_age_cutoff="$(date -u --date="${CLUSTER_TTL}" '+%Y-%m-%dT%H:%M:%SZ')" -echo "deprovisioning clusters with resource groups created before ${azure_rg_age_cutoff} ..." - -# List resource groups with creation time via ARM REST API ($expand=createdTime). -rg_json="[]" -next_url="/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourcegroups?api-version=2021-04-01&\$expand=createdTime" -while [[ -n "${next_url}" ]]; do - page="$(az rest --method get --url "${next_url}")" - rg_json="$(jq -s '.[0] + [.[1].value[] | {name: .name, location: .location, created: .createdTime, state: .properties.provisioningState}]' \ - <(echo "${rg_json}") <(echo "${page}"))" - next_url="$(echo "${page}" | jq -r '.nextLink // empty')" -done - -while IFS=$'\t' read -r rg_name rg_location rg_created rg_state; do - [[ -z "${rg_name}" ]] && continue - [[ "${rg_state}" == "Deleting" ]] && continue - - # Match CI-created resource groups: the IPI installer creates resource groups - # named -rg where the infraID starts with "ci-op-" (the CI namespace). - if [[ ! "${rg_name}" =~ ^ci-op- ]]; then - continue - fi - - if [[ "${rg_created}" > "${azure_rg_age_cutoff}" ]]; then - continue - fi - - # Derive the infraID from the resource group name by stripping the -rg suffix. - # If the RG doesn't end in -rg, use the full name as infraID. - if [[ "${rg_name}" =~ ^(.+)-rg$ ]]; then - infraID="${BASH_REMATCH[1]}" - else - infraID="${rg_name}" - fi - - workdir="${logdir}/${infraID}" - mkdir -p "${workdir}" - cat <"${workdir}/metadata.json" -{ - "infraID":"${infraID}", - "azure":{ - "region":"${rg_location}", - "resourceGroupName":"${rg_name}", - "cloudName":"AzurePublicCloud" - } -} -EOF - echo "will deprovision Azure cluster ${infraID} in ${rg_location} (rg: ${rg_name}, created: ${rg_created})" -done < <(echo "${rg_json}" | jq -r '.[] | [.name, .location, .created, .state] | @tsv') - -# log installer version for debugging purposes -openshift-install version - -clusters=$( find "${logdir}" -mindepth 1 -type d ) -for workdir in $(shuf <<< ${clusters}); do - queue deprovision "${workdir}" -done - -if ! wait; then - echo "At least one deprovision job failed or timed out." -fi - -# Force-delete resource groups that openshift-install failed to clean up -for workdir in $(find "${logdir}" -mindepth 1 -type d); do - if [[ -f "${workdir}/failure" ]]; then - rg_name="$(jq -r '.azure.resourceGroupName' "${workdir}/metadata.json")" - echo "openshift-install failed for ${rg_name}, force-deleting resource group ..." - if az group delete --name "${rg_name}" --yes --no-wait; then - echo "Initiated force-deletion of resource group ${rg_name}" - rm "${workdir}/failure" - touch "${workdir}/warning" - else - echo "Failed to force-delete resource group ${rg_name}" - fi - fi -done - -WARNINGS="$(find ${clusters} -name warning -printf '%H\n' | sort)" -if [[ -n "${WARNINGS}" ]]; then - echo "The following clusters required force-deletion of their resource groups:" - xargs --max-args 1 basename <<< $WARNINGS -fi - -FAILED="$(find ${clusters} -name failure -printf '%H\n' | sort)" -if [[ -n "${FAILED}" ]]; then - echo "Deprovision failed on the following clusters:" - xargs --max-args 1 basename <<< $FAILED - exit 1 -fi - -echo "Deprovision finished successfully" From d8e20482b70b7502640a18056bde9239efed21ed Mon Sep 17 00:00:00 2001 From: Hilliary Lipsig Date: Tue, 11 Aug 2026 15:57:31 -0700 Subject: [PATCH 07/11] ci(hypershift): run Managed HSM coverage on AKS Move Managed HSM provisioning into the AKS workflow and execute TestCreateClusterCustomConfig against the ephemeral RSA-HSM key. Remove the self-managed v2 wiring and retain fail-loud delete and purge cleanup. Signed-off-by: Hilliary Lipsig Commit-Message-Assisted-by: Claude (via Claude Code) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...ift-hypershift-release-5.0__periodics.yaml | 16 +- ...hift-hypershift-release-5.0-periodics.yaml | 166 +++++++++--------- .../hypershift-azure-aks-e2e-workflow.yaml | 6 +- .../managed-hsm/deprovision}/OWNERS | 0 ...e-aks-managed-hsm-deprovision-commands.sh} | 8 +- ...managed-hsm-deprovision-ref.metadata.json} | 2 +- ...zure-aks-managed-hsm-deprovision-ref.yaml} | 14 +- .../managed-hsm/provision}/OWNERS | 0 ...ure-aks-managed-hsm-provision-commands.sh} | 33 +++- ...s-managed-hsm-provision-ref.metadata.json} | 2 +- ...-azure-aks-managed-hsm-provision-ref.yaml} | 12 +- ...zure-create-selfmanaged-guests-commands.sh | 7 - ...ft-azure-e2e-v2-self-managed-workflow.yaml | 4 - .../azure/managed-hsm/provision/OWNERS | 11 -- ...ft-azure-run-e2e-v2-selfmanaged-chain.yaml | 6 - .../hypershift-azure-run-e2e-commands.sh | 18 +- 16 files changed, 160 insertions(+), 145 deletions(-) rename ci-operator/step-registry/hypershift/azure/{managed-hsm => aks/managed-hsm/deprovision}/OWNERS (100%) rename ci-operator/step-registry/hypershift/azure/{managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-commands.sh => aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-commands.sh} (88%) rename ci-operator/step-registry/hypershift/azure/{managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.metadata.json => aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.metadata.json} (61%) rename ci-operator/step-registry/hypershift/azure/{managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.yaml => aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.yaml} (52%) rename ci-operator/step-registry/hypershift/azure/{managed-hsm/deprovision => aks/managed-hsm/provision}/OWNERS (100%) rename ci-operator/step-registry/hypershift/azure/{managed-hsm/provision/hypershift-azure-managed-hsm-provision-commands.sh => aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-commands.sh} (75%) rename ci-operator/step-registry/hypershift/azure/{managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.metadata.json => aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.metadata.json} (62%) rename ci-operator/step-registry/hypershift/azure/{managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.yaml => aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.yaml} (60%) delete mode 100644 ci-operator/step-registry/hypershift/azure/managed-hsm/provision/OWNERS diff --git a/ci-operator/config/openshift/hypershift/openshift-hypershift-release-5.0__periodics.yaml b/ci-operator/config/openshift/hypershift/openshift-hypershift-release-5.0__periodics.yaml index 238dbb77b30d5..d186c7f7b86e9 100644 --- a/ci-operator/config/openshift/hypershift/openshift-hypershift-release-5.0__periodics.yaml +++ b/ci-operator/config/openshift/hypershift/openshift-hypershift-release-5.0__periodics.yaml @@ -340,18 +340,20 @@ tests: ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true" HYPERSHIFT_AZURE_LOCATION: centralus workflow: hypershift-azure-e2e-v2-self-managed -- as: e2e-v2-azure-self-managed-managed-hsm +- as: e2e-aks-managed-hsm cron: 0 6 * * * steps: - cluster_profile: hypershift-azure - dependencies: - OCP_IMAGE_N1: release:n1minor + cluster_profile: hypershift-aks env: - ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true" - HYPERSHIFT_AZURE_LOCATION: centralus + AUTH_THROUGH_CERTS: "true" + CI_TESTS_RUN: ^TestCreateClusterCustomConfig$ + HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_OFFER: "" + HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_PUBLISHER: "" + HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_SKU: "" + HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_VERSION: "" HYPERSHIFT_AZURE_MANAGED_HSM: "true" HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION: eastus - workflow: hypershift-azure-e2e-v2-self-managed + workflow: hypershift-azure-aks-e2e - as: e2e-azure-kubevirt-ovn cron: 0 4 * * * steps: diff --git a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-5.0-periodics.yaml b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-5.0-periodics.yaml index aee1ab1b84843..62236a4fe4915 100644 --- a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-5.0-periodics.yaml +++ b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-5.0-periodics.yaml @@ -166,6 +166,89 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build07 + cron: 0 6 * * * + decorate: true + decoration_config: + skip_cloning: true + extra_refs: + - base_ref: release-5.0 + org: openshift + repo: hypershift + labels: + ci-operator.openshift.io/cloud: hypershift-aks + ci-operator.openshift.io/cloud-cluster-profile: hypershift-aks + ci-operator.openshift.io/variant: periodics + ci.openshift.io/generator: prowgen + job-release: "5.0" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-hypershift-release-5.0-periodics-e2e-aks-managed-hsm + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=e2e-aks-managed-hsm + - --variant=periodics + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build07 cron: 0 2 * * * @@ -3005,89 +3088,6 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator -- agent: kubernetes - cluster: build07 - cron: 0 6 * * * - decorate: true - decoration_config: - skip_cloning: true - extra_refs: - - base_ref: release-5.0 - org: openshift - repo: hypershift - labels: - ci-operator.openshift.io/cloud: hypershift-azure - ci-operator.openshift.io/cloud-cluster-profile: hypershift-azure - ci-operator.openshift.io/variant: periodics - ci.openshift.io/generator: prowgen - job-release: "5.0" - pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: periodic-ci-openshift-hypershift-release-5.0-periodics-e2e-v2-azure-self-managed-managed-hsm - spec: - containers: - - args: - - --gcs-upload-secret=/secrets/gcs/service-account.json - - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - - --lease-server-credentials-file=/etc/boskos/credentials - - --report-credentials-file=/etc/report/credentials - - --secret-dir=/secrets/ci-pull-credentials - - --target=e2e-v2-azure-self-managed-managed-hsm - - --variant=periodics - command: - - ci-operator - env: - - name: HTTP_SERVER_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest - imagePullPolicy: Always - name: "" - ports: - - containerPort: 8080 - name: http - resources: - requests: - cpu: 10m - volumeMounts: - - mountPath: /etc/boskos - name: boskos - readOnly: true - - mountPath: /secrets/ci-pull-credentials - name: ci-pull-credentials - readOnly: true - - mountPath: /secrets/gcs - name: gcs-credentials - readOnly: true - - mountPath: /secrets/manifest-tool - name: manifest-tool-local-pusher - readOnly: true - - mountPath: /etc/pull-secret - name: pull-secret - readOnly: true - - mountPath: /etc/report - name: result-aggregator - readOnly: true - serviceAccountName: ci-operator - volumes: - - name: boskos - secret: - items: - - key: credentials - path: credentials - secretName: boskos-credentials - - name: ci-pull-credentials - secret: - secretName: ci-pull-credentials - - name: manifest-tool-local-pusher - secret: - secretName: manifest-tool-local-pusher - - name: pull-secret - secret: - secretName: registry-pull-credentials - - name: result-aggregator - secret: - secretName: result-aggregator - agent: kubernetes cluster: build01 cron: 37 9,21 * * * diff --git a/ci-operator/step-registry/hypershift/azure/aks/e2e/hypershift-azure-aks-e2e-workflow.yaml b/ci-operator/step-registry/hypershift/azure/aks/e2e/hypershift-azure-aks-e2e-workflow.yaml index ae475917694f2..41795439c01f5 100644 --- a/ci-operator/step-registry/hypershift/azure/aks/e2e/hypershift-azure-aks-e2e-workflow.yaml +++ b/ci-operator/step-registry/hypershift/azure/aks/e2e/hypershift-azure-aks-e2e-workflow.yaml @@ -17,17 +17,19 @@ workflow: Learn more about HyperShift here: https://github.com/openshift/hypershift - Track HyperShift's development here: https://issues.redhat.com/projects/CNTRLPLANE/summary + Track HyperShift's development here: https://issues.redhat.com/projects/CNTRLPLANE/summary steps: pre: - chain: cucushift-installer-rehearse-azure-aks-provision - ref: hypershift-azure-aks-attach-kv + - ref: hypershift-azure-aks-managed-hsm-provision - ref: hypershift-install - ref: cucushift-hypershift-extended-k8s-mgmt-get-guest-annotations - ref: hypershift-resolve-nodepool-releases test: - ref: hypershift-azure-run-e2e post: + - ref: hypershift-azure-aks-managed-hsm-deprovision - chain: cucushift-installer-rehearse-azure-aks-deprovision env: AKS_NODE_COUNT: "6" @@ -48,6 +50,8 @@ workflow: AKS_CERT_ROTATION_POLL_INTERVAL: "1m" AKS_USE_HYPERSHIFT_MI: "true" ENABLE_SIZE_TAGGING: "true" + HYPERSHIFT_AZURE_MANAGED_HSM: "false" + HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION: "eastus" HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_OFFER: "aro4" HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_PUBLISHER: "azureopenshift" HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_VERSION: "419.6.20250523" diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/OWNERS b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/OWNERS similarity index 100% rename from ci-operator/step-registry/hypershift/azure/managed-hsm/OWNERS rename to ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/OWNERS diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-commands.sh b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-commands.sh similarity index 88% rename from ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-commands.sh rename to ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-commands.sh index 1a61061bc2903..99e3075a77ba5 100644 --- a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-commands.sh +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-commands.sh @@ -37,18 +37,20 @@ fi HSM_NAME_FILE="${SHARED_DIR}/azure_managed_hsm_name" RESOURCE_GROUP_FILE="${SHARED_DIR}/azure_managed_hsm_resource_group" -if [[ ! -s "${HSM_NAME_FILE}" || ! -s "${RESOURCE_GROUP_FILE}" ]]; then +LOCATION_FILE="${SHARED_DIR}/azure_managed_hsm_location" +if [[ ! -s "${HSM_NAME_FILE}" || ! -s "${RESOURCE_GROUP_FILE}" || ! -s "${LOCATION_FILE}" ]]; then echo "Managed HSM resource information was not written; nothing to delete" exit 0 fi -AZURE_AUTH_LOCATION="${CLUSTER_PROFILE_DIR}/osServicePrincipal.json" +AZURE_AUTH_LOCATION="/etc/hypershift-ci-jobs-azurecreds/credentials.json" AZURE_AUTH_CLIENT_ID="$(jq -er .clientId "${AZURE_AUTH_LOCATION}")" AZURE_AUTH_CLIENT_SECRET="$(jq -er .clientSecret "${AZURE_AUTH_LOCATION}")" AZURE_AUTH_TENANT_ID="$(jq -er .tenantId "${AZURE_AUTH_LOCATION}")" AZURE_AUTH_SUBSCRIPTION_ID="$(jq -er .subscriptionId "${AZURE_AUTH_LOCATION}")" HSM_NAME="$(<"${HSM_NAME_FILE}")" RESOURCE_GROUP="$(<"${RESOURCE_GROUP_FILE}")" +HSM_LOCATION="$(<"${LOCATION_FILE}")" az cloud set --name AzureCloud az login \ @@ -75,7 +77,7 @@ if [[ "${PURGE_HSM}" == "true" ]]; then echo "Purging Managed HSM ${HSM_NAME}" az keyvault purge \ --hsm-name "${HSM_NAME}" \ - --location "${HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION}" \ + --location "${HSM_LOCATION}" \ --output none fi diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.metadata.json b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.metadata.json similarity index 61% rename from ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.metadata.json rename to ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.metadata.json index 4e43cd7bd2937..2ff9c9c333873 100644 --- a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.metadata.json +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.metadata.json @@ -1,5 +1,5 @@ { - "path": "hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.yaml", + "path": "hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.yaml", "owners": { "approvers": [ "csrwng", diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.yaml b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.yaml similarity index 52% rename from ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.yaml rename to ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.yaml index 56282f409ab80..cae3637904a07 100644 --- a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/hypershift-azure-managed-hsm-deprovision-ref.yaml +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.yaml @@ -1,15 +1,15 @@ ref: - as: hypershift-azure-managed-hsm-deprovision - best_effort: true + as: hypershift-azure-aks-managed-hsm-deprovision cli: latest - commands: hypershift-azure-managed-hsm-deprovision-commands.sh + commands: hypershift-azure-aks-managed-hsm-deprovision-commands.sh + credentials: + - mount_path: /etc/hypershift-ci-jobs-azurecreds + name: hypershift-ci-jobs-azurecreds + namespace: test-credentials env: - default: "false" documentation: Whether an ephemeral Managed HSM was provisioned for the test. name: HYPERSHIFT_AZURE_MANAGED_HSM - - default: eastus - documentation: Azure location containing the Managed HSM to purge. - name: HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION from: hypershift-tests grace_period: 5m0s resources: @@ -19,4 +19,4 @@ ref: timeout: 15m0s documentation: |- Deletes and purges the ephemeral Managed HSM created for the HyperShift Azure - KMS test using the cluster profile credentials, then removes its resource group. + KMS test using the AKS workflow credentials, then removes its resource group. diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/OWNERS b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/OWNERS similarity index 100% rename from ci-operator/step-registry/hypershift/azure/managed-hsm/deprovision/OWNERS rename to ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/OWNERS diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-commands.sh b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-commands.sh similarity index 75% rename from ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-commands.sh rename to ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-commands.sh index 1fdd0f3611f5e..cdca320ba4baf 100644 --- a/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-commands.sh +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-commands.sh @@ -26,14 +26,25 @@ retry() { done } -AZURE_AUTH_LOCATION="${CLUSTER_PROFILE_DIR}/osServicePrincipal.json" -WORKLOAD_IDENTITIES_FILE="/etc/hypershift-ci-jobs-self-managed-azure-e2e/workload-identities.json" +read_kms_credentials() { + KMS_CREDENTIALS="$(az keyvault secret show \ + --vault-name "${KEY_VAULT_NAME}" \ + --name "${KMS_CREDENTIALS_SECRET}" \ + --query value \ + -o tsv)" + [[ -n "${KMS_CREDENTIALS}" ]] +} + +AZURE_AUTH_LOCATION="/etc/hypershift-ci-jobs-azurecreds/credentials.json" +AZURE_KMS_INFO_LOCATION="/etc/hypershift-ci-jobs-azurecreds/aks-kms-info.json" +AZURE_KEY_VAULT_INFO_LOCATION="/etc/hypershift-ci-jobs-azurecreds/keyvault-info.json" AZURE_AUTH_CLIENT_ID="$(jq -er .clientId "${AZURE_AUTH_LOCATION}")" AZURE_AUTH_CLIENT_SECRET="$(jq -er .clientSecret "${AZURE_AUTH_LOCATION}")" AZURE_AUTH_TENANT_ID="$(jq -er .tenantId "${AZURE_AUTH_LOCATION}")" AZURE_AUTH_SUBSCRIPTION_ID="$(jq -er .subscriptionId "${AZURE_AUTH_LOCATION}")" -KMS_CLIENT_ID="$(jq -er .kmsClientID "${WORKLOAD_IDENTITIES_FILE}")" +KMS_CREDENTIALS_SECRET="$(jq -er '."aks-kms-credentials-secret"' "${AZURE_KMS_INFO_LOCATION}")" +KEY_VAULT_NAME="$(jq -er .keyvaultName "${AZURE_KEY_VAULT_INFO_LOCATION}")" az cloud set --name AzureCloud az login \ @@ -45,6 +56,9 @@ az login \ az account set --subscription "${AZURE_AUTH_SUBSCRIPTION_ID}" ADMIN_OBJECT_ID="$(az ad sp show --id "${AZURE_AUTH_CLIENT_ID}" --query id -o tsv)" +KMS_CREDENTIALS="" +retry 10 30 read_kms_credentials +KMS_CLIENT_ID="$(jq -er .client_id <<<"${KMS_CREDENTIALS}")" KMS_OBJECT_ID="$(az ad sp show --id "${KMS_CLIENT_ID}" --query id -o tsv)" UNIQUE_SUFFIX="$(printf '%s' "${PROW_JOB_ID}" | sha256sum | cut -c1-16)" @@ -55,6 +69,7 @@ KEY_NAME="etcd-encryption" # Write cleanup inputs before provisioning so post steps can remove partial resources. printf '%s\n' "${HSM_NAME}" > "${SHARED_DIR}/azure_managed_hsm_name" printf '%s\n' "${RESOURCE_GROUP}" > "${SHARED_DIR}/azure_managed_hsm_resource_group" +printf '%s\n' "${HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION}" > "${SHARED_DIR}/azure_managed_hsm_location" az group create \ --name "${RESOURCE_GROUP}" \ @@ -68,12 +83,18 @@ az keyvault create \ --location "${HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION}" \ --administrators "${ADMIN_OBJECT_ID}" \ --retention-days 7 \ + --no-wait \ --output none +az keyvault wait \ + --hsm-name "${HSM_NAME}" \ + --resource-group "${RESOURCE_GROUP}" \ + --created + SECURITY_DOMAIN_DIR="$(mktemp -d)" trap 'rm -rf "${SECURITY_DOMAIN_DIR}"' EXIT -security_domain_args=() +security_domain_certificates=() for index in 0 1 2; do openssl req \ -newkey rsa:2048 \ @@ -84,13 +105,13 @@ for index in 0 1 2; do -out "${SECURITY_DOMAIN_DIR}/cert-${index}.pem" \ -subj "/CN=HyperShiftCI${index}" \ 2>/dev/null - security_domain_args+=(--sd-wrapping-keys "${SECURITY_DOMAIN_DIR}/cert-${index}.pem") + security_domain_certificates+=("${SECURITY_DOMAIN_DIR}/cert-${index}.pem") done az keyvault security-domain download \ --hsm-name "${HSM_NAME}" \ --security-domain-file "${SECURITY_DOMAIN_DIR}/security-domain.json" \ - "${security_domain_args[@]}" \ + --sd-wrapping-keys "${security_domain_certificates[@]}" \ --sd-quorum 2 \ --output none diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.metadata.json b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.metadata.json similarity index 62% rename from ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.metadata.json rename to ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.metadata.json index 875f285e83425..a0179b23f78ca 100644 --- a/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.metadata.json +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.metadata.json @@ -1,5 +1,5 @@ { - "path": "hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.yaml", + "path": "hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.yaml", "owners": { "approvers": [ "csrwng", diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.yaml b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.yaml similarity index 60% rename from ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.yaml rename to ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.yaml index 47b0e76253241..695203df86f61 100644 --- a/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/hypershift-azure-managed-hsm-provision-ref.yaml +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.yaml @@ -1,10 +1,10 @@ ref: - as: hypershift-azure-managed-hsm-provision + as: hypershift-azure-aks-managed-hsm-provision cli: latest - commands: hypershift-azure-managed-hsm-provision-commands.sh + commands: hypershift-azure-aks-managed-hsm-provision-commands.sh credentials: - - mount_path: /etc/hypershift-ci-jobs-self-managed-azure-e2e - name: hypershift-ci-jobs-self-managed-azure-e2e + - mount_path: /etc/hypershift-ci-jobs-azurecreds + name: hypershift-ci-jobs-azurecreds namespace: test-credentials env: - default: "false" @@ -22,5 +22,5 @@ ref: timeout: 45m0s documentation: |- Provisions and activates an ephemeral Azure Managed HSM, creates an RSA-HSM - key using the cluster profile credentials, grants the self-managed HyperShift - KMS workload identity access to it, and writes the key ID to SHARED_DIR. + key using the AKS workflow credentials, grants the HyperShift KMS managed + identity access to it, and writes the key ID to SHARED_DIR. diff --git a/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh b/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh index ff2de1e48ddcb..0341fb58a9d52 100644 --- a/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh +++ b/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh @@ -10,11 +10,4 @@ if [[ -f "${SHARED_DIR}/nodepool_release_images" ]]; then source "${SHARED_DIR}/nodepool_release_images" fi -if [[ -s "${SHARED_DIR}/azure_managed_hsm_key_id" ]]; then - set +x - export AZURE_ENCRYPTION_KEY_ID - AZURE_ENCRYPTION_KEY_ID="$(<"${SHARED_DIR}/azure_managed_hsm_key_id")" - set -x -fi - /hypershift/bin/create-guests diff --git a/ci-operator/step-registry/hypershift/azure/e2e/v2-self-managed/hypershift-azure-e2e-v2-self-managed-workflow.yaml b/ci-operator/step-registry/hypershift/azure/e2e/v2-self-managed/hypershift-azure-e2e-v2-self-managed-workflow.yaml index eeac3c0dc4d58..bad47d829cea0 100644 --- a/ci-operator/step-registry/hypershift/azure/e2e/v2-self-managed/hypershift-azure-e2e-v2-self-managed-workflow.yaml +++ b/ci-operator/step-registry/hypershift/azure/e2e/v2-self-managed/hypershift-azure-e2e-v2-self-managed-workflow.yaml @@ -25,22 +25,18 @@ workflow: - ref: hypershift-azure-setup-private-link - ref: hypershift-install - ref: hypershift-resolve-nodepool-releases - - ref: hypershift-azure-managed-hsm-provision - chain: hypershift-azure-create-selfmanaged-guests test: - chain: hypershift-azure-run-e2e-v2-selfmanaged post: - chain: hypershift-azure-dump-selfmanaged-guests - chain: hypershift-azure-destroy-selfmanaged-guests - - ref: hypershift-azure-managed-hsm-deprovision - chain: hypershift-destroy-nested-management-cluster env: CLOUD_PROVIDER: "Azure" HYPERSHIFT_PLATFORM: "azure" HYPERSHIFT_NODE_COUNT: "6" HYPERSHIFT_AZURE_LOCATION: "centralus" - HYPERSHIFT_AZURE_MANAGED_HSM: "false" - HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION: "eastus" AZURE_SELF_MANAGED: "true" HYPERSHIFT_EXTERNAL_DNS_DOMAIN: "sm.hcp-sm-azure.azure.devcluster.openshift.com" HYPERSHIFT_ETCD_STORAGE_CLASS: "managed-csi-premium-v2" diff --git a/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/OWNERS b/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/OWNERS deleted file mode 100644 index 6c30e7a30d980..0000000000000 --- a/ci-operator/step-registry/hypershift/azure/managed-hsm/provision/OWNERS +++ /dev/null @@ -1,11 +0,0 @@ -approvers: -- csrwng -- enxebre -- sjenning -- bryan-cox -options: {} -reviewers: -- csrwng -- enxebre -- sjenning -- bryan-cox diff --git a/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml b/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml index 9700a9038a417..65a380fea1401 100644 --- a/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml +++ b/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml @@ -10,12 +10,6 @@ chain: commands: |- set -xuo pipefail export KUBECONFIG="${SHARED_DIR}/management_cluster_kubeconfig" - if [[ -s "${SHARED_DIR}/azure_managed_hsm_key_id" ]]; then - set +x - export AZURE_ENCRYPTION_KEY_ID - AZURE_ENCRYPTION_KEY_ID="$(<"${SHARED_DIR}/azure_managed_hsm_key_id")" - set -x - fi /hypershift/bin/run-tests timeout: 150m grace_period: 5m diff --git a/ci-operator/step-registry/hypershift/azure/run-e2e/hypershift-azure-run-e2e-commands.sh b/ci-operator/step-registry/hypershift/azure/run-e2e/hypershift-azure-run-e2e-commands.sh index fe9fc4ccf7cf4..5d4a3736f550b 100644 --- a/ci-operator/step-registry/hypershift/azure/run-e2e/hypershift-azure-run-e2e-commands.sh +++ b/ci-operator/step-registry/hypershift/azure/run-e2e/hypershift-azure-run-e2e-commands.sh @@ -19,6 +19,11 @@ AZURE_OIDC_ISSUER_URL="$(<"${AZURE_OIDC_ISSUER_URL_LOCATION}" jq -r .oidcIssuerU AZURE_KMS_INFO_LOCATION="/etc/hypershift-ci-jobs-azurecreds/aks-kms-info.json" AKS_KMS_KEY="$(jq -r '."aks-kms-key"' "${AZURE_KMS_INFO_LOCATION}")" AKS_KMS_CREDENTIALS_SECRET="$(jq -r '."aks-kms-credentials-secret"' "${AZURE_KMS_INFO_LOCATION}")" +HIDE_AZURE_KMS_KEY="false" +if [[ -s "${SHARED_DIR}/azure_managed_hsm_key_id" ]]; then + AKS_KMS_KEY="$(<"${SHARED_DIR}/azure_managed_hsm_key_id")" + HIDE_AZURE_KMS_KEY="true" +fi az --version az login --service-principal -u "${AZURE_AUTH_CLIENT_ID}" -p "${AZURE_AUTH_CLIENT_SECRET}" --tenant "${AZURE_AUTH_TENANT_ID}" --output none @@ -145,6 +150,10 @@ if [[ "${OAUTH_EXTERNAL_OIDC_PROVIDER}" != "" ]]; then esac fi +if [[ "${HIDE_AZURE_KMS_KEY}" == "true" ]]; then + set +x +fi + hack/ci-test-e2e.sh -test.v \ -test.run=${CI_TESTS_RUN:-} \ -test.parallel=20 \ @@ -164,11 +173,16 @@ hack/ci-test-e2e.sh -test.v \ ${MI_ARGS:-} \ ${DP_ARGS:-} \ ${AZURE_MULTI_ARCH_PARAMS:-} \ - --e2e.azure-encryption-key-id=${AKS_KMS_KEY} \ + --e2e.azure-encryption-key-id="${AKS_KMS_KEY}" \ --e2e.azure-kms-credentials-secret-name=${AKS_KMS_CREDENTIALS_SECRET} \ ${MARKETPLACE_IMAGE_PARAMS} \ --e2e.latest-release-image="${OCP_IMAGE_LATEST}" \ ${OAUTH_EXTERNAL_OIDC_PARAM:-} \ --e2e.previous-release-image="${OCP_IMAGE_PREVIOUS}" \ ${ADDITIONAL_PULL_SECRET_PARAMS:-} & -wait $! \ No newline at end of file +e2e_pid=$! + +if [[ "${HIDE_AZURE_KMS_KEY}" == "true" ]]; then + set -x +fi +wait "${e2e_pid}" \ No newline at end of file From 4db9433c6aea3cff568b1bcb3e0b7f71825fb580 Mon Sep 17 00:00:00 2001 From: Hilliary Lipsig Date: Tue, 11 Aug 2026 16:02:13 -0700 Subject: [PATCH 08/11] ci(hypershift): update Managed HSM owners Remove sjenning from the approver and reviewer lists for the AKS Managed HSM steps and regenerate their metadata. Signed-off-by: Hilliary Lipsig Commit-Message-Assisted-by: Claude (via Claude Code) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../hypershift/azure/aks/managed-hsm/deprovision/OWNERS | 2 -- ...pershift-azure-aks-managed-hsm-deprovision-ref.metadata.json | 2 -- .../hypershift/azure/aks/managed-hsm/provision/OWNERS | 2 -- ...hypershift-azure-aks-managed-hsm-provision-ref.metadata.json | 2 -- 4 files changed, 8 deletions(-) diff --git a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/OWNERS b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/OWNERS index 6c30e7a30d980..2c6a9dc194302 100644 --- a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/OWNERS +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/OWNERS @@ -1,11 +1,9 @@ approvers: - csrwng - enxebre -- sjenning - bryan-cox options: {} reviewers: - csrwng - enxebre -- sjenning - bryan-cox diff --git a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.metadata.json b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.metadata.json index 2ff9c9c333873..f4e7f00b6b9f4 100644 --- a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.metadata.json +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.metadata.json @@ -4,13 +4,11 @@ "approvers": [ "csrwng", "enxebre", - "sjenning", "bryan-cox" ], "reviewers": [ "csrwng", "enxebre", - "sjenning", "bryan-cox" ] } diff --git a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/OWNERS b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/OWNERS index 6c30e7a30d980..2c6a9dc194302 100644 --- a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/OWNERS +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/OWNERS @@ -1,11 +1,9 @@ approvers: - csrwng - enxebre -- sjenning - bryan-cox options: {} reviewers: - csrwng - enxebre -- sjenning - bryan-cox diff --git a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.metadata.json b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.metadata.json index a0179b23f78ca..6f645c8b9de18 100644 --- a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.metadata.json +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.metadata.json @@ -4,13 +4,11 @@ "approvers": [ "csrwng", "enxebre", - "sjenning", "bryan-cox" ], "reviewers": [ "csrwng", "enxebre", - "sjenning", "bryan-cox" ] } From 267474859229ed6d932ae700b42a26454a674e13 Mon Sep 17 00:00:00 2001 From: Hilliary Lipsig Date: Tue, 11 Aug 2026 16:07:27 -0700 Subject: [PATCH 09/11] ci(hypershift): enforce Managed HSM cleanup Fail when cleanup state is missing or Azure state checks are inconclusive. Confirm purge completion, wait for resource group deletion, and allow enough time for synchronous cleanup. Signed-off-by: Hilliary Lipsig Commit-Message-Assisted-by: Claude (via Claude Code) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...re-aks-managed-hsm-deprovision-commands.sh | 45 +++++++++++++++---- ...azure-aks-managed-hsm-deprovision-ref.yaml | 2 +- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-commands.sh b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-commands.sh index 99e3075a77ba5..0f7d9080477e3 100644 --- a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-commands.sh +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-commands.sh @@ -30,6 +30,15 @@ deleted_hsm_exists() { [[ "${count}" == "1" ]] } +deleted_hsm_absent() { + local count + count="$(az keyvault list-deleted \ + --resource-type hsm \ + --query "[?name=='${HSM_NAME}'] | length(@)" \ + -o tsv)" + [[ "${count}" == "0" ]] +} + if [[ "${HYPERSHIFT_AZURE_MANAGED_HSM}" != "true" ]]; then echo "Managed HSM deprovisioning is disabled" exit 0 @@ -39,8 +48,8 @@ HSM_NAME_FILE="${SHARED_DIR}/azure_managed_hsm_name" RESOURCE_GROUP_FILE="${SHARED_DIR}/azure_managed_hsm_resource_group" LOCATION_FILE="${SHARED_DIR}/azure_managed_hsm_location" if [[ ! -s "${HSM_NAME_FILE}" || ! -s "${RESOURCE_GROUP_FILE}" || ! -s "${LOCATION_FILE}" ]]; then - echo "Managed HSM resource information was not written; nothing to delete" - exit 0 + echo "Managed HSM cleanup cannot continue because resource information is missing" >&2 + exit 1 fi AZURE_AUTH_LOCATION="/etc/hypershift-ci-jobs-azurecreds/credentials.json" @@ -61,27 +70,45 @@ az login \ --output none az account set --subscription "${AZURE_AUTH_SUBSCRIPTION_ID}" -PURGE_HSM=true -if az keyvault show --hsm-name "${HSM_NAME}" --output none 2>/dev/null; then +ACTIVE_HSM_COUNT="$(az keyvault list \ + --resource-type hsm \ + --query "[?name=='${HSM_NAME}'] | length(@)" \ + -o tsv)" +DELETED_HSM_COUNT="$(az keyvault list-deleted \ + --resource-type hsm \ + --query "[?name=='${HSM_NAME}'] | length(@)" \ + -o tsv)" + +if [[ "${ACTIVE_HSM_COUNT}" == "1" ]]; then echo "Deleting Managed HSM ${HSM_NAME}" az keyvault delete --hsm-name "${HSM_NAME}" --output none retry 20 30 deleted_hsm_exists -elif deleted_hsm_exists; then +elif [[ "${ACTIVE_HSM_COUNT}" != "0" ]]; then + echo "Unexpected active Managed HSM count for ${HSM_NAME}: ${ACTIVE_HSM_COUNT}" >&2 + exit 1 +elif [[ "${DELETED_HSM_COUNT}" == "1" ]]; then echo "Managed HSM ${HSM_NAME} is already soft-deleted" +elif [[ "${DELETED_HSM_COUNT}" != "0" ]]; then + echo "Unexpected deleted Managed HSM count for ${HSM_NAME}: ${DELETED_HSM_COUNT}" >&2 + exit 1 else echo "Managed HSM ${HSM_NAME} was not created; nothing to purge" - PURGE_HSM=false fi -if [[ "${PURGE_HSM}" == "true" ]]; then +if [[ "${ACTIVE_HSM_COUNT}" == "1" || "${DELETED_HSM_COUNT}" == "1" ]]; then echo "Purging Managed HSM ${HSM_NAME}" az keyvault purge \ --hsm-name "${HSM_NAME}" \ --location "${HSM_LOCATION}" \ --output none + retry 20 15 deleted_hsm_absent fi -if az group show --name "${RESOURCE_GROUP}" --output none 2>/dev/null; then +RESOURCE_GROUP_EXISTS="$(az group exists --name "${RESOURCE_GROUP}")" +if [[ "${RESOURCE_GROUP_EXISTS}" == "true" ]]; then echo "Deleting resource group ${RESOURCE_GROUP}" - az group delete --name "${RESOURCE_GROUP}" --yes --no-wait + az group delete --name "${RESOURCE_GROUP}" --yes +elif [[ "${RESOURCE_GROUP_EXISTS}" != "false" ]]; then + echo "Unexpected resource group existence result for ${RESOURCE_GROUP}: ${RESOURCE_GROUP_EXISTS}" >&2 + exit 1 fi diff --git a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.yaml b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.yaml index cae3637904a07..b998072b3572e 100644 --- a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.yaml +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.yaml @@ -16,7 +16,7 @@ ref: requests: cpu: 100m memory: 100Mi - timeout: 15m0s + timeout: 45m0s documentation: |- Deletes and purges the ephemeral Managed HSM created for the HyperShift Azure KMS test using the AKS workflow credentials, then removes its resource group. From a489b4d9d7fa121a12d5e6ba638027f287d4d715 Mon Sep 17 00:00:00 2001 From: Hilliary Lipsig Date: Wed, 12 Aug 2026 09:56:55 -0700 Subject: [PATCH 10/11] ci(hypershift): normalize Managed HSM periodic Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../openshift-hypershift-release-5.0__periodics.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-operator/config/openshift/hypershift/openshift-hypershift-release-5.0__periodics.yaml b/ci-operator/config/openshift/hypershift/openshift-hypershift-release-5.0__periodics.yaml index d186c7f7b86e9..6775a54086728 100644 --- a/ci-operator/config/openshift/hypershift/openshift-hypershift-release-5.0__periodics.yaml +++ b/ci-operator/config/openshift/hypershift/openshift-hypershift-release-5.0__periodics.yaml @@ -347,12 +347,12 @@ tests: env: AUTH_THROUGH_CERTS: "true" CI_TESTS_RUN: ^TestCreateClusterCustomConfig$ + HYPERSHIFT_AZURE_MANAGED_HSM: "true" + HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION: eastus HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_OFFER: "" HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_PUBLISHER: "" HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_SKU: "" HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_VERSION: "" - HYPERSHIFT_AZURE_MANAGED_HSM: "true" - HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION: eastus workflow: hypershift-azure-aks-e2e - as: e2e-azure-kubevirt-ovn cron: 0 4 * * * From c811564bf6df8b66a9e7fc6e54c9255799f75c43 Mon Sep 17 00:00:00 2001 From: Hilliary Lipsig Date: Wed, 12 Aug 2026 11:07:41 -0700 Subject: [PATCH 11/11] ci(hypershift): add Managed HSM owners Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../hypershift/azure/aks/managed-hsm/OWNERS | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 ci-operator/step-registry/hypershift/azure/aks/managed-hsm/OWNERS diff --git a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/OWNERS b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/OWNERS new file mode 100644 index 0000000000000..2c6a9dc194302 --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/OWNERS @@ -0,0 +1,9 @@ +approvers: +- csrwng +- enxebre +- bryan-cox +options: {} +reviewers: +- csrwng +- enxebre +- bryan-cox