diff --git a/ci-operator/config/openshift/hypershift/openshift-hypershift-release-5.0__periodics.yaml b/ci-operator/config/openshift/hypershift/openshift-hypershift-release-5.0__periodics.yaml index d395f51c24e2d..6775a54086728 100644 --- a/ci-operator/config/openshift/hypershift/openshift-hypershift-release-5.0__periodics.yaml +++ b/ci-operator/config/openshift/hypershift/openshift-hypershift-release-5.0__periodics.yaml @@ -340,6 +340,20 @@ tests: ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true" HYPERSHIFT_AZURE_LOCATION: centralus workflow: hypershift-azure-e2e-v2-self-managed +- as: e2e-aks-managed-hsm + cron: 0 6 * * * + steps: + cluster_profile: hypershift-aks + env: + AUTH_THROUGH_CERTS: "true" + CI_TESTS_RUN: ^TestCreateClusterCustomConfig$ + HYPERSHIFT_AZURE_MANAGED_HSM: "true" + HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION: eastus + HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_OFFER: "" + HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_PUBLISHER: "" + HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_SKU: "" + HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_VERSION: "" + workflow: hypershift-azure-aks-e2e - as: e2e-azure-kubevirt-ovn cron: 0 4 * * * steps: diff --git a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-5.0-periodics.yaml b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-5.0-periodics.yaml index 8a9c856dae65c..62236a4fe4915 100644 --- a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-5.0-periodics.yaml +++ b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-5.0-periodics.yaml @@ -166,6 +166,89 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build07 + cron: 0 6 * * * + decorate: true + decoration_config: + skip_cloning: true + extra_refs: + - base_ref: release-5.0 + org: openshift + repo: hypershift + labels: + ci-operator.openshift.io/cloud: hypershift-aks + ci-operator.openshift.io/cloud-cluster-profile: hypershift-aks + ci-operator.openshift.io/variant: periodics + ci.openshift.io/generator: prowgen + job-release: "5.0" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-hypershift-release-5.0-periodics-e2e-aks-managed-hsm + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=e2e-aks-managed-hsm + - --variant=periodics + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build07 cron: 0 2 * * * diff --git a/ci-operator/step-registry/hypershift/azure/aks/e2e/hypershift-azure-aks-e2e-workflow.yaml b/ci-operator/step-registry/hypershift/azure/aks/e2e/hypershift-azure-aks-e2e-workflow.yaml index ae475917694f2..41795439c01f5 100644 --- a/ci-operator/step-registry/hypershift/azure/aks/e2e/hypershift-azure-aks-e2e-workflow.yaml +++ b/ci-operator/step-registry/hypershift/azure/aks/e2e/hypershift-azure-aks-e2e-workflow.yaml @@ -17,17 +17,19 @@ workflow: Learn more about HyperShift here: https://github.com/openshift/hypershift - Track HyperShift's development here: https://issues.redhat.com/projects/CNTRLPLANE/summary + Track HyperShift's development here: https://issues.redhat.com/projects/CNTRLPLANE/summary steps: pre: - chain: cucushift-installer-rehearse-azure-aks-provision - ref: hypershift-azure-aks-attach-kv + - ref: hypershift-azure-aks-managed-hsm-provision - ref: hypershift-install - ref: cucushift-hypershift-extended-k8s-mgmt-get-guest-annotations - ref: hypershift-resolve-nodepool-releases test: - ref: hypershift-azure-run-e2e post: + - ref: hypershift-azure-aks-managed-hsm-deprovision - chain: cucushift-installer-rehearse-azure-aks-deprovision env: AKS_NODE_COUNT: "6" @@ -48,6 +50,8 @@ workflow: AKS_CERT_ROTATION_POLL_INTERVAL: "1m" AKS_USE_HYPERSHIFT_MI: "true" ENABLE_SIZE_TAGGING: "true" + HYPERSHIFT_AZURE_MANAGED_HSM: "false" + HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION: "eastus" HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_OFFER: "aro4" HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_PUBLISHER: "azureopenshift" HYPERSHIFT_AZURE_MARKETPLACE_IMAGE_VERSION: "419.6.20250523" diff --git a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/OWNERS b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/OWNERS new file mode 100644 index 0000000000000..2c6a9dc194302 --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/OWNERS @@ -0,0 +1,9 @@ +approvers: +- csrwng +- enxebre +- bryan-cox +options: {} +reviewers: +- csrwng +- enxebre +- bryan-cox diff --git a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/OWNERS b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/OWNERS new file mode 100644 index 0000000000000..2c6a9dc194302 --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/OWNERS @@ -0,0 +1,9 @@ +approvers: +- csrwng +- enxebre +- bryan-cox +options: {} +reviewers: +- csrwng +- enxebre +- bryan-cox diff --git a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-commands.sh b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-commands.sh new file mode 100644 index 0000000000000..0f7d9080477e3 --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-commands.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +set -euo pipefail + +retry() { + local attempts="$1" + local delay="$2" + shift 2 + + local attempt + for ((attempt = 1; attempt <= attempts; attempt++)); do + if "$@"; then + return 0 + fi + if ((attempt == attempts)); then + echo "Command failed after ${attempts} attempts: $*" >&2 + return 1 + fi + echo "Attempt ${attempt}/${attempts} failed; retrying in ${delay} seconds" + sleep "${delay}" + done +} + +deleted_hsm_exists() { + local count + count="$(az keyvault list-deleted \ + --resource-type hsm \ + --query "[?name=='${HSM_NAME}'] | length(@)" \ + -o tsv)" + [[ "${count}" == "1" ]] +} + +deleted_hsm_absent() { + local count + count="$(az keyvault list-deleted \ + --resource-type hsm \ + --query "[?name=='${HSM_NAME}'] | length(@)" \ + -o tsv)" + [[ "${count}" == "0" ]] +} + +if [[ "${HYPERSHIFT_AZURE_MANAGED_HSM}" != "true" ]]; then + echo "Managed HSM deprovisioning is disabled" + exit 0 +fi + +HSM_NAME_FILE="${SHARED_DIR}/azure_managed_hsm_name" +RESOURCE_GROUP_FILE="${SHARED_DIR}/azure_managed_hsm_resource_group" +LOCATION_FILE="${SHARED_DIR}/azure_managed_hsm_location" +if [[ ! -s "${HSM_NAME_FILE}" || ! -s "${RESOURCE_GROUP_FILE}" || ! -s "${LOCATION_FILE}" ]]; then + echo "Managed HSM cleanup cannot continue because resource information is missing" >&2 + exit 1 +fi + +AZURE_AUTH_LOCATION="/etc/hypershift-ci-jobs-azurecreds/credentials.json" +AZURE_AUTH_CLIENT_ID="$(jq -er .clientId "${AZURE_AUTH_LOCATION}")" +AZURE_AUTH_CLIENT_SECRET="$(jq -er .clientSecret "${AZURE_AUTH_LOCATION}")" +AZURE_AUTH_TENANT_ID="$(jq -er .tenantId "${AZURE_AUTH_LOCATION}")" +AZURE_AUTH_SUBSCRIPTION_ID="$(jq -er .subscriptionId "${AZURE_AUTH_LOCATION}")" +HSM_NAME="$(<"${HSM_NAME_FILE}")" +RESOURCE_GROUP="$(<"${RESOURCE_GROUP_FILE}")" +HSM_LOCATION="$(<"${LOCATION_FILE}")" + +az cloud set --name AzureCloud +az login \ + --service-principal \ + --username "${AZURE_AUTH_CLIENT_ID}" \ + --password "${AZURE_AUTH_CLIENT_SECRET}" \ + --tenant "${AZURE_AUTH_TENANT_ID}" \ + --output none +az account set --subscription "${AZURE_AUTH_SUBSCRIPTION_ID}" + +ACTIVE_HSM_COUNT="$(az keyvault list \ + --resource-type hsm \ + --query "[?name=='${HSM_NAME}'] | length(@)" \ + -o tsv)" +DELETED_HSM_COUNT="$(az keyvault list-deleted \ + --resource-type hsm \ + --query "[?name=='${HSM_NAME}'] | length(@)" \ + -o tsv)" + +if [[ "${ACTIVE_HSM_COUNT}" == "1" ]]; then + echo "Deleting Managed HSM ${HSM_NAME}" + az keyvault delete --hsm-name "${HSM_NAME}" --output none + retry 20 30 deleted_hsm_exists +elif [[ "${ACTIVE_HSM_COUNT}" != "0" ]]; then + echo "Unexpected active Managed HSM count for ${HSM_NAME}: ${ACTIVE_HSM_COUNT}" >&2 + exit 1 +elif [[ "${DELETED_HSM_COUNT}" == "1" ]]; then + echo "Managed HSM ${HSM_NAME} is already soft-deleted" +elif [[ "${DELETED_HSM_COUNT}" != "0" ]]; then + echo "Unexpected deleted Managed HSM count for ${HSM_NAME}: ${DELETED_HSM_COUNT}" >&2 + exit 1 +else + echo "Managed HSM ${HSM_NAME} was not created; nothing to purge" +fi + +if [[ "${ACTIVE_HSM_COUNT}" == "1" || "${DELETED_HSM_COUNT}" == "1" ]]; then + echo "Purging Managed HSM ${HSM_NAME}" + az keyvault purge \ + --hsm-name "${HSM_NAME}" \ + --location "${HSM_LOCATION}" \ + --output none + retry 20 15 deleted_hsm_absent +fi + +RESOURCE_GROUP_EXISTS="$(az group exists --name "${RESOURCE_GROUP}")" +if [[ "${RESOURCE_GROUP_EXISTS}" == "true" ]]; then + echo "Deleting resource group ${RESOURCE_GROUP}" + az group delete --name "${RESOURCE_GROUP}" --yes +elif [[ "${RESOURCE_GROUP_EXISTS}" != "false" ]]; then + echo "Unexpected resource group existence result for ${RESOURCE_GROUP}: ${RESOURCE_GROUP_EXISTS}" >&2 + exit 1 +fi diff --git a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.metadata.json b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.metadata.json new file mode 100644 index 0000000000000..f4e7f00b6b9f4 --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.metadata.json @@ -0,0 +1,15 @@ +{ + "path": "hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.yaml", + "owners": { + "approvers": [ + "csrwng", + "enxebre", + "bryan-cox" + ], + "reviewers": [ + "csrwng", + "enxebre", + "bryan-cox" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.yaml b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.yaml new file mode 100644 index 0000000000000..b998072b3572e --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/deprovision/hypershift-azure-aks-managed-hsm-deprovision-ref.yaml @@ -0,0 +1,22 @@ +ref: + as: hypershift-azure-aks-managed-hsm-deprovision + cli: latest + commands: hypershift-azure-aks-managed-hsm-deprovision-commands.sh + credentials: + - mount_path: /etc/hypershift-ci-jobs-azurecreds + name: hypershift-ci-jobs-azurecreds + namespace: test-credentials + env: + - default: "false" + documentation: Whether an ephemeral Managed HSM was provisioned for the test. + name: HYPERSHIFT_AZURE_MANAGED_HSM + from: hypershift-tests + grace_period: 5m0s + resources: + requests: + cpu: 100m + memory: 100Mi + timeout: 45m0s + documentation: |- + Deletes and purges the ephemeral Managed HSM created for the HyperShift Azure + KMS test using the AKS workflow credentials, then removes its resource group. diff --git a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/OWNERS b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/OWNERS new file mode 100644 index 0000000000000..2c6a9dc194302 --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/OWNERS @@ -0,0 +1,9 @@ +approvers: +- csrwng +- enxebre +- bryan-cox +options: {} +reviewers: +- csrwng +- enxebre +- bryan-cox diff --git a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-commands.sh b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-commands.sh new file mode 100644 index 0000000000000..cdca320ba4baf --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-commands.sh @@ -0,0 +1,152 @@ +#!/bin/bash + +set -euo pipefail + +if [[ "${HYPERSHIFT_AZURE_MANAGED_HSM}" != "true" ]]; then + echo "Managed HSM provisioning is disabled" + exit 0 +fi + +retry() { + local attempts="$1" + local delay="$2" + shift 2 + + local attempt + for ((attempt = 1; attempt <= attempts; attempt++)); do + if "$@"; then + return 0 + fi + if ((attempt == attempts)); then + echo "Command failed after ${attempts} attempts: $*" >&2 + return 1 + fi + echo "Attempt ${attempt}/${attempts} failed; retrying in ${delay} seconds" + sleep "${delay}" + done +} + +read_kms_credentials() { + KMS_CREDENTIALS="$(az keyvault secret show \ + --vault-name "${KEY_VAULT_NAME}" \ + --name "${KMS_CREDENTIALS_SECRET}" \ + --query value \ + -o tsv)" + [[ -n "${KMS_CREDENTIALS}" ]] +} + +AZURE_AUTH_LOCATION="/etc/hypershift-ci-jobs-azurecreds/credentials.json" +AZURE_KMS_INFO_LOCATION="/etc/hypershift-ci-jobs-azurecreds/aks-kms-info.json" +AZURE_KEY_VAULT_INFO_LOCATION="/etc/hypershift-ci-jobs-azurecreds/keyvault-info.json" + +AZURE_AUTH_CLIENT_ID="$(jq -er .clientId "${AZURE_AUTH_LOCATION}")" +AZURE_AUTH_CLIENT_SECRET="$(jq -er .clientSecret "${AZURE_AUTH_LOCATION}")" +AZURE_AUTH_TENANT_ID="$(jq -er .tenantId "${AZURE_AUTH_LOCATION}")" +AZURE_AUTH_SUBSCRIPTION_ID="$(jq -er .subscriptionId "${AZURE_AUTH_LOCATION}")" +KMS_CREDENTIALS_SECRET="$(jq -er '."aks-kms-credentials-secret"' "${AZURE_KMS_INFO_LOCATION}")" +KEY_VAULT_NAME="$(jq -er .keyvaultName "${AZURE_KEY_VAULT_INFO_LOCATION}")" + +az cloud set --name AzureCloud +az login \ + --service-principal \ + --username "${AZURE_AUTH_CLIENT_ID}" \ + --password "${AZURE_AUTH_CLIENT_SECRET}" \ + --tenant "${AZURE_AUTH_TENANT_ID}" \ + --output none +az account set --subscription "${AZURE_AUTH_SUBSCRIPTION_ID}" + +ADMIN_OBJECT_ID="$(az ad sp show --id "${AZURE_AUTH_CLIENT_ID}" --query id -o tsv)" +KMS_CREDENTIALS="" +retry 10 30 read_kms_credentials +KMS_CLIENT_ID="$(jq -er .client_id <<<"${KMS_CREDENTIALS}")" +KMS_OBJECT_ID="$(az ad sp show --id "${KMS_CLIENT_ID}" --query id -o tsv)" + +UNIQUE_SUFFIX="$(printf '%s' "${PROW_JOB_ID}" | sha256sum | cut -c1-16)" +HSM_NAME="hshift-${UNIQUE_SUFFIX}" +RESOURCE_GROUP="${HSM_NAME}-rg" +KEY_NAME="etcd-encryption" + +# Write cleanup inputs before provisioning so post steps can remove partial resources. +printf '%s\n' "${HSM_NAME}" > "${SHARED_DIR}/azure_managed_hsm_name" +printf '%s\n' "${RESOURCE_GROUP}" > "${SHARED_DIR}/azure_managed_hsm_resource_group" +printf '%s\n' "${HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION}" > "${SHARED_DIR}/azure_managed_hsm_location" + +az group create \ + --name "${RESOURCE_GROUP}" \ + --location "${HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION}" \ + --tags "prow-job-id=${PROW_JOB_ID}" \ + --output none + +az keyvault create \ + --hsm-name "${HSM_NAME}" \ + --resource-group "${RESOURCE_GROUP}" \ + --location "${HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION}" \ + --administrators "${ADMIN_OBJECT_ID}" \ + --retention-days 7 \ + --no-wait \ + --output none + +az keyvault wait \ + --hsm-name "${HSM_NAME}" \ + --resource-group "${RESOURCE_GROUP}" \ + --created + +SECURITY_DOMAIN_DIR="$(mktemp -d)" +trap 'rm -rf "${SECURITY_DOMAIN_DIR}"' EXIT + +security_domain_certificates=() +for index in 0 1 2; do + openssl req \ + -newkey rsa:2048 \ + -nodes \ + -keyout "${SECURITY_DOMAIN_DIR}/key-${index}.pem" \ + -x509 \ + -days 1 \ + -out "${SECURITY_DOMAIN_DIR}/cert-${index}.pem" \ + -subj "/CN=HyperShiftCI${index}" \ + 2>/dev/null + security_domain_certificates+=("${SECURITY_DOMAIN_DIR}/cert-${index}.pem") +done + +az keyvault security-domain download \ + --hsm-name "${HSM_NAME}" \ + --security-domain-file "${SECURITY_DOMAIN_DIR}/security-domain.json" \ + --sd-wrapping-keys "${security_domain_certificates[@]}" \ + --sd-quorum 2 \ + --output none + +retry 10 30 az keyvault role assignment create \ + --hsm-name "${HSM_NAME}" \ + --assignee "${ADMIN_OBJECT_ID}" \ + --role "Managed HSM Crypto Officer" \ + --scope "/" \ + --output none + +retry 10 30 az keyvault role assignment create \ + --hsm-name "${HSM_NAME}" \ + --assignee "${ADMIN_OBJECT_ID}" \ + --role "Managed HSM Crypto User" \ + --scope "/keys" \ + --output none + +retry 12 30 az keyvault key create \ + --hsm-name "${HSM_NAME}" \ + --name "${KEY_NAME}" \ + --kty RSA-HSM \ + --output none + +KEY_ID="$(az keyvault key show \ + --hsm-name "${HSM_NAME}" \ + --name "${KEY_NAME}" \ + --query key.kid \ + -o tsv)" + +retry 10 30 az keyvault role assignment create \ + --hsm-name "${HSM_NAME}" \ + --assignee "${KMS_OBJECT_ID}" \ + --role "Managed HSM Crypto User" \ + --scope "/keys/${KEY_NAME}" \ + --output none + +printf '%s\n' "${KEY_ID}" > "${SHARED_DIR}/azure_managed_hsm_key_id" +echo "Managed HSM key is ready for the HyperShift KMS test" diff --git a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.metadata.json b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.metadata.json new file mode 100644 index 0000000000000..6f645c8b9de18 --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.metadata.json @@ -0,0 +1,15 @@ +{ + "path": "hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.yaml", + "owners": { + "approvers": [ + "csrwng", + "enxebre", + "bryan-cox" + ], + "reviewers": [ + "csrwng", + "enxebre", + "bryan-cox" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.yaml b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.yaml new file mode 100644 index 0000000000000..695203df86f61 --- /dev/null +++ b/ci-operator/step-registry/hypershift/azure/aks/managed-hsm/provision/hypershift-azure-aks-managed-hsm-provision-ref.yaml @@ -0,0 +1,26 @@ +ref: + as: hypershift-azure-aks-managed-hsm-provision + cli: latest + commands: hypershift-azure-aks-managed-hsm-provision-commands.sh + credentials: + - mount_path: /etc/hypershift-ci-jobs-azurecreds + name: hypershift-ci-jobs-azurecreds + namespace: test-credentials + env: + - default: "false" + documentation: Whether to provision an ephemeral Managed HSM for Azure KMS testing. + name: HYPERSHIFT_AZURE_MANAGED_HSM + - default: eastus + documentation: Azure location in which to provision the Managed HSM. + name: HYPERSHIFT_AZURE_MANAGED_HSM_LOCATION + from: hypershift-tests + grace_period: 5m0s + resources: + requests: + cpu: 100m + memory: 100Mi + timeout: 45m0s + documentation: |- + Provisions and activates an ephemeral Azure Managed HSM, creates an RSA-HSM + key using the AKS workflow credentials, grants the HyperShift KMS managed + identity access to it, and writes the key ID to SHARED_DIR. diff --git a/ci-operator/step-registry/hypershift/azure/run-e2e/hypershift-azure-run-e2e-commands.sh b/ci-operator/step-registry/hypershift/azure/run-e2e/hypershift-azure-run-e2e-commands.sh index fe9fc4ccf7cf4..5d4a3736f550b 100644 --- a/ci-operator/step-registry/hypershift/azure/run-e2e/hypershift-azure-run-e2e-commands.sh +++ b/ci-operator/step-registry/hypershift/azure/run-e2e/hypershift-azure-run-e2e-commands.sh @@ -19,6 +19,11 @@ AZURE_OIDC_ISSUER_URL="$(<"${AZURE_OIDC_ISSUER_URL_LOCATION}" jq -r .oidcIssuerU AZURE_KMS_INFO_LOCATION="/etc/hypershift-ci-jobs-azurecreds/aks-kms-info.json" AKS_KMS_KEY="$(jq -r '."aks-kms-key"' "${AZURE_KMS_INFO_LOCATION}")" AKS_KMS_CREDENTIALS_SECRET="$(jq -r '."aks-kms-credentials-secret"' "${AZURE_KMS_INFO_LOCATION}")" +HIDE_AZURE_KMS_KEY="false" +if [[ -s "${SHARED_DIR}/azure_managed_hsm_key_id" ]]; then + AKS_KMS_KEY="$(<"${SHARED_DIR}/azure_managed_hsm_key_id")" + HIDE_AZURE_KMS_KEY="true" +fi az --version az login --service-principal -u "${AZURE_AUTH_CLIENT_ID}" -p "${AZURE_AUTH_CLIENT_SECRET}" --tenant "${AZURE_AUTH_TENANT_ID}" --output none @@ -145,6 +150,10 @@ if [[ "${OAUTH_EXTERNAL_OIDC_PROVIDER}" != "" ]]; then esac fi +if [[ "${HIDE_AZURE_KMS_KEY}" == "true" ]]; then + set +x +fi + hack/ci-test-e2e.sh -test.v \ -test.run=${CI_TESTS_RUN:-} \ -test.parallel=20 \ @@ -164,11 +173,16 @@ hack/ci-test-e2e.sh -test.v \ ${MI_ARGS:-} \ ${DP_ARGS:-} \ ${AZURE_MULTI_ARCH_PARAMS:-} \ - --e2e.azure-encryption-key-id=${AKS_KMS_KEY} \ + --e2e.azure-encryption-key-id="${AKS_KMS_KEY}" \ --e2e.azure-kms-credentials-secret-name=${AKS_KMS_CREDENTIALS_SECRET} \ ${MARKETPLACE_IMAGE_PARAMS} \ --e2e.latest-release-image="${OCP_IMAGE_LATEST}" \ ${OAUTH_EXTERNAL_OIDC_PARAM:-} \ --e2e.previous-release-image="${OCP_IMAGE_PREVIOUS}" \ ${ADDITIONAL_PULL_SECRET_PARAMS:-} & -wait $! \ No newline at end of file +e2e_pid=$! + +if [[ "${HIDE_AZURE_KMS_KEY}" == "true" ]]; then + set -x +fi +wait "${e2e_pid}" \ No newline at end of file