From 8e3905f46b6cc3504943aa3b8680f93c240f61c1 Mon Sep 17 00:00:00 2001 From: Hilliary Lipsig Date: Sun, 9 Aug 2026 16:51:50 -0700 Subject: [PATCH 1/6] Add Azure IPI resource garbage collection periodic Add periodic-ipi-deprovision-azure to clean up leaked Azure IPI cluster resources, matching the existing AWS and GCP GC patterns. Uses a 6-hour TTL to avoid disrupting active tests. Co-Authored-By: Claude Opus 4.6 --- .../jobs/infra-build-farm-periodics.yaml | 55 +++++++ .../ipi-deprovision/ipi-deprovision-bc.yaml | 4 +- core-services/ipi-deprovision/azure.sh | 137 ++++++++++++++++++ 3 files changed, 195 insertions(+), 1 deletion(-) create mode 100755 core-services/ipi-deprovision/azure.sh diff --git a/ci-operator/jobs/infra-build-farm-periodics.yaml b/ci-operator/jobs/infra-build-farm-periodics.yaml index 2dd1f4025e4c1..d35539ef74789 100644 --- a/ci-operator/jobs/infra-build-farm-periodics.yaml +++ b/ci-operator/jobs/infra-build-farm-periodics.yaml @@ -440,6 +440,61 @@ periodics: - key: gce.json path: gce.json secretName: cluster-secrets-gcp-openshift-gce-devel-ci-2 +- agent: kubernetes + cluster: build02 + decorate: true + decoration_config: + timeout: 8h0m0s + extra_refs: + - base_ref: main + org: openshift + repo: release + workdir: true + interval: 4h + labels: + ci.openshift.io/role: cloud-azure + pj-rehearse.openshift.io/can-be-rehearsed: "true" + max_concurrency: 1 + name: periodic-ipi-deprovision-azure + reporter_config: + slack: + channel: '#ops-testplatform' + job_states_to_report: + - failure + - error + report_template: '@azure-ci-admin Job *{{.Spec.Job}}* failed. <{{.Status.URL}}|View + logs>' + spec: + containers: + - command: + - ./core-services/ipi-deprovision/azure.sh + env: + - name: HOME + value: /tmp + - name: AZURE_AUTH_LOCATION + value: /azure/osServicePrincipal.json + - name: CLUSTER_TTL + value: 6 hours ago + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ipi-deprovision_latest + imagePullPolicy: Always + name: ipi-deprovision + resources: + requests: + cpu: "1" + memory: 600Mi + volumeMounts: + - mountPath: /azure + name: cluster-secrets-azure4 + nodeSelector: + beta.kubernetes.io/arch: amd64 + serviceAccountName: ipi-deprovisioner + volumes: + - name: cluster-secrets-azure4 + secret: + items: + - key: osServicePrincipal.json + path: osServicePrincipal.json + secretName: cluster-secrets-azure4 - agent: kubernetes cluster: build02 cron: '@daily' diff --git a/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml b/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml index 7a3bc8aebd254..8bb2297a73c81 100644 --- a/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml +++ b/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml @@ -47,7 +47,9 @@ spec: COPY ./openshift-install /usr/bin/openshift-install COPY ./hypershift /usr/bin/hypershift - RUN dnf install --nogpg -y google-cloud-sdk unzip jq less && \ + RUN rpm --import https://packages.microsoft.com/keys/microsoft.asc && \ + echo -e "[azure-cli]\nname=Azure CLI\nbaseurl=https://packages.microsoft.com/yumrepos/azure-cli\nenabled=1\ngpgcheck=1\ngpgkey=https://packages.microsoft.com/keys/microsoft.asc" > /etc/yum.repos.d/azure-cli.repo && \ + dnf install --nogpg -y google-cloud-sdk azure-cli unzip jq less && \ dnf clean all && \ curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ unzip awscliv2.zip && ./aws/install diff --git a/core-services/ipi-deprovision/azure.sh b/core-services/ipi-deprovision/azure.sh new file mode 100755 index 0000000000000..9b02486e0131a --- /dev/null +++ b/core-services/ipi-deprovision/azure.sh @@ -0,0 +1,137 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wait; fi' TERM + +function queue() { + local LIVE="$(jobs | wc -l)" + while [[ "${LIVE}" -ge 10 ]]; do + sleep 1 + LIVE="$(jobs | wc -l)" + done + echo "${@}" + "${@}" & +} + +function deprovision() { + WORKDIR="${1}" + timeout --signal=SIGTERM 30m openshift-install --dir "${WORKDIR}" --log-level error destroy cluster && touch "${WORKDIR}/success" || touch "${WORKDIR}/failure" +} + +logdir="${ARTIFACTS}/deprovision" +mkdir -p "${logdir}" + +AZURE_AUTH_LOCATION="${AZURE_AUTH_LOCATION:-/azure/osServicePrincipal.json}" +export AZURE_AUTH_LOCATION + +# Disable tracing due to credential handling +[[ $- == *x* ]] && WAS_TRACING=true || WAS_TRACING=false +set +x +AZURE_CLIENT_ID="$(jq -r .clientId "${AZURE_AUTH_LOCATION}")" +AZURE_CLIENT_SECRET="$(jq -r .clientSecret "${AZURE_AUTH_LOCATION}")" +AZURE_TENANT_ID="$(jq -r .tenantId "${AZURE_AUTH_LOCATION}")" +AZURE_SUBSCRIPTION_ID="$(jq -r .subscriptionId "${AZURE_AUTH_LOCATION}")" + +az login --service-principal \ + -u "${AZURE_CLIENT_ID}" \ + -p "${AZURE_CLIENT_SECRET}" \ + --tenant "${AZURE_TENANT_ID}" \ + --output none +$WAS_TRACING && set -x + +az account set --subscription "${AZURE_SUBSCRIPTION_ID}" +echo "Azure subscription: ${AZURE_SUBSCRIPTION_ID}" + +azure_rg_age_cutoff="$(date -u --date="${CLUSTER_TTL}" '+%Y-%m-%dT%H:%M:%SZ')" +echo "deprovisioning clusters with resource groups created before ${azure_rg_age_cutoff} ..." + +# List resource groups with creation time via ARM REST API ($expand=createdTime). +rg_json="[]" +next_url="/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourcegroups?api-version=2021-04-01&\$expand=createdTime" +while [[ -n "${next_url}" ]]; do + page="$(az rest --method get --url "${next_url}")" + rg_json="$(jq -s '.[0] + [.[1].value[] | {name: .name, location: .location, created: .createdTime, state: .properties.provisioningState}]' \ + <(echo "${rg_json}") <(echo "${page}"))" + next_url="$(echo "${page}" | jq -r '.nextLink // empty')" +done + +while IFS=$'\t' read -r rg_name rg_location rg_created rg_state; do + [[ -z "${rg_name}" ]] && continue + [[ "${rg_state}" == "Deleting" ]] && continue + + # Match CI-created resource groups: the IPI installer creates resource groups + # named -rg where the infraID starts with "ci-op-" (the CI namespace). + if [[ ! "${rg_name}" =~ ^ci-op- ]]; then + continue + fi + + if [[ "${rg_created}" > "${azure_rg_age_cutoff}" ]]; then + continue + fi + + # Derive the infraID from the resource group name by stripping the -rg suffix. + # If the RG doesn't end in -rg, use the full name as infraID. + if [[ "${rg_name}" =~ ^(.+)-rg$ ]]; then + infraID="${BASH_REMATCH[1]}" + else + infraID="${rg_name}" + fi + + workdir="${logdir}/${infraID}" + mkdir -p "${workdir}" + cat <"${workdir}/metadata.json" +{ + "infraID":"${infraID}", + "azure":{ + "region":"${rg_location}", + "resourceGroupName":"${rg_name}", + "cloudName":"AzurePublicCloud" + } +} +EOF + echo "will deprovision Azure cluster ${infraID} in ${rg_location} (rg: ${rg_name}, created: ${rg_created})" +done < <(echo "${rg_json}" | jq -r '.[] | [.name, .location, .created, .state] | @tsv') + +# log installer version for debugging purposes +openshift-install version + +clusters=$( find "${logdir}" -mindepth 1 -type d ) +for workdir in $(shuf <<< ${clusters}); do + queue deprovision "${workdir}" +done + +if ! wait; then + echo "At least one deprovision job failed or timed out." +fi + +# Force-delete resource groups that openshift-install failed to clean up +for workdir in $(find "${logdir}" -mindepth 1 -type d); do + if [[ -f "${workdir}/failure" ]]; then + rg_name="$(jq -r '.azure.resourceGroupName' "${workdir}/metadata.json")" + echo "openshift-install failed for ${rg_name}, force-deleting resource group ..." + if az group delete --name "${rg_name}" --yes --no-wait; then + echo "Initiated force-deletion of resource group ${rg_name}" + rm "${workdir}/failure" + touch "${workdir}/warning" + else + echo "Failed to force-delete resource group ${rg_name}" + fi + fi +done + +WARNINGS="$(find ${clusters} -name warning -printf '%H\n' | sort)" +if [[ -n "${WARNINGS}" ]]; then + echo "The following clusters required force-deletion of their resource groups:" + xargs --max-args 1 basename <<< $WARNINGS +fi + +FAILED="$(find ${clusters} -name failure -printf '%H\n' | sort)" +if [[ -n "${FAILED}" ]]; then + echo "Deprovision failed on the following clusters:" + xargs --max-args 1 basename <<< $FAILED + exit 1 +fi + +echo "Deprovision finished successfully" From 4d1c4b87ca3ef7a60cd1adf531731dfe41d3f40d Mon Sep 17 00:00:00 2001 From: Hilliary Lipsig Date: Mon, 10 Aug 2026 10:35:51 -0700 Subject: [PATCH 2/6] Wait for Azure resource group deletion before marking success Remove --no-wait from the fallback az group delete so the script only marks a resource group as cleaned up after deletion completes, avoiding false success when async deletion fails. Co-Authored-By: Claude Opus 4.6 --- core-services/ipi-deprovision/azure.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core-services/ipi-deprovision/azure.sh b/core-services/ipi-deprovision/azure.sh index 9b02486e0131a..ce7c8d8034a46 100755 --- a/core-services/ipi-deprovision/azure.sh +++ b/core-services/ipi-deprovision/azure.sh @@ -111,8 +111,8 @@ for workdir in $(find "${logdir}" -mindepth 1 -type d); do if [[ -f "${workdir}/failure" ]]; then rg_name="$(jq -r '.azure.resourceGroupName' "${workdir}/metadata.json")" echo "openshift-install failed for ${rg_name}, force-deleting resource group ..." - if az group delete --name "${rg_name}" --yes --no-wait; then - echo "Initiated force-deletion of resource group ${rg_name}" + if az group delete --name "${rg_name}" --yes; then + echo "Successfully force-deleted resource group ${rg_name}" rm "${workdir}/failure" touch "${workdir}/warning" else From f863ea8be40d2929f4f77a143b763afeeb716fd5 Mon Sep 17 00:00:00 2001 From: Hilliary Lipsig Date: Mon, 10 Aug 2026 10:38:17 -0700 Subject: [PATCH 3/6] Fix empty find path and enable RPM GPG verification Use ${logdir} instead of ${clusters} for marker-file searches to avoid searching cwd when no clusters are found. Enable GPG signature verification for all repos by adding gpgcheck to the Google Cloud SDK repo config, importing both GPG keys, and removing --nogpg from dnf. Co-Authored-By: Claude Opus 4.6 --- clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml | 7 ++++--- core-services/ipi-deprovision/azure.sh | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml b/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml index 8bb2297a73c81..9d8fc4bfe46c9 100644 --- a/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml +++ b/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml @@ -41,15 +41,16 @@ spec: ARG JQ_VERSION=1.6 - RUN echo -e "[google-cloud-sdk]\nname=Google Cloud SDK\nbaseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el8-x86_64\nenabled=1" > /etc/yum.repos.d/google-cloud-sdk.repo + RUN echo -e "[google-cloud-sdk]\nname=Google Cloud SDK\nbaseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el8-x86_64\nenabled=1\ngpgcheck=1\ngpgkey=https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg" > /etc/yum.repos.d/google-cloud-sdk.repo COPY ./oc /usr/bin/oc COPY ./openshift-install /usr/bin/openshift-install COPY ./hypershift /usr/bin/hypershift - RUN rpm --import https://packages.microsoft.com/keys/microsoft.asc && \ + RUN rpm --import https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg && \ + rpm --import https://packages.microsoft.com/keys/microsoft.asc && \ echo -e "[azure-cli]\nname=Azure CLI\nbaseurl=https://packages.microsoft.com/yumrepos/azure-cli\nenabled=1\ngpgcheck=1\ngpgkey=https://packages.microsoft.com/keys/microsoft.asc" > /etc/yum.repos.d/azure-cli.repo && \ - dnf install --nogpg -y google-cloud-sdk azure-cli unzip jq less && \ + dnf install -y google-cloud-sdk azure-cli unzip jq less && \ dnf clean all && \ curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ unzip awscliv2.zip && ./aws/install diff --git a/core-services/ipi-deprovision/azure.sh b/core-services/ipi-deprovision/azure.sh index ce7c8d8034a46..236503efd15ca 100755 --- a/core-services/ipi-deprovision/azure.sh +++ b/core-services/ipi-deprovision/azure.sh @@ -121,13 +121,13 @@ for workdir in $(find "${logdir}" -mindepth 1 -type d); do fi done -WARNINGS="$(find ${clusters} -name warning -printf '%H\n' | sort)" +WARNINGS="$(find "${logdir}" -mindepth 2 -name warning -printf '%H\n' | sort)" if [[ -n "${WARNINGS}" ]]; then echo "The following clusters required force-deletion of their resource groups:" xargs --max-args 1 basename <<< $WARNINGS fi -FAILED="$(find ${clusters} -name failure -printf '%H\n' | sort)" +FAILED="$(find "${logdir}" -mindepth 2 -name failure -printf '%H\n' | sort)" if [[ -n "${FAILED}" ]]; then echo "Deprovision failed on the following clusters:" xargs --max-args 1 basename <<< $FAILED From 1e89d88b990cf771ce0f9a06402e001bb85f4432 Mon Sep 17 00:00:00 2001 From: Hilliary Lipsig Date: Mon, 10 Aug 2026 10:41:53 -0700 Subject: [PATCH 4/6] Run Azure deprovision container as non-root Add securityContext with runAsNonRoot and runAsUser 65534 (nobody). None of the tools (az, openshift-install, jq) require root, and HOME=/tmp ensures config writes go to world-writable paths. Co-Authored-By: Claude Opus 4.6 --- ci-operator/jobs/infra-build-farm-periodics.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci-operator/jobs/infra-build-farm-periodics.yaml b/ci-operator/jobs/infra-build-farm-periodics.yaml index d35539ef74789..9a98471add13d 100644 --- a/ci-operator/jobs/infra-build-farm-periodics.yaml +++ b/ci-operator/jobs/infra-build-farm-periodics.yaml @@ -482,6 +482,9 @@ periodics: requests: cpu: "1" memory: 600Mi + securityContext: + runAsNonRoot: true + runAsUser: 65534 volumeMounts: - mountPath: /azure name: cluster-secrets-azure4 From 2e35dd98162c34313770fa1bc77de93a4e1d2fd5 Mon Sep 17 00:00:00 2001 From: Hilliary Lipsig Date: Mon, 10 Aug 2026 20:41:41 -0700 Subject: [PATCH 5/6] Fix find format directive and add type filter in summary collection The warning and failure marker find commands used %H (starting-point directory) instead of %h (containing directory), causing basename to always print the logdir name rather than individual cluster IDs. Also add -type f to restrict matches to regular files only. Co-Authored-By: Claude Opus 4.6 --- core-services/ipi-deprovision/azure.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core-services/ipi-deprovision/azure.sh b/core-services/ipi-deprovision/azure.sh index 236503efd15ca..0eaff713e056b 100755 --- a/core-services/ipi-deprovision/azure.sh +++ b/core-services/ipi-deprovision/azure.sh @@ -121,13 +121,13 @@ for workdir in $(find "${logdir}" -mindepth 1 -type d); do fi done -WARNINGS="$(find "${logdir}" -mindepth 2 -name warning -printf '%H\n' | sort)" +WARNINGS="$(find "${logdir}" -mindepth 2 -type f -name warning -printf '%h\n' | sort)" if [[ -n "${WARNINGS}" ]]; then echo "The following clusters required force-deletion of their resource groups:" xargs --max-args 1 basename <<< $WARNINGS fi -FAILED="$(find "${logdir}" -mindepth 2 -name failure -printf '%H\n' | sort)" +FAILED="$(find "${logdir}" -mindepth 2 -type f -name failure -printf '%h\n' | sort)" if [[ -n "${FAILED}" ]]; then echo "Deprovision failed on the following clusters:" xargs --max-args 1 basename <<< $FAILED From cdbbfa462a35f0cc2aedd0cfa30254f0936b6227 Mon Sep 17 00:00:00 2001 From: Hilliary Lipsig Date: Mon, 10 Aug 2026 20:44:46 -0700 Subject: [PATCH 6/6] Disable rehearsal for Azure IPI deprovision periodic This job requires real Azure credentials and resources, so it cannot be meaningfully rehearsed in PR context. Marking it as non-rehearsable prevents the rehearsal system from failing on missing build history. Co-Authored-By: Claude Opus 4.6 --- ci-operator/jobs/infra-build-farm-periodics.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-operator/jobs/infra-build-farm-periodics.yaml b/ci-operator/jobs/infra-build-farm-periodics.yaml index 9a98471add13d..fe4fd31b58817 100644 --- a/ci-operator/jobs/infra-build-farm-periodics.yaml +++ b/ci-operator/jobs/infra-build-farm-periodics.yaml @@ -453,7 +453,7 @@ periodics: interval: 4h labels: ci.openshift.io/role: cloud-azure - pj-rehearse.openshift.io/can-be-rehearsed: "true" + pj-rehearse.openshift.io/can-be-rehearsed: "false" max_concurrency: 1 name: periodic-ipi-deprovision-azure reporter_config: