diff --git a/ci-operator/jobs/infra-build-farm-periodics.yaml b/ci-operator/jobs/infra-build-farm-periodics.yaml index 2dd1f4025e4c1..fe4fd31b58817 100644 --- a/ci-operator/jobs/infra-build-farm-periodics.yaml +++ b/ci-operator/jobs/infra-build-farm-periodics.yaml @@ -440,6 +440,64 @@ periodics: - key: gce.json path: gce.json secretName: cluster-secrets-gcp-openshift-gce-devel-ci-2 +- agent: kubernetes + cluster: build02 + decorate: true + decoration_config: + timeout: 8h0m0s + extra_refs: + - base_ref: main + org: openshift + repo: release + workdir: true + interval: 4h + labels: + ci.openshift.io/role: cloud-azure + pj-rehearse.openshift.io/can-be-rehearsed: "false" + max_concurrency: 1 + name: periodic-ipi-deprovision-azure + reporter_config: + slack: + channel: '#ops-testplatform' + job_states_to_report: + - failure + - error + report_template: '@azure-ci-admin Job *{{.Spec.Job}}* failed. <{{.Status.URL}}|View + logs>' + spec: + containers: + - command: + - ./core-services/ipi-deprovision/azure.sh + env: + - name: HOME + value: /tmp + - name: AZURE_AUTH_LOCATION + value: /azure/osServicePrincipal.json + - name: CLUSTER_TTL + value: 6 hours ago + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ipi-deprovision_latest + imagePullPolicy: Always + name: ipi-deprovision + resources: + requests: + cpu: "1" + memory: 600Mi + securityContext: + runAsNonRoot: true + runAsUser: 65534 + volumeMounts: + - mountPath: /azure + name: cluster-secrets-azure4 + nodeSelector: + beta.kubernetes.io/arch: amd64 + serviceAccountName: ipi-deprovisioner + volumes: + - name: cluster-secrets-azure4 + secret: + items: + - key: osServicePrincipal.json + path: osServicePrincipal.json + secretName: cluster-secrets-azure4 - agent: kubernetes cluster: build02 cron: '@daily' diff --git a/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml b/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml index 7a3bc8aebd254..9d8fc4bfe46c9 100644 --- a/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml +++ b/clusters/app.ci/ipi-deprovision/ipi-deprovision-bc.yaml @@ -41,13 +41,16 @@ spec: ARG JQ_VERSION=1.6 - RUN echo -e "[google-cloud-sdk]\nname=Google Cloud SDK\nbaseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el8-x86_64\nenabled=1" > /etc/yum.repos.d/google-cloud-sdk.repo + RUN echo -e "[google-cloud-sdk]\nname=Google Cloud SDK\nbaseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el8-x86_64\nenabled=1\ngpgcheck=1\ngpgkey=https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg" > /etc/yum.repos.d/google-cloud-sdk.repo COPY ./oc /usr/bin/oc COPY ./openshift-install /usr/bin/openshift-install COPY ./hypershift /usr/bin/hypershift - RUN dnf install --nogpg -y google-cloud-sdk unzip jq less && \ + RUN rpm --import https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg && \ + rpm --import https://packages.microsoft.com/keys/microsoft.asc && \ + echo -e "[azure-cli]\nname=Azure CLI\nbaseurl=https://packages.microsoft.com/yumrepos/azure-cli\nenabled=1\ngpgcheck=1\ngpgkey=https://packages.microsoft.com/keys/microsoft.asc" > /etc/yum.repos.d/azure-cli.repo && \ + dnf install -y google-cloud-sdk azure-cli unzip jq less && \ dnf clean all && \ curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ unzip awscliv2.zip && ./aws/install diff --git a/core-services/ipi-deprovision/azure.sh b/core-services/ipi-deprovision/azure.sh new file mode 100755 index 0000000000000..0eaff713e056b --- /dev/null +++ b/core-services/ipi-deprovision/azure.sh @@ -0,0 +1,137 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wait; fi' TERM + +function queue() { + local LIVE="$(jobs | wc -l)" + while [[ "${LIVE}" -ge 10 ]]; do + sleep 1 + LIVE="$(jobs | wc -l)" + done + echo "${@}" + "${@}" & +} + +function deprovision() { + WORKDIR="${1}" + timeout --signal=SIGTERM 30m openshift-install --dir "${WORKDIR}" --log-level error destroy cluster && touch "${WORKDIR}/success" || touch "${WORKDIR}/failure" +} + +logdir="${ARTIFACTS}/deprovision" +mkdir -p "${logdir}" + +AZURE_AUTH_LOCATION="${AZURE_AUTH_LOCATION:-/azure/osServicePrincipal.json}" +export AZURE_AUTH_LOCATION + +# Disable tracing due to credential handling +[[ $- == *x* ]] && WAS_TRACING=true || WAS_TRACING=false +set +x +AZURE_CLIENT_ID="$(jq -r .clientId "${AZURE_AUTH_LOCATION}")" +AZURE_CLIENT_SECRET="$(jq -r .clientSecret "${AZURE_AUTH_LOCATION}")" +AZURE_TENANT_ID="$(jq -r .tenantId "${AZURE_AUTH_LOCATION}")" +AZURE_SUBSCRIPTION_ID="$(jq -r .subscriptionId "${AZURE_AUTH_LOCATION}")" + +az login --service-principal \ + -u "${AZURE_CLIENT_ID}" \ + -p "${AZURE_CLIENT_SECRET}" \ + --tenant "${AZURE_TENANT_ID}" \ + --output none +$WAS_TRACING && set -x + +az account set --subscription "${AZURE_SUBSCRIPTION_ID}" +echo "Azure subscription: ${AZURE_SUBSCRIPTION_ID}" + +azure_rg_age_cutoff="$(date -u --date="${CLUSTER_TTL}" '+%Y-%m-%dT%H:%M:%SZ')" +echo "deprovisioning clusters with resource groups created before ${azure_rg_age_cutoff} ..." + +# List resource groups with creation time via ARM REST API ($expand=createdTime). +rg_json="[]" +next_url="/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourcegroups?api-version=2021-04-01&\$expand=createdTime" +while [[ -n "${next_url}" ]]; do + page="$(az rest --method get --url "${next_url}")" + rg_json="$(jq -s '.[0] + [.[1].value[] | {name: .name, location: .location, created: .createdTime, state: .properties.provisioningState}]' \ + <(echo "${rg_json}") <(echo "${page}"))" + next_url="$(echo "${page}" | jq -r '.nextLink // empty')" +done + +while IFS=$'\t' read -r rg_name rg_location rg_created rg_state; do + [[ -z "${rg_name}" ]] && continue + [[ "${rg_state}" == "Deleting" ]] && continue + + # Match CI-created resource groups: the IPI installer creates resource groups + # named -rg where the infraID starts with "ci-op-" (the CI namespace). + if [[ ! "${rg_name}" =~ ^ci-op- ]]; then + continue + fi + + if [[ "${rg_created}" > "${azure_rg_age_cutoff}" ]]; then + continue + fi + + # Derive the infraID from the resource group name by stripping the -rg suffix. + # If the RG doesn't end in -rg, use the full name as infraID. + if [[ "${rg_name}" =~ ^(.+)-rg$ ]]; then + infraID="${BASH_REMATCH[1]}" + else + infraID="${rg_name}" + fi + + workdir="${logdir}/${infraID}" + mkdir -p "${workdir}" + cat <"${workdir}/metadata.json" +{ + "infraID":"${infraID}", + "azure":{ + "region":"${rg_location}", + "resourceGroupName":"${rg_name}", + "cloudName":"AzurePublicCloud" + } +} +EOF + echo "will deprovision Azure cluster ${infraID} in ${rg_location} (rg: ${rg_name}, created: ${rg_created})" +done < <(echo "${rg_json}" | jq -r '.[] | [.name, .location, .created, .state] | @tsv') + +# log installer version for debugging purposes +openshift-install version + +clusters=$( find "${logdir}" -mindepth 1 -type d ) +for workdir in $(shuf <<< ${clusters}); do + queue deprovision "${workdir}" +done + +if ! wait; then + echo "At least one deprovision job failed or timed out." +fi + +# Force-delete resource groups that openshift-install failed to clean up +for workdir in $(find "${logdir}" -mindepth 1 -type d); do + if [[ -f "${workdir}/failure" ]]; then + rg_name="$(jq -r '.azure.resourceGroupName' "${workdir}/metadata.json")" + echo "openshift-install failed for ${rg_name}, force-deleting resource group ..." + if az group delete --name "${rg_name}" --yes; then + echo "Successfully force-deleted resource group ${rg_name}" + rm "${workdir}/failure" + touch "${workdir}/warning" + else + echo "Failed to force-delete resource group ${rg_name}" + fi + fi +done + +WARNINGS="$(find "${logdir}" -mindepth 2 -type f -name warning -printf '%h\n' | sort)" +if [[ -n "${WARNINGS}" ]]; then + echo "The following clusters required force-deletion of their resource groups:" + xargs --max-args 1 basename <<< $WARNINGS +fi + +FAILED="$(find "${logdir}" -mindepth 2 -type f -name failure -printf '%h\n' | sort)" +if [[ -n "${FAILED}" ]]; then + echo "Deprovision failed on the following clusters:" + xargs --max-args 1 basename <<< $FAILED + exit 1 +fi + +echo "Deprovision finished successfully"