Skip to content

Commit a9286fd

Browse files
zanejohnson-azurezanejohnson-azure
authored
deploy new image to prod clusters using helm chart
1 parent 60df738 commit a9286fd

File tree

9 files changed

+3903
-0
lines changed

9 files changed

+3903
-0
lines changed

‎.pipelines/ci-aks-prod-release.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -475,3 +475,39 @@ extends:
475475
displayName: Ev2 - Monitoring
476476
inputs:
477477
Ev2MonintoringUrl: $(Ev2MonintoringUrl)
478+
479+
480+
# =============================================================================
481+
# Stage 3: Deploy ama-logs to AKS Clusters via Helm
482+
# =============================================================================
483+
# To add a new cluster, simply add an entry to the template list below.
484+
# Each cluster only needs: clusterName, resourceGroup, region, subscriptionId, workspaceId, and imageTag.
485+
# =============================================================================
486+
- stage: Stage_3
487+
displayName: Deploy ama-logs to CI AKS Prod Clusters via Helm
488+
dependsOn: Stage_2
489+
pool:
490+
name: Azure-Pipelines-CI-Test-EO
491+
image: ci-1es-managed-ubuntu-2204
492+
os: linux
493+
jobs:
494+
# Monitoring-Model-Cluster-WCUS
495+
- template: .pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml@self
496+
parameters:
497+
clusterName: 'Monitoring-Model-Cluster-WCUS'
498+
resourceGroup: 'monitoring-model-cluster-wcus'
499+
region: 'westcentralus'
500+
subscriptionId: '9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb'
501+
workspaceId: '22f38e11-4f59-480c-b4b8-2573156b6e06' # "Monitoring-Model-Cluster-WCUS"
502+
imageTag: '$(AgentImageTagSuffix)'
503+
environment: 'CI-Deploy-To-Prod-Cluster-1'
504+
# Monitoring-Model-Cluster-WEU
505+
- template: .pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml@self
506+
parameters:
507+
clusterName: 'Monitoring-Model-Cluster-WEU'
508+
resourceGroup: 'monitoring-model-cluster-weu'
509+
region: 'westeurope'
510+
subscriptionId: '9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb'
511+
workspaceId: '5c269467-32a9-4468-a1d6-ec1cac551e74' # "Monitoring-Model-Cluster-WEU"
512+
imageTag: '$(AgentImageTagSuffix)'
513+
environment: 'CI-Deploy-To-Prod-Cluster-2'

‎.pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml‎

Lines changed: 258 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,258 @@
1+
# Template for deploying ama-logs to an AKS cluster via Helm
2+
# Usage:
3+
# - template: helm-deploy-templates/ama-logs-helm-deploy.yaml
4+
# parameters:
5+
# clusterName: 'my-cluster'
6+
# resourceGroup: 'my-rg'
7+
# region: 'eastus'
8+
# subscriptionId: '9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb'
9+
# workspaceId: 'your-workspace-id'
10+
# imageTag: '$(AgentImageTagSuffix)' # e.g., 3.1.32
11+
# environment: 'deployment environment'
12+
13+
parameters:
14+
# Required cluster parameters
15+
- name: clusterName
16+
type: string
17+
- name: resourceGroup
18+
type: string
19+
- name: region
20+
type: string
21+
- name: subscriptionId
22+
type: string
23+
- name: workspaceId
24+
type: string
25+
displayName: 'Log Analytics Workspace ID'
26+
- name: imageTag
27+
type: string
28+
displayName: 'Image tag suffix (e.g., 3.1.32)'
29+
- name: environment
30+
type: string
31+
displayName: 'Azure DevOps Environment name'
32+
33+
# Optional parameters with defaults
34+
- name: cloudEnvironment
35+
type: string
36+
default: 'azurepubliccloud'
37+
- name: kubernetesVersion
38+
type: string
39+
default: '1.32.7'
40+
- name: azureSubscription
41+
type: string
42+
default: 'ContainerInsights_Build_Subscription_CI'
43+
- name: namespace
44+
type: string
45+
default: 'default'
46+
- name: releaseName
47+
type: string
48+
default: 'azuremonitor-containers'
49+
- name: helmVersion
50+
type: string
51+
default: '3.12.3'
52+
- name: dependsOn
53+
type: object
54+
default: []
55+
56+
jobs:
57+
- deployment: Deploy_AmaLogs_${{ replace(parameters.clusterName, '-', '_') }}
58+
displayName: 'Deploy: ama-logs to ${{ parameters.clusterName }}'
59+
environment: ${{ parameters.environment }}
60+
dependsOn: ${{ parameters.dependsOn }}
61+
variables:
62+
- name: OneESPT
63+
value: true
64+
readonly: true
65+
- name: OneESPT.BuildType
66+
value: Official
67+
readonly: true
68+
- name: OneESPT.OS
69+
value: linux
70+
readonly: true
71+
- name: skipComponentGovernanceDetection
72+
value: true
73+
- name: Codeql.SkipTaskAutoInjection
74+
value: true
75+
- name: AKS_RESOURCE_ID
76+
value: '/subscriptions/${{ parameters.subscriptionId }}/resourceGroups/${{ parameters.resourceGroup }}/providers/Microsoft.ContainerService/managedClusters/${{ parameters.clusterName }}'
77+
strategy:
78+
runOnce:
79+
deploy:
80+
steps:
81+
- checkout: self
82+
displayName: Checkout repository
83+
- task: HelmInstaller@1
84+
displayName: Install Helm
85+
inputs:
86+
helmVersionToInstall: '${{ parameters.helmVersion }}'
87+
- task: HelmDeploy@0
88+
displayName: 'Helm Deploy: ama-logs to ${{ parameters.clusterName }}'
89+
inputs:
90+
connectionType: 'Azure Resource Manager'
91+
azureSubscription: '${{ parameters.azureSubscription }}'
92+
azureResourceGroup: '${{ parameters.resourceGroup }}'
93+
kubernetesCluster: '${{ parameters.clusterName }}'
94+
useClusterAdmin: true
95+
namespace: '${{ parameters.namespace }}'
96+
command: 'upgrade'
97+
chartType: 'FilePath'
98+
# The following chart is copied over from feature branch https://github.com/microsoft/Docker-Provider/tree/longw/addon-to-extension-quick-Test
99+
# TODO: When it is merged to ci_prod, the following chartPath will be updated accordingly.
100+
chartPath: '$(Build.SourcesDirectory)/charts/azuremonitor-containerinsights-for-prod-clusters'
101+
releaseName: '${{ parameters.releaseName }}'
102+
overrideValues: 'global.commonGlobals.CloudEnvironment=${{ parameters.cloudEnvironment }},global.commonGlobals.Region=${{ parameters.region }},OmsAgent.aksResourceID=$(AKS_RESOURCE_ID),OmsAgent.workspaceID=${{ parameters.workspaceId }},OmsAgent.imageTagLinux=${{ parameters.imageTag }},OmsAgent.imageTagWindows=win-${{ parameters.imageTag }}'
103+
waitForExecution: false
104+
arguments: '--timeout 10m --install'
105+
- task: AzureCLI@2
106+
displayName: Verify Deployment
107+
inputs:
108+
azureSubscription: '${{ parameters.azureSubscription }}'
109+
scriptType: bash
110+
scriptLocation: inlineScript
111+
inlineScript: |
112+
echo "=========================================="
113+
echo "Deployment Summary"
114+
echo "=========================================="
115+
echo "Cluster: ${{ parameters.clusterName }}"
116+
echo "Resource Group: ${{ parameters.resourceGroup }}"
117+
echo "Region: ${{ parameters.region }}"
118+
echo "Linux Image Tag: ${{ parameters.imageTag }}"
119+
echo "Windows Image Tag: win-${{ parameters.imageTag }}"
120+
echo ""
121+
122+
echo "Getting AKS credentials..."
123+
az aks get-credentials --resource-group ${{ parameters.resourceGroup }} --name ${{ parameters.clusterName }} --overwrite-existing --admin
124+
125+
echo ""
126+
echo "=========================================="
127+
echo "Waiting for pods to be ready (60s max)..."
128+
echo "=========================================="
129+
130+
# Wait for daemonset pods (pods are always deployed to kube-system namespace)
131+
kubectl rollout status daemonset/ama-logs -n kube-system --timeout=60s || echo "Warning: Daemonset rollout not complete within timeout"
132+
133+
# Wait for replicaset pod
134+
kubectl rollout status deployment/ama-logs-rs -n kube-system --timeout=60s 2>/dev/null || \
135+
kubectl rollout status replicaset -l rsName=ama-logs-rs -n kube-system --timeout=60s 2>/dev/null || \
136+
echo "Warning: ReplicaSet rollout status check skipped"
137+
138+
echo ""
139+
echo "=========================================="
140+
echo "Pod Status"
141+
echo "=========================================="
142+
kubectl get pods -n kube-system | grep ama-logs || echo "No ama-logs pods found"
143+
144+
echo ""
145+
echo "=========================================="
146+
echo "Helm Release Status"
147+
echo "=========================================="
148+
helm status ${{ parameters.releaseName }} -n ${{ parameters.namespace }} 2>/dev/null || echo "Helm release status not available"
149+
150+
echo ""
151+
echo "=========================================="
152+
echo "Image Verification"
153+
echo "=========================================="
154+
EXPECTED_LINUX_TAG="${{ parameters.imageTag }}"
155+
EXPECTED_WINDOWS_TAG="win-${{ parameters.imageTag }}"
156+
VERIFICATION_PASSED=true
157+
158+
echo "Expected Linux image tag: $EXPECTED_LINUX_TAG"
159+
echo "Expected Windows image tag: $EXPECTED_WINDOWS_TAG"
160+
echo ""
161+
162+
# ---- 1. Linux DaemonSet Pod Verification (ama-logs + ama-logs-prometheus containers) ----
163+
# Note: Pods are always deployed to kube-system namespace regardless of helm release namespace
164+
echo "--- 1. Linux DaemonSet Verification ---"
165+
166+
# Check ama-logs container
167+
LINUX_AMA_LOGS_IMAGE=$(kubectl get pods -n kube-system -l component=ama-logs-agent -o jsonpath='{.items[0].spec.containers[?(@.name=="ama-logs")].image}' 2>/dev/null)
168+
169+
if [ -z "$LINUX_AMA_LOGS_IMAGE" ]; then
170+
echo "❌ ERROR: Could not retrieve ama-logs container image from Linux DaemonSet"
171+
VERIFICATION_PASSED=false
172+
else
173+
echo "ama-logs container image: $LINUX_AMA_LOGS_IMAGE"
174+
LINUX_AMA_LOGS_TAG=$(echo "$LINUX_AMA_LOGS_IMAGE" | cut -d':' -f2)
175+
if [ "$LINUX_AMA_LOGS_TAG" == "$EXPECTED_LINUX_TAG" ]; then
176+
echo "✅ ama-logs container: PASSED"
177+
else
178+
echo "❌ ama-logs container MISMATCH! Expected: $EXPECTED_LINUX_TAG, Actual: $LINUX_AMA_LOGS_TAG"
179+
VERIFICATION_PASSED=false
180+
fi
181+
fi
182+
183+
# Check ama-logs-prometheus container
184+
LINUX_PROM_IMAGE=$(kubectl get pods -n kube-system -l component=ama-logs-agent -o jsonpath='{.items[0].spec.containers[?(@.name=="ama-logs-prometheus")].image}' 2>/dev/null)
185+
186+
if [ -z "$LINUX_PROM_IMAGE" ]; then
187+
echo "❌ ERROR: Could not retrieve ama-logs-prometheus container image from Linux DaemonSet"
188+
VERIFICATION_PASSED=false
189+
else
190+
echo "ama-logs-prometheus container image: $LINUX_PROM_IMAGE"
191+
LINUX_PROM_TAG=$(echo "$LINUX_PROM_IMAGE" | cut -d':' -f2)
192+
if [ "$LINUX_PROM_TAG" == "$EXPECTED_LINUX_TAG" ]; then
193+
echo "✅ ama-logs-prometheus container: PASSED"
194+
else
195+
echo "❌ ama-logs-prometheus container MISMATCH! Expected: $EXPECTED_LINUX_TAG, Actual: $LINUX_PROM_TAG"
196+
VERIFICATION_PASSED=false
197+
fi
198+
fi
199+
200+
echo ""
201+
202+
# ---- 2. ReplicaSet Pod Verification (ama-logs container) ----
203+
echo "--- 2. ReplicaSet Verification ---"
204+
205+
RS_AMA_LOGS_IMAGE=$(kubectl get pods -n kube-system -l rsName=ama-logs-rs -o jsonpath='{.items[0].spec.containers[?(@.name=="ama-logs")].image}' 2>/dev/null)
206+
207+
if [ -z "$RS_AMA_LOGS_IMAGE" ]; then
208+
echo "❌ ERROR: Could not retrieve ama-logs container image from ReplicaSet"
209+
VERIFICATION_PASSED=false
210+
else
211+
echo "ama-logs container image: $RS_AMA_LOGS_IMAGE"
212+
RS_AMA_LOGS_TAG=$(echo "$RS_AMA_LOGS_IMAGE" | cut -d':' -f2)
213+
if [ "$RS_AMA_LOGS_TAG" == "$EXPECTED_LINUX_TAG" ]; then
214+
echo "✅ ReplicaSet ama-logs container: PASSED"
215+
else
216+
echo "❌ ReplicaSet ama-logs container MISMATCH! Expected: $EXPECTED_LINUX_TAG, Actual: $RS_AMA_LOGS_TAG"
217+
VERIFICATION_PASSED=false
218+
fi
219+
fi
220+
221+
echo ""
222+
223+
# ---- 3. Windows DaemonSet Pod Verification (ama-logs-windows container) ----
224+
echo "--- 3. Windows DaemonSet Verification ---"
225+
226+
WINDOWS_POD_EXISTS=$(kubectl get pods -n kube-system -l component=ama-logs-agent-windows -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
227+
228+
if [ -z "$WINDOWS_POD_EXISTS" ]; then
229+
echo "INFO: No Windows ama-logs pods found (cluster may not have Windows nodes)"
230+
echo "Skipping Windows verification"
231+
else
232+
WINDOWS_IMAGE=$(kubectl get pods -n kube-system -l component=ama-logs-agent-windows -o jsonpath='{.items[0].spec.containers[?(@.name=="ama-logs-windows")].image}' 2>/dev/null)
233+
234+
if [ -z "$WINDOWS_IMAGE" ]; then
235+
echo "❌ ERROR: Could not retrieve ama-logs-windows container image"
236+
VERIFICATION_PASSED=false
237+
else
238+
echo "ama-logs-windows container image: $WINDOWS_IMAGE"
239+
WINDOWS_TAG=$(echo "$WINDOWS_IMAGE" | cut -d':' -f2)
240+
if [ "$WINDOWS_TAG" == "$EXPECTED_WINDOWS_TAG" ]; then
241+
echo "✅ Windows ama-logs-windows container: PASSED"
242+
else
243+
echo "❌ Windows ama-logs-windows container MISMATCH! Expected: $EXPECTED_WINDOWS_TAG, Actual: $WINDOWS_TAG"
244+
VERIFICATION_PASSED=false
245+
fi
246+
fi
247+
fi
248+
249+
echo ""
250+
echo "=========================================="
251+
echo "Final Verification Result"
252+
echo "=========================================="
253+
if [ "$VERIFICATION_PASSED" = true ]; then
254+
echo "✅ SUCCESS: All image tag verifications PASSED"
255+
else
256+
echo "❌ FAILED: One or more image tag verifications failed"
257+
exit 1
258+
fi

‎charts/azuremonitor-containerinsights-for-prod-clusters/Chart.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
apiVersion: v2
2+
description: azure-monitor-containers helm chart
3+
name: azuremonitor-containers
4+
version: 3.2.1-dev-test

0 commit comments

Comments
 (0)