diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.AksPipeline.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.AksPipeline.cs index 606988be4ce..3e6db1a9474 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.AksPipeline.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.AksPipeline.cs @@ -7,12 +7,15 @@ #pragma warning disable ASPIREFILESYSTEM001 // IFileSystemService/TempDirectory are experimental using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; using System.Text.RegularExpressions; using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Dcp.Process; using Aspire.Hosting.Kubernetes; using Aspire.Hosting.Kubernetes.Resources; using Aspire.Hosting.Pipelines; +using Azure.Core; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -208,6 +211,27 @@ private static AksNodePoolResource FindNodePoolResource( /// subsequent Helm and kubectl commands target the AKS cluster. /// private async Task GetAksCredentialsAsync(PipelineStepContext context) + { + // Get the actual provisioned cluster name from the Bicep output. + // The Azure.Provisioning SDK may add a unique suffix to the name + // (e.g., take('aks-${uniqueString(resourceGroup().id)}', 63)). + var clusterName = await NameOutputReference.GetValueAsync(context.CancellationToken).ConfigureAwait(false) + ?? Name; + + await GetAksCredentialsAsync( + context, + clusterName, + savedResourceGroup: null, + subscriptionId: null, + verifyClusterExists: false).ConfigureAwait(false); + } + + private async Task GetAksCredentialsAsync( + PipelineStepContext context, + string clusterName, + string? savedResourceGroup, + string? subscriptionId, + bool verifyClusterExists) { var getCredsTask = await context.ReportingStep.CreateTaskAsync( $"Fetching AKS credentials for {Name}", @@ -217,28 +241,24 @@ private async Task GetAksCredentialsAsync(PipelineStepContext context) { try { - // Get the actual provisioned cluster name from the Bicep output. - // The Azure.Provisioning SDK may add a unique suffix to the name - // (e.g., take('aks-${uniqueString(resourceGroup().id)}', 63)). - var clusterName = await NameOutputReference.GetValueAsync(context.CancellationToken).ConfigureAwait(false) - ?? Name; - var azPath = (AzCliPathResolverForTesting ?? FindAzCli)(); // Defense-in-depth: validate that values used as CLI arguments // contain only expected characters (alphanumeric, hyphens, underscores, dots). ValidateAzureResourceName(clusterName, "cluster name"); - // Resolve the scope this cluster actually lives in before touching the CLI. A cluster - // adopted via AsExistingInResourceGroup(...) can sit outside the app's own - // subscription/resource group, and the provisioner already targets that scope. - var (scopedSubscription, scopedResourceGroup) = GetExplicitScopeValues(); - - var (subscriptionId, savedResourceGroup) = await ResolveDeploymentScopeAsync( - scopedSubscription, - scopedResourceGroup, - context.Services, - context.CancellationToken).ConfigureAwait(false); + if (subscriptionId is null) + { + // Resolve the scope this cluster actually lives in before touching the CLI. A cluster + // adopted via AsExistingInResourceGroup(...) can sit outside the app's own + // subscription/resource group, and the provisioner already targets that scope. + var (scopedSubscription, scopedResourceGroup) = GetExplicitScopeValues(); + (subscriptionId, savedResourceGroup) = await ResolveDeploymentScopeAsync( + scopedSubscription, + scopedResourceGroup, + context.Services, + context.CancellationToken).ConfigureAwait(false); + } ValidateAzureResourceName(subscriptionId, "subscription ID"); @@ -256,6 +276,28 @@ Task RunAzAsync(string path, string arguments) ValidateAzureResourceName(resourceGroup, "resource group"); + KubernetesEnvironment.SkipDestroyCleanup = false; + if (verifyClusterExists && + !await AksResourceExistsAsync( + azPath, + subscriptionId, + resourceGroup, + clusterName, + RunAzAsync).ConfigureAwait(false)) + { + KubernetesEnvironment.KubeConfigPath = null; + KubernetesEnvironment.SkipDestroyCleanup = true; + + context.Logger.LogInformation( + "AKS cluster '{ClusterName}' no longer exists in resource group '{ResourceGroup}'. Skipping cluster cleanup.", + clusterName, + resourceGroup); + await getCredsTask.SucceedAsync( + $"AKS cluster {clusterName} no longer exists; cluster cleanup will be skipped", + context.CancellationToken).ConfigureAwait(false); + return; + } + // Fetch kubeconfig content to stdout using --file - to avoid az CLI // writing credentials with potentially permissive file permissions. // We then write the content ourselves to a temp file with controlled access. @@ -319,6 +361,133 @@ await getCredsTask.FailAsync( } } + private async Task GetAksCredentialsForDestroyAsync(PipelineStepContext context) + { + var deploymentStateManager = context.Services.GetRequiredService(); + var deploymentStateSection = await deploymentStateManager + .AcquireSectionAsync($"Azure:Deployments:{Name}", context.CancellationToken) + .ConfigureAwait(false); + + if (deploymentStateSection.Data.Count == 0) + { + throw new InvalidOperationException( + $"No Azure deployment state was found for AKS environment '{Name}'. " + + "Cluster cleanup cannot run without an isolated kubeconfig."); + } + + // Azure deployment outputs are persisted as a JSON string with the ARM output shape: + // { + // "id": { "type": "String", "value": "/subscriptions/.../managedClusters/aks-abc123" }, + // "name": { "type": "String", "value": "aks-abc123" } + // } + // Read it directly because the provisioning step that normally populates Outputs is not + // part of a fresh destroy process. + ResourceIdentifier? clusterResourceId; + try + { + clusterResourceId = GetPersistedAksResourceId(deploymentStateSection.Data); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + throw new InvalidOperationException( + $"The Azure deployment state for AKS environment '{Name}' contains invalid outputs.", + ex); + } + + if (clusterResourceId is null) + { + throw new InvalidOperationException( + $"The Azure deployment state for AKS environment '{Name}' does not contain the deployed cluster identity."); + } + + // Scope is persisted as a JSON string using the same shape produced by + // BicepUtilities.SetScopeAsync: + // { "resourceGroup": "shared-rg", "subscription": "00000000-..." } + // A missing property means the resource did not pin that scope value, so only that value + // falls back to global Azure deployment state. Older state without Scope uses the persisted + // resource ID so a changed AppHost scope cannot redirect cleanup to another cluster or wait + // on provisioning that is not part of the destroy graph. + string subscriptionId; + string? resourceGroupName; + if (deploymentStateSection.Data["Scope"] is not null) + { + string? scopedSubscription; + string? scopedResourceGroup; + try + { + var scopeJson = deploymentStateSection.Data["Scope"]!.GetValue(); + var scope = JsonNode.Parse(scopeJson)?.AsObject() + ?? throw new InvalidOperationException("The persisted scope is not a JSON object."); + scopedSubscription = scope["subscription"]?.GetValue(); + scopedResourceGroup = scope["resourceGroup"]?.GetValue(); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + throw new InvalidOperationException( + $"The Azure deployment state for AKS environment '{Name}' contains an invalid scope.", + ex); + } + + (subscriptionId, resourceGroupName) = await ResolveDeploymentScopeAsync( + scopedSubscription, + scopedResourceGroup, + context.Services, + context.CancellationToken).ConfigureAwait(false); + } + else + { + subscriptionId = clusterResourceId.SubscriptionId!; + resourceGroupName = clusterResourceId.ResourceGroupName!; + } + + await GetAksCredentialsAsync( + context, + clusterResourceId.Name, + resourceGroupName, + subscriptionId, + verifyClusterExists: true).ConfigureAwait(false); + } + + private static bool HasPersistedAksIdentity(JsonObject deploymentState) + { + try + { + return GetPersistedAksResourceId(deploymentState) is not null; + } + catch (Exception ex) when (ex is JsonException or InvalidOperationException or FormatException) + { + // Malformed or partial state must not prevent the aggregate Azure destroy path from + // deleting the containing resource group. A directly targeted Kubernetes cleanup still + // invokes the credential step, which reports the invalid state rather than using ambient + // Kubernetes credentials. + return false; + } + } + + private static ResourceIdentifier? GetPersistedAksResourceId(JsonObject deploymentState) + { + var outputsJson = deploymentState["Outputs"]?.GetValue(); + if (string.IsNullOrEmpty(outputsJson)) + { + return null; + } + + var resourceId = JsonNode.Parse(outputsJson)?["id"]?["value"]?.GetValue(); + if (!ResourceIdentifier.TryParse(resourceId, out var parsedResourceId) || + parsedResourceId is null || + string.IsNullOrEmpty(parsedResourceId.SubscriptionId) || + string.IsNullOrEmpty(parsedResourceId.ResourceGroupName) || + !string.Equals( + parsedResourceId.ResourceType.ToString(), + "Microsoft.ContainerService/managedClusters", + StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + return parsedResourceId; + } + /// /// Applies the AGC ApplicationLoadBalancer custom resource for the supplied /// into the cluster. Polls the @@ -795,6 +964,35 @@ internal static async Task FetchKubeConfigAsync( return result.StandardOutput; } + internal static async Task AksResourceExistsAsync( + string azPath, + string subscriptionId, + string resourceGroup, + string clusterName, + Func> runAzCommandAsync) + { + var result = await runAzCommandAsync( + azPath, + BuildAksResourceExistsArguments(subscriptionId, resourceGroup, clusterName)).ConfigureAwait(false); + + if (result.ExitCode != 0) + { + // Azure CLI reports an out-of-band deleted resource group as: + // (ResourceGroupNotFound) Resource group 'deployment-rg' could not be found. + // This proves the persisted AKS resource is absent, so cluster cleanup can be skipped. + if (result.StandardError.Contains("(ResourceGroupNotFound)", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + throw new InvalidOperationException( + $"az resource list failed while checking AKS cluster existence " + + $"(exit code {result.ExitCode}): {result.StandardError}"); + } + + return !string.IsNullOrWhiteSpace(result.StandardOutput); + } + internal static string BuildGetCredentialsArguments( string subscriptionId, string resourceGroup, @@ -804,6 +1002,13 @@ internal static string BuildGetCredentialsArguments( internal static string BuildResourceGroupQueryArguments(string subscriptionId, string clusterName) => $"resource list --resource-type Microsoft.ContainerService/managedClusters --name \"{clusterName}\" --query [].resourceGroup -o tsv --subscription \"{subscriptionId}\""; + internal static string BuildAksResourceExistsArguments( + string subscriptionId, + string resourceGroup, + string clusterName) + => $"resource list --resource-group \"{resourceGroup}\" --resource-type Microsoft.ContainerService/managedClusters " + + $"--name \"{clusterName}\" --query [0].id -o tsv --subscription \"{subscriptionId}\""; + /// /// Runs an az CLI command using the shared ProcessSpec/ProcessUtil infrastructure. /// Returns the captured stdout, stderr, and exit code. diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs index f0e545326c2..ad2029fc09f 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs @@ -3,11 +3,15 @@ #pragma warning disable ASPIREAZURE003 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. #pragma warning disable ASPIREPIPELINES001 +#pragma warning disable ASPIREPIPELINES002 #pragma warning disable ASPIREAZURE001 +using System.Text.Json.Nodes; using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Kubernetes; using Aspire.Hosting.Pipelines; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; namespace Aspire.Hosting.Azure.Kubernetes; @@ -38,6 +42,8 @@ public AzureKubernetesEnvironmentResource( // can observe the annotations. // - aks-get-credentials-{name}: fetches AKS credentials into an isolated // kubeconfig file after AKS is provisioned, before Helm prepare runs. + // - aks-get-credentials-for-destroy-{name}: fetches credentials from saved + // deployment state before cluster-scoped destroy steps run. Annotations.Add(new PipelineStepAnnotation(_ => { var k8sEnv = KubernetesEnvironment; @@ -69,10 +75,138 @@ public AzureKubernetesEnvironmentResource( RequiredBySteps = [$"prepare-{k8sEnv.Name}"] }; - return Task.FromResult>([prepareStep, getCredentialsStep]); + var getDestroyCredentialsStep = new PipelineStep + { + Name = $"aks-get-credentials-for-destroy-{Name}", + Description = $"Fetches AKS credentials for destroying {Name}", + Action = ctx => GetAksCredentialsForDestroyAsync(ctx), + // Keep this separate from the deploy credential step: depending on Azure + // provisioning here would pull provisioning into the destroy graph. + DependsOnSteps = [WellKnownPipelineSteps.DestroyPrereq] + }; + + return Task.FromResult>([prepareStep, getCredentialsStep, getDestroyCredentialsStep]); + })); + + Annotations.Add(new PipelineConfigurationAnnotation(async context => + { + var k8sEnv = KubernetesEnvironment; + var getDestroyCredentialsStep = context.GetSteps(this) + .Single(step => step.Name == $"aks-get-credentials-for-destroy-{Name}"); + var kubernetesDestroySteps = context + .GetSteps(HelmDeploymentEngine.GetKubernetesDestroyTag(k8sEnv.Name)) + .ToList(); + + var deploymentStateManager = context.Services.GetRequiredService(); + var deploymentStateSection = await deploymentStateManager + .AcquireSectionAsync($"Azure:Deployments:{Name}") + .ConfigureAwait(false); + + var azureEnvironment = context.Model.Resources.OfType().SingleOrDefault() + ?? throw new InvalidOperationException( + $"Azure environment resource required by AKS environment '{Name}' was not found."); + var destroyAzureStep = context.GetSteps(azureEnvironment) + .SingleOrDefault(step => step.Name == $"destroy-azure-{azureEnvironment.Name}") + ?? throw new InvalidOperationException( + $"Azure destroy step for environment '{azureEnvironment.Name}' was not found."); + + // A never-deployed AKS environment has no isolated kubeconfig to acquire. Likewise, a + // partially deployed environment can persist the cluster ID before any Helm release saves + // destroy state. In either case, aggregate Azure cleanup must skip cluster-scoped destroy + // steps rather than block on reacquiring credentials when there is nothing known to clean + // up. Explicitly targeting one of those Kubernetes cleanup steps still runs through the + // credential prerequisite and fails rather than allowing the command to fall back to the + // caller's ambient Kubernetes context. + var targetStep = context.Services.GetRequiredService>().Value.Step; + var hasPersistedAksIdentity = HasPersistedAksIdentity(deploymentStateSection.Data); + var hasPersistedKubernetesCleanupState = hasPersistedAksIdentity && + await HasPersistedKubernetesCleanupStateAsync( + deploymentStateManager, + context.Model, + k8sEnv).ConfigureAwait(false); + + if (!hasPersistedAksIdentity || !hasPersistedKubernetesCleanupState) + { + if (string.Equals(targetStep, WellKnownPipelineSteps.Destroy, StringComparison.Ordinal)) + { + foreach (var kubernetesDestroyStep in kubernetesDestroySteps) + { + kubernetesDestroyStep.RequiredBySteps.RemoveAll( + static stepName => string.Equals(stepName, WellKnownPipelineSteps.Destroy, StringComparison.Ordinal)); + } + + return; + } + + if (string.Equals(targetStep, destroyAzureStep.Name, StringComparison.Ordinal)) + { + return; + } + } + + foreach (var kubernetesDestroyStep in kubernetesDestroySteps) + { + kubernetesDestroyStep.DependsOn(getDestroyCredentialsStep); + + // The direct Helm uninstall step is an explicit, no-confirmation alternative to the + // aggregate destroy step. It needs isolated credentials when targeted directly, but + // Azure cleanup must not schedule both alternatives and uninstall the release twice. + if (!string.Equals( + kubernetesDestroyStep.Name, + HelmDeploymentEngine.GetHelmUninstallStepName(k8sEnv.Name), + StringComparison.Ordinal)) + { + destroyAzureStep.DependsOn(kubernetesDestroyStep); + } + } })); } + private static async Task HasPersistedKubernetesCleanupStateAsync( + IDeploymentStateManager deploymentStateManager, + DistributedApplicationModel model, + KubernetesEnvironmentResource environment) + { + var environmentState = await deploymentStateManager + .AcquireSectionAsync($"Helm:{environment.Name}") + .ConfigureAwait(false); + if (HasPersistedHelmReleaseState(environmentState.Data)) + { + return true; + } + + foreach (var chart in model.Resources.OfType()) + { + if (!chart.DestroyOnUninstall || + !string.Equals(chart.Parent.Name, environment.Name, StringComparison.Ordinal)) + { + continue; + } + + var chartState = await deploymentStateManager + .AcquireSectionAsync($"HelmChart:{environment.Name}:{chart.Name}") + .ConfigureAwait(false); + if (HasPersistedHelmReleaseState(chartState.Data)) + { + return true; + } + } + + return false; + } + + private static bool HasPersistedHelmReleaseState(JsonObject deploymentState) + { + try + { + return !string.IsNullOrEmpty(deploymentState["ReleaseName"]?.GetValue()); + } + catch (Exception ex) when (ex is InvalidOperationException or FormatException) + { + return false; + } + } + /// /// Gets the underlying Kubernetes environment resource used for Helm-based deployment. /// diff --git a/src/Aspire.Hosting.Kubernetes/CertManagerExtensions.cs b/src/Aspire.Hosting.Kubernetes/CertManagerExtensions.cs index d3fb43f0e02..2e557dd09f7 100644 --- a/src/Aspire.Hosting.Kubernetes/CertManagerExtensions.cs +++ b/src/Aspire.Hosting.Kubernetes/CertManagerExtensions.cs @@ -398,7 +398,8 @@ private static Task> BuildIssuerDeleteSteps( Name = $"cm-issuer-delete-{captured.Name}", Description = $"Deletes cert-manager ClusterIssuer '{captured.Name}'", Action = ctx => DeleteClusterIssuerAsync(ctx, certManager, captured), - DependsOnSteps = [WellKnownPipelineSteps.DestroyPrereq] + DependsOnSteps = [WellKnownPipelineSteps.DestroyPrereq], + Tags = [HelmDeploymentEngine.GetKubernetesDestroyTag(certManager.Parent.Name)] }; // Run before the cert-manager helm chart is uninstalled. Once the chart goes, @@ -499,6 +500,14 @@ private static async Task DeleteClusterIssuerAsync( CertManagerIssuerResource issuer) { var environment = certManager.Parent; + if (environment.SkipDestroyCleanup) + { + context.Logger.LogInformation( + "Skipping cert-manager cleanup for Kubernetes environment '{EnvironmentName}' because the cluster no longer exists.", + environment.Name); + return; + } + // Match the lowercase normalization used at apply time so we target the same object. var k8sIssuerName = issuer.Name.ToKubernetesResourceName(); diff --git a/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs b/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs index 644dfc68d96..58e973e022a 100644 --- a/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs +++ b/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs @@ -31,6 +31,9 @@ internal static partial class HelmDeploymentEngine private const string HelmUninstallTag = "helm-uninstall"; internal const string PrintSummaryTag = "print-summary"; + internal static string GetKubernetesDestroyTag(string environmentName) => $"kubernetes-destroy-{environmentName}"; + internal static string GetHelmUninstallStepName(string environmentName) => $"helm-uninstall-{environmentName}"; + /// /// Gets the environment-specific values file name, mirroring Docker Compose's .env.{envName} pattern. /// @@ -184,8 +187,17 @@ internal static Task> CreateStepsAsync( { Name = $"destroy-helm-{environment.Name}", Description = $"Confirms and destroys the Helm deployment for {environment.Name}.", + Tags = [GetKubernetesDestroyTag(environment.Name)], Action = async ctx => { + if (environment.SkipDestroyCleanup) + { + ctx.Logger.LogInformation( + "Skipping Helm cleanup for Kubernetes environment '{EnvironmentName}' because the cluster no longer exists.", + environment.Name); + return; + } + // Check deployment state to verify this environment was actually deployed var deploymentStateManager = ctx.Services.GetRequiredService(); var stateSection = await deploymentStateManager.AcquireSectionAsync($"Helm:{environment.Name}", ctx.CancellationToken).ConfigureAwait(false); @@ -205,11 +217,6 @@ await ctx.ReportingStep.CompleteAsync( var @namespace = savedNamespace ?? "default"; await ConfirmDestroyAsync(ctx, $"Uninstall Helm release '{savedReleaseName}' from namespace '{@namespace}'? This action cannot be undone.").ConfigureAwait(false); - var helmRunner = ctx.Services.GetRequiredService(); - // Defer the prereq check until state exists so `aspire destroy` against a - // never-deployed environment can still report "Nothing to destroy" without - // requiring Helm on PATH. - await HelmVersionValidator.EnsureMinimumVersionAsync(helmRunner, ctx.CancellationToken).ConfigureAwait(false); await HelmUninstallAsync(ctx, environment, savedReleaseName, @namespace).ConfigureAwait(false); ctx.Summary.Add("🗑️ Helm Release", savedReleaseName); @@ -226,12 +233,11 @@ await ctx.ReportingStep.CompleteAsync( // Step 5: Helm uninstall (teardown, callable directly via aspire do without confirmation) var helmUninstallStep = new PipelineStep { - Name = $"helm-uninstall-{environment.Name}", + Name = GetHelmUninstallStepName(environment.Name), Description = $"Uninstalls the Helm release for {environment.Name}.", - Tags = [HelmUninstallTag], + Tags = [HelmUninstallTag, GetKubernetesDestroyTag(environment.Name)], Action = ctx => HelmUninstallAsync(ctx, environment) }; - helmUninstallStep.DependsOn($"check-helm-prereqs-{environment.Name}"); steps.Add(helmUninstallStep); return Task.FromResult>(steps); @@ -577,6 +583,11 @@ private static async Task PrintDeploymentInstructionsAsync( private static async Task HelmUninstallAsync(PipelineStepContext context, KubernetesEnvironmentResource environment) { + if (TrySkipDestroyCleanup(context, environment)) + { + return; + } + var @namespace = await ResolveNamespaceAsync(context, environment).ConfigureAwait(false); var releaseName = await ResolveReleaseNameAsync(context, environment).ConfigureAwait(false); await HelmUninstallAsync(context, environment, releaseName, @namespace).ConfigureAwait(false); @@ -584,6 +595,11 @@ private static async Task HelmUninstallAsync(PipelineStepContext context, Kubern private static async Task HelmUninstallAsync(PipelineStepContext context, KubernetesEnvironmentResource environment, string releaseName, string @namespace) { + if (TrySkipDestroyCleanup(context, environment)) + { + return; + } + var uninstallTask = await context.ReportingStep.CreateTaskAsync( new MarkdownString($"Uninstalling Helm release **{releaseName}** from namespace **{@namespace}**"), context.CancellationToken).ConfigureAwait(false); @@ -593,7 +609,15 @@ private static async Task HelmUninstallAsync(PipelineStepContext context, Kubern try { var helmRunner = context.Services.GetRequiredService(); - var arguments = $"uninstall {releaseName} --namespace {@namespace}"; + // Keep the preflight inside the action so AKS destroy can skip cleanup for a cluster + // that no longer exists without requiring Helm on the machine. + await HelmVersionValidator.EnsureMinimumVersionAsync( + helmRunner, + context.CancellationToken).ConfigureAwait(false); + + // The release can already be absent after a direct uninstall or a prior destroy whose + // state cleanup failed. Keep retries idempotent without masking unrelated Helm errors. + var arguments = $"uninstall {releaseName} --namespace {@namespace} --ignore-not-found"; if (environment.KubeConfigPath is not null) { @@ -631,6 +655,19 @@ await uninstallTask.CompleteAsync( } } + private static bool TrySkipDestroyCleanup(PipelineStepContext context, KubernetesEnvironmentResource environment) + { + if (!environment.SkipDestroyCleanup) + { + return false; + } + + context.Logger.LogInformation( + "Skipping Helm cleanup for Kubernetes environment '{EnvironmentName}' because the cluster no longer exists.", + environment.Name); + return true; + } + private static async Task ConfirmDestroyAsync(PipelineStepContext context, string message) { var options = context.Services.GetRequiredService>(); diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs index d4b40c93d05..68d0913b578 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs @@ -131,6 +131,10 @@ public sealed class KubernetesEnvironmentResource : Resource, IComputeEnvironmen /// public string? KubeConfigPath { get; set; } + // AKS destroy sets this when persisted state points to a cluster that no longer exists. + // Cluster-scoped cleanup must then no-op instead of falling back to ambient credentials. + internal bool SkipDestroyCleanup { get; set; } + /// /// Gets or sets the parent compute environment resource that owns this Kubernetes environment. /// When set, resources with WithComputeEnvironment targeting the parent will also diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesHelmChartExtensions.cs b/src/Aspire.Hosting.Kubernetes/KubernetesHelmChartExtensions.cs index 7607f2ebcd6..32f457f90de 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesHelmChartExtensions.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesHelmChartExtensions.cs @@ -112,14 +112,9 @@ public static IResourceBuilder AddHelmChart( Name = $"helm-uninstall-{name}", Description = $"Uninstalls Helm chart '{name}' from namespace '{@namespace}'", Action = ctx => UninstallHelmChartAsync(ctx, environment, resource, releaseName, @namespace), - DependsOnSteps = [WellKnownPipelineSteps.DestroyPrereq] + DependsOnSteps = [WellKnownPipelineSteps.DestroyPrereq], + Tags = [HelmDeploymentEngine.GetKubernetesDestroyTag(environment.Name)] }; - // The uninstall path shells out to `helm uninstall`, so it must observe the same - // Helm CLI / version preflight as the deploy path. Without this dep, a missing or - // too-old Helm during teardown would surface as the raw spawn / unknown-flag error - // the env-wide `check-helm-prereqs-{env}` step exists to convert into an actionable - // message. Install is already covered transitively via `helm-deploy-{env}`. - uninstallStep.DependsOn($"check-helm-prereqs-{environment.Name}"); uninstallStep.RequiredBy(WellKnownPipelineSteps.Destroy); steps.Add(uninstallStep); } @@ -392,6 +387,14 @@ private static async Task UninstallHelmChartAsync( string defaultReleaseName, string defaultNamespace) { + if (environment.SkipDestroyCleanup) + { + context.Logger.LogInformation( + "Skipping Helm chart cleanup for Kubernetes environment '{EnvironmentName}' because the cluster no longer exists.", + environment.Name); + return; + } + var logger = context.Services.GetRequiredService>(); var helmRunner = context.Services.GetRequiredService(); var deploymentStateManager = context.Services.GetRequiredService(); @@ -407,12 +410,24 @@ private static async Task UninstallHelmChartAsync( var releaseName = !string.IsNullOrEmpty(savedReleaseName) ? savedReleaseName : defaultReleaseName; var @namespace = !string.IsNullOrEmpty(savedNamespace) ? savedNamespace : defaultNamespace; + // Keep the preflight inside the action so AKS destroy can skip cleanup for a cluster + // that no longer exists without requiring Helm on the machine. + await HelmVersionValidator.EnsureMinimumVersionAsync( + helmRunner, + context.CancellationToken).ConfigureAwait(false); + logger.LogInformation( "Uninstalling Helm release '{ReleaseName}' for chart '{ChartName}' from namespace '{Namespace}'.", releaseName, chart.Name, @namespace); var arguments = new StringBuilder(); arguments.Append(CultureInfo.InvariantCulture, $"uninstall {releaseName} --namespace {@namespace}"); + // The chart state is deleted before later destroy steps run, so a retry can reach this + // command after Helm already removed the release. Helm's --ignore-not-found only converts + // that missing-release case to success; authentication, connectivity, and other failures + // still return a nonzero exit code. + // See https://helm.sh/docs/helm/helm_uninstall/. + arguments.Append(" --ignore-not-found"); if (environment.KubeConfigPath is not null) { diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksWithHelmChartDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksWithHelmChartDeploymentTests.cs index d92f4fced95..374e52dd870 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/AksWithHelmChartDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksWithHelmChartDeploymentTests.cs @@ -151,7 +151,7 @@ private async Task DeployAksWithHelmChartCore(CancellationToken cancellationToke // Step 9: Deploy to AKS output.WriteLine("Step 9: Starting AKS deployment with external Helm chart..."); - await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.TypeAsync("aspire deploy"); await auto.EnterAsync(); await auto.WaitForPipelineSuccessAsync(timeout: TimeSpan.FromMinutes(30)); await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); @@ -215,20 +215,22 @@ await auto.TypeAsync( await auto.EnterAsync(); await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - // Step 15: Destroy and verify the external Helm chart was uninstalled too. - output.WriteLine("Step 15: Destroying deployment..."); - await auto.AspireDestroyAsync(counter); - - // Step 16: Verify the podinfo release is gone (this is the WithDestroy() contract). - output.WriteLine("Step 16: Verifying podinfo Helm release was uninstalled by aspire destroy..."); - await auto.TypeAsync( - "RELEASES=$(helm list -n podinfo -q 2>/dev/null); " + - "if [ -z \"$RELEASES\" ]; then echo 'VERIFY_OK: podinfo release was uninstalled'; " + - "else echo \"FAIL: podinfo release still exists: $RELEASES\"; exit 1; fi"); + // Step 15: Replace the ambient kubeconfig so destroy proves it acquires AKS + // credentials itself instead of reusing the context configured in Step 10. + output.WriteLine("Step 15: Clearing ambient Kubernetes credentials..."); + await auto.TypeAsync("export KUBECONFIG=$(mktemp)"); await auto.EnterAsync(); - await auto.WaitUntilTextAsync("VERIFY_OK", timeout: TimeSpan.FromMinutes(2)); await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + // Step 16: Destroy the application and opted-in external Helm chart before + // deleting the AKS resource group. + output.WriteLine("Step 16: Destroying deployment..."); + await auto.TypeAsync("aspire destroy --yes"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync("helm-uninstall-podinfo", timeout: TimeSpan.FromMinutes(10)); + await auto.WaitForPipelineSuccessAsync(timeout: TimeSpan.FromMinutes(20)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(1)); + // Step 17: Exit terminal await auto.TypeAsync("exit"); await auto.EnterAsync(); diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Aspire.Hosting.Azure.Kubernetes.Tests.csproj b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Aspire.Hosting.Azure.Kubernetes.Tests.csproj index 8831dc60110..c218dc9c093 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Aspire.Hosting.Azure.Kubernetes.Tests.csproj +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Aspire.Hosting.Azure.Kubernetes.Tests.csproj @@ -17,6 +17,7 @@ + diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs index 617749fc98f..21d5bd317d2 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +#pragma warning disable ASPIREAZURE001 #pragma warning disable ASPIREAZURE003 #pragma warning disable ASPIREPIPELINES001 #pragma warning disable ASPIREPIPELINES002 @@ -16,6 +17,7 @@ using Aspire.Hosting.Tests; using Aspire.Hosting.Utils; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging.Abstractions; namespace Aspire.Hosting.Azure.Tests; @@ -195,6 +197,931 @@ public async Task KubernetesPipelineStepsFlowThroughAksEnvironment() Assert.DoesNotContain(logs, msg => msg.Contains("aks-k8s")); } + [Fact] + public async Task DestroyPipelineFetchesCredentialsBeforeClusterCleanupAndDeletesAzureLast() + { + using var workspace = TemporaryWorkspace.Create(output); + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish, + workspace.Path, + step: WellKnownPipelineSteps.Diagnostics); + + var reporter = new TestPipelineActivityReporter(output); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(reporter); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + aks.AddHelmChart("podinfo", "oci://ghcr.io/stefanprodan/charts/podinfo", "6.7.1") + .WithDestroy(); + aks.AddCertManager("cert-manager") + .AddIssuer("letsencrypt"); + builder.AddContainer("api", "myimage") + .WithHttpEndpoint(targetPort: 8080); + + await using var app = builder.Build(); + await app.RunAsync(); + + var diagnosticLines = reporter.LoggedMessages + .Where(s => s.StepTitle == "diagnostics") + .Select(s => s.Message) + .SelectMany(message => message.Split('\n')) + .Select(line => line.Trim()) + .ToList(); + + Assert.Equal( + "Direct dependencies: destroy-prereq", + GetDirectDependencies(diagnosticLines, "aks-get-credentials-for-destroy-aks")); + Assert.Equal( + "Direct dependencies: aks-get-credentials-for-destroy-aks, destroy-prereq", + GetDirectDependencies(diagnosticLines, "destroy-helm-aks")); + Assert.Equal( + "Direct dependencies: aks-get-credentials-for-destroy-aks", + GetDirectDependencies(diagnosticLines, "helm-uninstall-aks")); + Assert.Equal( + "Direct dependencies: aks-get-credentials-for-destroy-aks, cm-issuer-delete-letsencrypt, destroy-prereq", + GetDirectDependencies(diagnosticLines, "helm-uninstall-cert-manager-chart")); + Assert.Equal( + "Direct dependencies: aks-get-credentials-for-destroy-aks, destroy-prereq", + GetDirectDependencies(diagnosticLines, "helm-uninstall-podinfo")); + Assert.Equal( + "Direct dependencies: aks-get-credentials-for-destroy-aks, destroy-prereq", + GetDirectDependencies(diagnosticLines, "cm-issuer-delete-letsencrypt")); + Assert.Equal( + "Direct dependencies: cm-issuer-delete-letsencrypt, destroy-helm-aks, destroy-prereq, helm-uninstall-cert-manager-chart, helm-uninstall-podinfo", + GetDirectDependencies(diagnosticLines, "destroy-azure-azure-environment")); + } + + [Fact] + public async Task DestroyRetryAfterExternalChartCleanupStillReachesAzureDeletion() + { + using var workspace = TemporaryWorkspace.Create(output); + var stateManager = new InMemoryDeploymentStateManager(); + stateManager.SetSection("Azure", new JsonObject + { + ["ResourceGroup"] = "app-resource-group", + ["SubscriptionId"] = "00000000-1111-2222-3333-444444444444" + }); + stateManager.SetSection("Azure:Deployments:aks", new JsonObject + { + ["Outputs"] = new JsonObject + { + ["id"] = new JsonObject + { + ["type"] = "String", + ["value"] = "/subscriptions/00000000-5555-6666-7777-888888888888/" + + "resourceGroups/cluster-resource-group/providers/Microsoft.ContainerService/" + + "managedClusters/aks-physical-name" + }, + ["name"] = new JsonObject + { + ["type"] = "String", + ["value"] = "aks-physical-name" + } + }.ToJsonString(), + ["Scope"] = new JsonObject + { + ["resourceGroup"] = "cluster-resource-group", + ["subscription"] = "00000000-5555-6666-7777-888888888888" + }.ToJsonString() + }); + stateManager.SetSection("HelmChart:aks:podinfo", new JsonObject + { + ["ReleaseName"] = "podinfo", + ["Namespace"] = "podinfo" + }); + + var uninstallCount = 0; + var fakeHelm = new FakeHelmRunner + { + CommandResultFactory = arguments => + { + if (!arguments.StartsWith("uninstall podinfo ", StringComparison.OrdinalIgnoreCase)) + { + return (0, null); + } + + if (Interlocked.Increment(ref uninstallCount) == 1 || + arguments.Contains(" --ignore-not-found", StringComparison.Ordinal)) + { + return (0, null); + } + + // Helm reports a missing release in this form after the first destroy removed it. + return (1, "Error: uninstall: Release not loaded: podinfo: release: not found"); + } + }; + + async Task RunDestroyAsync() + { + var reporter = new TestPipelineActivityReporter(output); + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish, + workspace.Path, + step: WellKnownPipelineSteps.Destroy); + builder.Services.AddSingleton(stateManager); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(reporter); + builder.Services.AddSingleton(fakeHelm); + builder.Services.Configure(o => o.SkipConfirmation = true); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + aks.Resource.AzCliPathResolverForTesting = () => "/fake/az"; + aks.Resource.AzCommandRunnerForTesting = (_, _, _) => Task.FromResult( + new AzureKubernetesEnvironmentResource.AzCommandResult(0, "apiVersion: v1", string.Empty)); + aks.AddHelmChart("podinfo", "oci://example.com/chart", "1.0.0") + .WithDestroy(); + + // Force the real Azure destroy step to fail after its Kubernetes prerequisites. + // The retry must reach this same failure instead of being blocked by the now-missing release. + builder.Services.RemoveAll(); + + await using var app = builder.Build(); + await app.RunAsync().WaitAsync(TimeSpan.FromSeconds(10)); + return reporter; + } + + var firstReporter = await RunDestroyAsync(); + var chartStateAfterFirstDestroy = await stateManager.AcquireSectionAsync( + "HelmChart:aks:podinfo", + TestContext.Current.CancellationToken); + + Assert.Empty(chartStateAfterFirstDestroy.Data); + Assert.StartsWith( + "Step 'destroy-azure-azure-environment' failed:", + firstReporter.CompletionMessage, + StringComparison.Ordinal); + + var retryReporter = await RunDestroyAsync(); + + Assert.Equal(1, uninstallCount); + Assert.All( + fakeHelm.Arguments.Where(arguments => arguments.StartsWith("uninstall", StringComparison.OrdinalIgnoreCase)), + arguments => Assert.Contains(" --ignore-not-found", arguments, StringComparison.Ordinal)); + Assert.Equal( + ["destroy-azure-azure-environment", "destroy-prereq"], + retryReporter.CreatedSteps + .Where(step => step.Contains("destroy", StringComparison.Ordinal) || + step.StartsWith("helm-uninstall-", StringComparison.Ordinal)) + .Order(StringComparer.Ordinal)); + Assert.StartsWith( + "Step 'destroy-azure-azure-environment' failed:", + retryReporter.CompletionMessage, + StringComparison.Ordinal); + } + + [Fact] + public async Task DestroyPipelineUsesMatchingCredentialsForEachAksEnvironment() + { + using var workspace = TemporaryWorkspace.Create(output); + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish, + workspace.Path, + step: WellKnownPipelineSteps.Diagnostics); + + var reporter = new TestPipelineActivityReporter(output); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(reporter); + + var east = builder.AddAzureKubernetesEnvironment("east"); + var west = builder.AddAzureKubernetesEnvironment("west"); + builder.AddContainer("east-api", "myimage") + .WithComputeEnvironment(east); + builder.AddContainer("west-api", "myimage") + .WithComputeEnvironment(west); + + await using var app = builder.Build(); + await app.RunAsync(); + + var diagnosticLines = reporter.LoggedMessages + .Where(s => s.StepTitle == "diagnostics") + .Select(s => s.Message) + .SelectMany(message => message.Split('\n')) + .Select(line => line.Trim()) + .ToList(); + + Assert.Equal( + "Direct dependencies: aks-get-credentials-for-destroy-east, destroy-prereq", + GetDirectDependencies(diagnosticLines, "destroy-helm-east")); + Assert.Equal( + "Direct dependencies: aks-get-credentials-for-destroy-west, destroy-prereq", + GetDirectDependencies(diagnosticLines, "destroy-helm-west")); + Assert.Equal( + "Direct dependencies: aks-get-credentials-for-destroy-east", + GetDirectDependencies(diagnosticLines, "helm-uninstall-east")); + Assert.Equal( + "Direct dependencies: aks-get-credentials-for-destroy-west", + GetDirectDependencies(diagnosticLines, "helm-uninstall-west")); + Assert.Equal( + "Direct dependencies: destroy-helm-east, destroy-helm-west, destroy-prereq", + GetDirectDependencies(diagnosticLines, "destroy-azure-azure-environment")); + } + + [Fact] + public async Task DestroyPipelineUsesPersistedAksOutputAndScopeWithoutProvisioning() + { + using var workspace = TemporaryWorkspace.Create(output); + var stateManager = new InMemoryDeploymentStateManager(); + stateManager.SetSection("Azure", new JsonObject + { + ["ResourceGroup"] = "app-resource-group", + ["SubscriptionId"] = "00000000-1111-2222-3333-444444444444" + }); + stateManager.SetSection("Azure:Deployments:aks", new JsonObject + { + ["Outputs"] = new JsonObject + { + ["id"] = new JsonObject + { + ["type"] = "String", + ["value"] = "/subscriptions/00000000-5555-6666-7777-888888888888/resourceGroups/cluster-resource-group/providers/Microsoft.ContainerService/managedClusters/aks-physical-name" + }, + ["name"] = new JsonObject + { + ["type"] = "String", + ["value"] = "aks-physical-name" + } + }.ToJsonString(), + ["Scope"] = new JsonObject + { + ["resourceGroup"] = "cluster-resource-group", + ["subscription"] = "00000000-5555-6666-7777-888888888888" + }.ToJsonString() + }); + stateManager.SetSection("Helm:aks", new JsonObject + { + ["ReleaseName"] = "aks", + ["Namespace"] = "default" + }); + var azArguments = new List(); + + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish, + workspace.Path, + step: WellKnownPipelineSteps.Destroy); + builder.Services.AddSingleton(stateManager); + builder.Services.AddSingleton(); + builder.Services.Configure(o => o.SkipConfirmation = true); + var aks = builder.AddAzureKubernetesEnvironment("aks"); + aks.Resource.AzCliPathResolverForTesting = () => "/fake/az"; + aks.Resource.AzCommandRunnerForTesting = (_, arguments, _) => + { + azArguments.Add(arguments); + return Task.FromResult(new AzureKubernetesEnvironmentResource.AzCommandResult( + 0, + "apiVersion: v1", + string.Empty)); + }; + + // Keep the test on the real destroy target while excluding the final ARM deletion, + // which has separate coverage and would require Azure credentials. + var azureEnvironment = builder.Resources.OfType().Single(); + azureEnvironment.Annotations.Add(new PipelineConfigurationAnnotation(context => + { + var destroyAzureStep = context.GetSteps(azureEnvironment) + .Single(step => step.Name == $"destroy-azure-{azureEnvironment.Name}"); + destroyAzureStep.RequiredBySteps.Remove(WellKnownPipelineSteps.Destroy); + return Task.CompletedTask; + })); + + await using var app = builder.Build(); + await app.RunAsync().WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal( + [ + "resource list --resource-group \"cluster-resource-group\" --resource-type Microsoft.ContainerService/managedClusters " + + "--name \"aks-physical-name\" --query [0].id -o tsv --subscription \"00000000-5555-6666-7777-888888888888\"", + "aks get-credentials --resource-group \"cluster-resource-group\" --name \"aks-physical-name\" --file - " + + "--subscription \"00000000-5555-6666-7777-888888888888\"" + ], + azArguments); + Assert.Empty(aks.Resource.Outputs); + } + + [Fact] + public async Task DestroyPipelineUsesPersistedAksResourceIdWhenScopeIsAbsent() + { + const string clusterSubscriptionId = "00000000-5555-6666-7777-888888888888"; + const string clusterResourceGroup = "cluster-resource-group"; + const string clusterName = "aks-physical-name"; + using var workspace = TemporaryWorkspace.Create(output); + var stateManager = new InMemoryDeploymentStateManager(); + stateManager.SetSection("Azure", new JsonObject + { + ["ResourceGroup"] = "current-app-resource-group", + ["SubscriptionId"] = "00000000-1111-2222-3333-444444444444" + }); + stateManager.SetSection("Azure:Deployments:aks", new JsonObject + { + ["Outputs"] = new JsonObject + { + ["id"] = new JsonObject + { + ["type"] = "String", + ["value"] = $"/subscriptions/{clusterSubscriptionId}/resourceGroups/{clusterResourceGroup}/providers/Microsoft.ContainerService/managedClusters/{clusterName}" + } + }.ToJsonString() + }); + stateManager.SetSection("Helm:aks", new JsonObject + { + ["ReleaseName"] = "aks", + ["Namespace"] = "default" + }); + var azArguments = new List(); + + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish, + workspace.Path, + step: WellKnownPipelineSteps.Destroy); + builder.Services.AddSingleton(stateManager); + builder.Services.AddSingleton(); + builder.Services.Configure(o => o.SkipConfirmation = true); + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + // A fresh destroy does not run provisioning, so resolving either reference would wait + // indefinitely. Compatibility state without Scope must use the persisted resource ID. + aks.Resource.Scope = new AzureBicepResourceScope( + aks.Resource.NameOutputReference, + aks.Resource.Id); + aks.Resource.AzCliPathResolverForTesting = () => "/fake/az"; + aks.Resource.AzCommandRunnerForTesting = (_, arguments, _) => + { + azArguments.Add(arguments); + return Task.FromResult(new AzureKubernetesEnvironmentResource.AzCommandResult( + 0, + "apiVersion: v1", + string.Empty)); + }; + + var azureEnvironment = builder.Resources.OfType().Single(); + azureEnvironment.Annotations.Add(new PipelineConfigurationAnnotation(context => + { + var destroyAzureStep = context.GetSteps(azureEnvironment) + .Single(step => step.Name == $"destroy-azure-{azureEnvironment.Name}"); + destroyAzureStep.RequiredBySteps.Remove(WellKnownPipelineSteps.Destroy); + return Task.CompletedTask; + })); + + await using var app = builder.Build(); + await app.RunAsync().WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal( + [ + $"resource list --resource-group \"{clusterResourceGroup}\" --resource-type Microsoft.ContainerService/managedClusters " + + $"--name \"{clusterName}\" --query [0].id -o tsv --subscription \"{clusterSubscriptionId}\"", + $"aks get-credentials --resource-group \"{clusterResourceGroup}\" --name \"{clusterName}\" --file - " + + $"--subscription \"{clusterSubscriptionId}\"" + ], + azArguments); + Assert.Empty(aks.Resource.Outputs); + } + + [Fact] + public async Task DestroyPipelineSkipsClusterCleanupWhenAksDeploymentStateHasNoIdentity() + { + using var workspace = TemporaryWorkspace.Create(output); + var stateManager = new InMemoryDeploymentStateManager(); + stateManager.SetSection("Azure:Deployments:aks", new JsonObject + { + ["Location"] = "westus2" + }); + stateManager.SetSection("Helm:aks", new JsonObject + { + ["ReleaseName"] = "same-name-as-ambient-release", + ["Namespace"] = "default" + }); + + var reporter = new TestPipelineActivityReporter(output); + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish, + workspace.Path, + step: WellKnownPipelineSteps.Destroy); + builder.Services.AddSingleton(stateManager); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(reporter); + builder.Services.Configure(o => o.SkipConfirmation = true); + var aks = builder.AddAzureKubernetesEnvironment("aks"); + aks.Resource.AzCliPathResolverForTesting = () => + throw new InvalidOperationException("The Azure CLI must not run for incomplete deployment state."); + + await using var app = builder.Build(); + await app.RunAsync().WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal( + ["destroy", "destroy-azure-azure-environment", "destroy-prereq"], + reporter.CreatedSteps + .Where(step => step.Contains("destroy", StringComparison.Ordinal)) + .Order(StringComparer.Ordinal)); + Assert.Contains( + reporter.CompletedSteps, + step => step is ("destroy-azure-azure-environment", _, CompletionState.Completed)); + Assert.Null(aks.Resource.KubernetesEnvironment.KubeConfigPath); + } + + [Fact] + public async Task DestroyPipelineSkipsClusterCleanupWhenPersistedAksIdentityHasNoKubernetesCleanupState() + { + const string subscriptionId = "00000000-5555-6666-7777-888888888888"; + const string resourceGroup = "cluster-resource-group"; + const string clusterName = "aks-physical-name"; + + using var workspace = TemporaryWorkspace.Create(output); + var stateManager = new InMemoryDeploymentStateManager(); + stateManager.SetSection("Azure:Deployments:aks", new JsonObject + { + ["Outputs"] = new JsonObject + { + ["id"] = new JsonObject + { + ["type"] = "String", + ["value"] = $"/subscriptions/{subscriptionId}/resourceGroups/{resourceGroup}/providers/Microsoft.ContainerService/managedClusters/{clusterName}" + } + }.ToJsonString() + }); + + var reporter = new TestPipelineActivityReporter(output); + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish, + workspace.Path, + step: WellKnownPipelineSteps.Destroy); + builder.Services.AddSingleton(stateManager); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(reporter); + builder.Services.Configure(o => o.SkipConfirmation = true); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + aks.Resource.AzCliPathResolverForTesting = () => + throw new InvalidOperationException("The Azure CLI must not run when no Kubernetes cleanup state was persisted."); + aks.AddHelmChart("same-name-as-ambient-release", "oci://example.com/chart", "1.0.0") + .WithDestroy(); + + await using var app = builder.Build(); + await app.RunAsync().WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal( + ["destroy", "destroy-azure-azure-environment", "destroy-prereq"], + reporter.CreatedSteps + .Where(step => step.Contains("destroy", StringComparison.Ordinal) || + step.StartsWith("helm-uninstall-", StringComparison.Ordinal)) + .Order(StringComparer.Ordinal)); + Assert.Contains( + reporter.CompletedSteps, + step => step is ("destroy-azure-azure-environment", _, CompletionState.Completed)); + Assert.Null(aks.Resource.KubernetesEnvironment.KubeConfigPath); + } + + [Fact] + public async Task DestroyPipelineSkipsClusterCleanupForNeverDeployedAksEnvironment() + { + using var workspace = TemporaryWorkspace.Create(output); + var reporter = new TestPipelineActivityReporter(output); + var stateManager = new InMemoryDeploymentStateManager(); + stateManager.SetSection("Sentinel", new JsonObject { ["Value"] = "cleared-by-destroy" }); + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish, + workspace.Path, + step: WellKnownPipelineSteps.Destroy); + builder.Services.AddSingleton(stateManager); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(reporter); + builder.Services.Configure(o => o.SkipConfirmation = true); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + aks.AddHelmChart("same-name-as-ambient-release", "oci://example.com/chart", "1.0.0") + .WithDestroy(); + + await using var app = builder.Build(); + await app.RunAsync().WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal( + ["destroy", "destroy-azure-azure-environment", "destroy-prereq"], + reporter.CreatedSteps + .Where(step => step.Contains("destroy", StringComparison.Ordinal) || + step.StartsWith("helm-uninstall-", StringComparison.Ordinal)) + .Order(StringComparer.Ordinal)); + var sentinelState = await stateManager.AcquireSectionAsync( + "Sentinel", + TestContext.Current.CancellationToken); + Assert.Empty(sentinelState.Data); + Assert.Null(aks.Resource.KubernetesEnvironment.KubeConfigPath); + } + + [Fact] + public async Task DirectAzureDestroySkipsClusterCleanupWithoutPersistedAksIdentity() + { + using var workspace = TemporaryWorkspace.Create(output); + var reporter = new TestPipelineActivityReporter(output); + var stateManager = new InMemoryDeploymentStateManager(); + stateManager.SetSection("Azure:Deployments:aks", new JsonObject + { + ["Location"] = "westus2" + }); + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish, + workspace.Path, + step: "destroy-azure-azure-environment"); + builder.Services.AddSingleton(stateManager); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(reporter); + builder.Services.Configure(o => o.SkipConfirmation = true); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + aks.AddHelmChart("same-name-as-ambient-release", "oci://example.com/chart", "1.0.0") + .WithDestroy(); + + await using var app = builder.Build(); + await app.RunAsync().WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal( + ["destroy-azure-azure-environment", "destroy-prereq"], + reporter.CreatedSteps + .Where(step => step.Contains("destroy", StringComparison.Ordinal) || + step.StartsWith("helm-uninstall-", StringComparison.Ordinal)) + .Order(StringComparer.Ordinal)); + Assert.Contains( + reporter.CompletedSteps, + step => step is ("destroy-azure-azure-environment", _, CompletionState.Completed)); + Assert.Null(aks.Resource.KubernetesEnvironment.KubeConfigPath); + } + + [Fact] + public async Task DirectAzureDestroySkipsClusterCleanupWhenPersistedAksIdentityHasNoKubernetesCleanupState() + { + const string subscriptionId = "00000000-5555-6666-7777-888888888888"; + const string resourceGroup = "cluster-resource-group"; + const string clusterName = "aks-physical-name"; + + using var workspace = TemporaryWorkspace.Create(output); + var reporter = new TestPipelineActivityReporter(output); + var stateManager = new InMemoryDeploymentStateManager(); + stateManager.SetSection("Azure:Deployments:aks", new JsonObject + { + ["Outputs"] = new JsonObject + { + ["id"] = new JsonObject + { + ["type"] = "String", + ["value"] = $"/subscriptions/{subscriptionId}/resourceGroups/{resourceGroup}/providers/Microsoft.ContainerService/managedClusters/{clusterName}" + } + }.ToJsonString() + }); + + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish, + workspace.Path, + step: "destroy-azure-azure-environment"); + builder.Services.AddSingleton(stateManager); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(reporter); + builder.Services.Configure(o => o.SkipConfirmation = true); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + aks.Resource.AzCliPathResolverForTesting = () => + throw new InvalidOperationException("The Azure CLI must not run when no Kubernetes cleanup state was persisted."); + aks.AddHelmChart("same-name-as-ambient-release", "oci://example.com/chart", "1.0.0") + .WithDestroy(); + + await using var app = builder.Build(); + await app.RunAsync().WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal( + ["destroy-azure-azure-environment", "destroy-prereq"], + reporter.CreatedSteps + .Where(step => step.Contains("destroy", StringComparison.Ordinal) || + step.StartsWith("helm-uninstall-", StringComparison.Ordinal)) + .Order(StringComparer.Ordinal)); + Assert.Contains( + reporter.CompletedSteps, + step => step is ("destroy-azure-azure-environment", _, CompletionState.Completed)); + Assert.Null(aks.Resource.KubernetesEnvironment.KubeConfigPath); + } + + [Theory] + [InlineData(0, "", "")] + [InlineData(3, "", "(ResourceGroupNotFound) Resource group 'cluster-resource-group' could not be found.")] + public async Task DirectAzureDestroySkipsClusterCleanupWhenPersistedAksNoLongerExists( + int resourceQueryExitCode, + string resourceQueryOutput, + string resourceQueryError) + { + const string subscriptionId = "00000000-5555-6666-7777-888888888888"; + const string resourceGroup = "cluster-resource-group"; + const string clusterName = "deleted-aks"; + using var workspace = TemporaryWorkspace.Create(output); + var reporter = new TestPipelineActivityReporter(output); + var stateManager = new InMemoryDeploymentStateManager(); + stateManager.SetSection("Azure:Deployments:aks", new JsonObject + { + ["Outputs"] = new JsonObject + { + ["id"] = new JsonObject + { + ["type"] = "String", + ["value"] = $"/subscriptions/{subscriptionId}/resourceGroups/{resourceGroup}/providers/Microsoft.ContainerService/managedClusters/{clusterName}" + } + }.ToJsonString() + }); + stateManager.SetSection("Helm:aks", new JsonObject + { + ["ReleaseName"] = "same-name-as-ambient-release", + ["Namespace"] = "default" + }); + var fakeHelm = new FakeHelmRunner { ThrowOnVersion = true }; + var azArguments = new List(); + + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish, + workspace.Path, + step: "destroy-azure-azure-environment"); + builder.Services.AddSingleton(stateManager); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(reporter); + builder.Services.AddSingleton(fakeHelm); + builder.Services.Configure(o => o.SkipConfirmation = true); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + aks.Resource.AzCliPathResolverForTesting = () => "/fake/az"; + aks.Resource.AzCommandRunnerForTesting = (_, arguments, _) => + { + azArguments.Add(arguments); + return Task.FromResult(new AzureKubernetesEnvironmentResource.AzCommandResult( + resourceQueryExitCode, + resourceQueryOutput, + resourceQueryError)); + }; + aks.AddHelmChart("same-name-as-ambient-release", "oci://example.com/chart", "1.0.0") + .WithDestroy(); + + await using var app = builder.Build(); + await app.RunAsync().WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal( + [ + $"resource list --resource-group \"{resourceGroup}\" --resource-type Microsoft.ContainerService/managedClusters " + + $"--name \"{clusterName}\" --query [0].id -o tsv --subscription \"{subscriptionId}\"" + ], + azArguments); + Assert.False(fakeHelm.WasUninstallCalled); + Assert.False(fakeHelm.WasVersionCalled); + Assert.Contains( + reporter.CompletedSteps, + step => step is ("destroy-azure-azure-environment", _, CompletionState.Completed)); + Assert.Null(aks.Resource.KubernetesEnvironment.KubeConfigPath); + Assert.True(aks.Resource.KubernetesEnvironment.SkipDestroyCleanup); + } + + [Fact] + public async Task DirectMainHelmUninstallSkipsAbsentClusterWithoutResolvingParameterBackedAnnotations() + { + const string subscriptionId = "00000000-5555-6666-7777-888888888888"; + const string resourceGroup = "cluster-resource-group"; + const string clusterName = "deleted-aks"; + using var workspace = TemporaryWorkspace.Create(output); + var reporter = new TestPipelineActivityReporter(output); + var stateManager = new InMemoryDeploymentStateManager(); + stateManager.SetSection("Azure:Deployments:aks", new JsonObject + { + ["Outputs"] = new JsonObject + { + ["id"] = new JsonObject + { + ["type"] = "String", + ["value"] = $"/subscriptions/{subscriptionId}/resourceGroups/{resourceGroup}/providers/Microsoft.ContainerService/managedClusters/{clusterName}" + } + }.ToJsonString() + }); + var fakeHelm = new FakeHelmRunner { ThrowOnVersion = true }; + var azArguments = new List(); + + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish, + workspace.Path, + step: "helm-uninstall-aks"); + builder.Services.AddSingleton(stateManager); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(reporter); + builder.Services.AddSingleton(fakeHelm); + + var releaseParameter = builder.AddParameter("helm-release"); + var namespaceParameter = builder.AddParameter("helm-namespace"); + var aks = builder.AddAzureKubernetesEnvironment("aks"); + aks.Resource.KubernetesEnvironment.Annotations.Add( + new HelmReleaseNameAnnotation(ReferenceExpression.Create($"{releaseParameter.Resource}"))); + aks.Resource.KubernetesEnvironment.Annotations.Add( + new KubernetesNamespaceAnnotation(ReferenceExpression.Create($"{namespaceParameter.Resource}"))); + aks.Resource.AzCliPathResolverForTesting = () => "/fake/az"; + aks.Resource.AzCommandRunnerForTesting = (_, arguments, _) => + { + azArguments.Add(arguments); + return Task.FromResult(new AzureKubernetesEnvironmentResource.AzCommandResult( + 0, + string.Empty, + string.Empty)); + }; + + await using var app = builder.Build(); + await app.RunAsync().WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal( + [ + $"resource list --resource-group \"{resourceGroup}\" --resource-type Microsoft.ContainerService/managedClusters " + + $"--name \"{clusterName}\" --query [0].id -o tsv --subscription \"{subscriptionId}\"" + ], + azArguments); + Assert.Equal( + ["aks-get-credentials-for-destroy-aks", "destroy-prereq", "helm-uninstall-aks"], + reporter.CreatedSteps + .Where(step => step.Contains("destroy", StringComparison.Ordinal) || + step.StartsWith("helm-uninstall-", StringComparison.Ordinal)) + .Order(StringComparer.Ordinal)); + Assert.False(fakeHelm.WasUninstallCalled); + Assert.False(fakeHelm.WasVersionCalled); + Assert.Contains( + reporter.CompletedSteps, + step => step is ("pipeline-execution", "Completed successfully", CompletionState.Completed)); + Assert.Null(aks.Resource.KubernetesEnvironment.KubeConfigPath); + Assert.True(aks.Resource.KubernetesEnvironment.SkipDestroyCleanup); + } + + [Fact] + public async Task DirectMainHelmUninstallSkipsAbsentClusterWithoutResolvingProvisioningBackedAnnotations() + { + const string subscriptionId = "00000000-5555-6666-7777-888888888888"; + const string resourceGroup = "cluster-resource-group"; + const string clusterName = "deleted-aks"; + using var workspace = TemporaryWorkspace.Create(output); + var reporter = new TestPipelineActivityReporter(output); + var stateManager = new InMemoryDeploymentStateManager(); + stateManager.SetSection("Azure:Deployments:aks", new JsonObject + { + ["Outputs"] = new JsonObject + { + ["id"] = new JsonObject + { + ["type"] = "String", + ["value"] = $"/subscriptions/{subscriptionId}/resourceGroups/{resourceGroup}/providers/Microsoft.ContainerService/managedClusters/{clusterName}" + } + }.ToJsonString() + }); + var fakeHelm = new FakeHelmRunner { ThrowOnVersion = true }; + var azArguments = new List(); + + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish, + workspace.Path, + step: "helm-uninstall-aks"); + builder.Services.AddSingleton(stateManager); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(reporter); + builder.Services.AddSingleton(fakeHelm); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + // A direct cleanup never provisions the AKS resource, so resolving this output would wait + // indefinitely on ProvisioningTaskCompletionSource if the absent-cluster no-op ran too late. + aks.Resource.KubernetesEnvironment.Annotations.Add( + new HelmReleaseNameAnnotation(ReferenceExpression.Create($"{aks.Resource.NameOutputReference}"))); + aks.Resource.AzCliPathResolverForTesting = () => "/fake/az"; + aks.Resource.AzCommandRunnerForTesting = (_, arguments, _) => + { + azArguments.Add(arguments); + return Task.FromResult(new AzureKubernetesEnvironmentResource.AzCommandResult( + 0, + string.Empty, + string.Empty)); + }; + + await using var app = builder.Build(); + await app.RunAsync().WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal( + [ + $"resource list --resource-group \"{resourceGroup}\" --resource-type Microsoft.ContainerService/managedClusters " + + $"--name \"{clusterName}\" --query [0].id -o tsv --subscription \"{subscriptionId}\"" + ], + azArguments); + Assert.Equal( + ["aks-get-credentials-for-destroy-aks", "destroy-prereq", "helm-uninstall-aks"], + reporter.CreatedSteps + .Where(step => step.Contains("destroy", StringComparison.Ordinal) || + step.StartsWith("helm-uninstall-", StringComparison.Ordinal)) + .Order(StringComparer.Ordinal)); + Assert.False(fakeHelm.WasUninstallCalled); + Assert.False(fakeHelm.WasVersionCalled); + Assert.Contains( + reporter.CompletedSteps, + step => step is ("pipeline-execution", "Completed successfully", CompletionState.Completed)); + Assert.Null(aks.Resource.KubernetesEnvironment.KubeConfigPath); + Assert.True(aks.Resource.KubernetesEnvironment.SkipDestroyCleanup); + } + + [Fact] + public async Task DirectMainHelmUninstallUsesPersistedAksCredentials() + { + const string subscriptionId = "00000000-5555-6666-7777-888888888888"; + const string resourceGroup = "cluster-resource-group"; + const string clusterName = "aks-physical-name"; + using var workspace = TemporaryWorkspace.Create(output); + var stateManager = new InMemoryDeploymentStateManager(); + stateManager.SetSection("Azure:Deployments:aks", new JsonObject + { + ["Outputs"] = new JsonObject + { + ["id"] = new JsonObject + { + ["type"] = "String", + ["value"] = $"/subscriptions/{subscriptionId}/resourceGroups/{resourceGroup}/providers/Microsoft.ContainerService/managedClusters/{clusterName}" + } + }.ToJsonString() + }); + var fakeHelm = new FakeHelmRunner(); + var azArguments = new List(); + + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish, + workspace.Path, + step: "helm-uninstall-aks"); + builder.Services.AddSingleton(stateManager); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(fakeHelm); + var aks = builder.AddAzureKubernetesEnvironment("aks"); + aks.Resource.KubernetesEnvironment.Annotations.Add( + new HelmReleaseNameAnnotation(ReferenceExpression.Create($"main-release"))); + aks.Resource.KubernetesEnvironment.Annotations.Add( + new KubernetesNamespaceAnnotation(ReferenceExpression.Create($"main-namespace"))); + aks.Resource.AzCliPathResolverForTesting = () => "/fake/az"; + aks.Resource.AzCommandRunnerForTesting = (_, arguments, _) => + { + azArguments.Add(arguments); + return Task.FromResult(new AzureKubernetesEnvironmentResource.AzCommandResult( + 0, + arguments.StartsWith("resource list", StringComparison.Ordinal) + ? $"/subscriptions/{subscriptionId}/resourceGroups/{resourceGroup}/providers/Microsoft.ContainerService/managedClusters/{clusterName}" + : "apiVersion: v1", + string.Empty)); + }; + + await using var app = builder.Build(); + await app.RunAsync().WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal( + [ + $"resource list --resource-group \"{resourceGroup}\" --resource-type Microsoft.ContainerService/managedClusters " + + $"--name \"{clusterName}\" --query [0].id -o tsv --subscription \"{subscriptionId}\"", + $"aks get-credentials --resource-group \"{resourceGroup}\" --name \"{clusterName}\" --file - " + + $"--subscription \"{subscriptionId}\"" + ], + azArguments); + var kubeConfigPath = Assert.IsType(aks.Resource.KubernetesEnvironment.KubeConfigPath); + Assert.Equal( + $"uninstall main-release --namespace main-namespace --ignore-not-found --kubeconfig \"{kubeConfigPath}\"", + Assert.Single( + fakeHelm.Arguments, + arguments => arguments.StartsWith("uninstall", StringComparison.OrdinalIgnoreCase))); + } + + [Fact] + public async Task DirectKubernetesCleanupFailsForNeverDeployedAksEnvironment() + { + using var workspace = TemporaryWorkspace.Create(output); + var reporter = new TestPipelineActivityReporter(output); + var stateManager = new InMemoryDeploymentStateManager(); + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish, + workspace.Path, + step: "helm-uninstall-same-name-as-ambient-release"); + builder.Services.AddSingleton(stateManager); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(reporter); + builder.Services.AddSingleton(); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + aks.AddHelmChart("same-name-as-ambient-release", "oci://example.com/chart", "1.0.0") + .WithDestroy(); + + await using var app = builder.Build(); + await app.RunAsync().WaitAsync(TimeSpan.FromSeconds(10)); + + Assert.Equal(CompletionState.CompletedWithError, reporter.ResultCompletionState); + Assert.Equal( + "Step 'aks-get-credentials-for-destroy-aks' failed: " + + "No Azure deployment state was found for AKS environment 'aks'. " + + "Cluster cleanup cannot run without an isolated kubeconfig.", + reporter.CompletionMessage); + Assert.Equal( + ["aks-get-credentials-for-destroy-aks", "destroy-prereq"], + reporter.CreatedSteps + .Where(step => step.Contains("destroy", StringComparison.Ordinal) || + step.StartsWith("helm-uninstall-", StringComparison.Ordinal)) + .Order(StringComparer.Ordinal)); + Assert.Null(aks.Resource.KubernetesEnvironment.KubeConfigPath); + } + + private static string GetDirectDependencies(List diagnosticLines, string stepName) + { + var targetLine = diagnosticLines.IndexOf($"If targeting '{stepName}':"); + Assert.InRange(targetLine, 0, diagnosticLines.Count - 2); + return diagnosticLines[targetLine + 1]; + } + [Fact] public async Task DeploymentScopeUsesCurrentDeploymentState() { @@ -373,6 +1300,40 @@ public async Task FetchKubeConfigThrowsWhenAzureCliFails() exception.Message); } + [Fact] + public async Task AksResourceExistsThrowsWhenAzureCliQueryFails() + { + var exception = await Assert.ThrowsAsync( + () => AzureKubernetesEnvironmentResource.AksResourceExistsAsync( + "/usr/bin/az", + "00000000-0000-0000-0000-000000000001", + "deployment-rg", + "deployment-aks", + (path, arguments) => Task.FromResult( + new AzureKubernetesEnvironmentResource.AzCommandResult(1, "", "authentication failed")))); + + Assert.Equal( + "az resource list failed while checking AKS cluster existence (exit code 1): authentication failed", + exception.Message); + } + + [Fact] + public async Task AksResourceDoesNotExistWhenResourceGroupWasDeleted() + { + var exists = await AzureKubernetesEnvironmentResource.AksResourceExistsAsync( + "/usr/bin/az", + "00000000-0000-0000-0000-000000000001", + "deleted-rg", + "deployment-aks", + (path, arguments) => Task.FromResult( + new AzureKubernetesEnvironmentResource.AzCommandResult( + 3, + "", + "(ResourceGroupNotFound) Resource group 'deleted-rg' could not be found."))); + + Assert.False(exists); + } + [Fact] public async Task GetCredentialsStepScopesEveryAzureCliCallToDeploymentSubscription() { diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Aspire.Hosting.Kubernetes.Tests.csproj b/tests/Aspire.Hosting.Kubernetes.Tests/Aspire.Hosting.Kubernetes.Tests.csproj index d20646ecbcd..634bf320726 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Aspire.Hosting.Kubernetes.Tests.csproj +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Aspire.Hosting.Kubernetes.Tests.csproj @@ -29,6 +29,7 @@ + diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/HelmVersionValidatorTests.cs b/tests/Aspire.Hosting.Kubernetes.Tests/HelmVersionValidatorTests.cs index 134498cfc38..73e124c2475 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/HelmVersionValidatorTests.cs +++ b/tests/Aspire.Hosting.Kubernetes.Tests/HelmVersionValidatorTests.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Aspire.Hosting.Tests; + namespace Aspire.Hosting.Kubernetes.Tests; public class HelmVersionValidatorTests diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesDeployTests.cs b/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesDeployTests.cs index 0901b6aa13c..e0af29ee2a2 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesDeployTests.cs +++ b/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesDeployTests.cs @@ -533,22 +533,17 @@ public async Task HelmUninstallStep_RequiredByDestroy() } [Fact] - public async Task HelmUninstallStep_DependsOnCheckHelmPrereqs() + public async Task HelmUninstallStep_ValidatesHelmVersionBeforeUninstall() { - // Regression coverage for PR #17491 review feedback: direct uninstall - // invokes `helm`, so it must gate on the same prereq check as deploy. - // `destroy-helm-{env}` defers the check until saved state exists so the - // no-state path can still report "Nothing to destroy" without Helm. using var workspace = TemporaryWorkspace.Create(outputHelper); - + var fakeHelm = new FakeHelmRunner { VersionExitCode = 1 }; var builder = TestDistributedApplicationBuilder.Create( DistributedApplicationOperation.Publish, workspace.Path, - step: WellKnownPipelineSteps.Diagnostics); - var mockActivityReporter = new TestPipelineActivityReporter(outputHelper); + step: "helm-uninstall-env"); builder.Services.AddSingleton(); - builder.Services.AddSingleton(mockActivityReporter); + builder.Services.AddSingleton(fakeHelm); builder.AddKubernetesEnvironment("env"); builder.AddContainer("api", "myimage"); @@ -556,45 +551,23 @@ public async Task HelmUninstallStep_DependsOnCheckHelmPrereqs() using var app = builder.Build(); await app.RunAsync(); - var logs = mockActivityReporter.LoggedMessages - .Where(s => s.StepTitle == "diagnostics") - .Select(s => s.Message) - .ToList(); - - var diagnosticLines = string.Join('\n', logs) - .Split('\n') - .Select(l => l.Trim()) - .ToList(); - - var destroyTargetLine = diagnosticLines.IndexOf("If targeting 'destroy-helm-env':"); - Assert.InRange(destroyTargetLine, 0, diagnosticLines.Count - 2); - Assert.Equal("Direct dependencies: destroy-prereq", diagnosticLines[destroyTargetLine + 1]); - - var uninstallTargetLine = diagnosticLines.IndexOf("If targeting 'helm-uninstall-env':"); - Assert.InRange(uninstallTargetLine, 0, diagnosticLines.Count - 2); - Assert.Equal("Direct dependencies: check-helm-prereqs-env", diagnosticLines[uninstallTargetLine + 1]); + Assert.Equal(["version --short"], fakeHelm.Arguments); + Assert.True(fakeHelm.WasVersionCalled); + Assert.False(fakeHelm.WasUninstallCalled); } [Fact] - public async Task PerChartHelmUninstallStep_DependsOnCheckHelmPrereqs() + public async Task PerChartHelmUninstallStep_ValidatesHelmVersionBeforeUninstall() { - // Regression coverage for PR #17491 review feedback: per-chart - // `helm-uninstall-{name}` steps created by `AddHelmChart(...).WithDestroy()` - // must depend on `check-helm-prereqs-{env}`. The install side is covered - // transitively (via `helm-deploy-{env}`), but the uninstall side previously - // only set `DependsOnSteps = [DestroyPrereq]`, so a missing or too-old - // Helm during chart teardown would bypass the validator and surface as - // the cryptic spawn / unknown-flag error this PR exists to prevent. using var workspace = TemporaryWorkspace.Create(outputHelper); - + var fakeHelm = new FakeHelmRunner { VersionExitCode = 1 }; var builder = TestDistributedApplicationBuilder.Create( DistributedApplicationOperation.Publish, workspace.Path, - step: WellKnownPipelineSteps.Diagnostics); - var mockActivityReporter = new TestPipelineActivityReporter(outputHelper); + step: "helm-uninstall-podinfo"); builder.Services.AddSingleton(); - builder.Services.AddSingleton(mockActivityReporter); + builder.Services.AddSingleton(fakeHelm); var k8s = builder.AddKubernetesEnvironment("env"); k8s.AddHelmChart("podinfo", "oci://ghcr.io/stefanprodan/charts/podinfo", "6.7.1") @@ -603,14 +576,9 @@ public async Task PerChartHelmUninstallStep_DependsOnCheckHelmPrereqs() using var app = builder.Build(); await app.RunAsync(); - var logs = mockActivityReporter.LoggedMessages - .Where(s => s.StepTitle == "diagnostics") - .Select(s => s.Message) - .ToList(); - - var chartUninstallLines = logs.Where(l => l.Contains("helm-uninstall-podinfo")).ToList(); - Assert.NotEmpty(chartUninstallLines); - Assert.Contains(chartUninstallLines, msg => msg.Contains("check-helm-prereqs-env")); + Assert.Equal(["version --short"], fakeHelm.Arguments); + Assert.True(fakeHelm.WasVersionCalled); + Assert.False(fakeHelm.WasUninstallCalled); } [Fact] @@ -1866,10 +1834,9 @@ public async Task DestroyHelm_WithState_RunsHelmUninstall() using var app = builder.Build(); await app.RunAsync(); - // Verify helm uninstall was called with saved state values - Assert.True(fakeHelm.WasUninstallCalled); - Assert.Contains("my-release", fakeHelm.LastArguments!); - Assert.Contains("my-namespace", fakeHelm.LastArguments!); + Assert.Equal( + "uninstall my-release --namespace my-namespace --ignore-not-found", + fakeHelm.LastArguments); } [Fact] @@ -1945,4 +1912,103 @@ public async Task DestroyHelm_WhenUninstallFails_PreservesState() var stateSection = await stateManager.AcquireSectionAsync("Helm:env"); Assert.Equal("my-release", stateSection.Data["ReleaseName"]?.ToString()); } + + [Fact] + public async Task DestroyHelm_WhenReleaseWasAlreadyRemoved_CompletesRetry() + { + using var workspace = TemporaryWorkspace.Create(outputHelper); + + var fakeHelm = new FakeHelmRunner + { + CommandResultFactory = arguments => + arguments.Contains(" --ignore-not-found", StringComparison.Ordinal) + ? (0, null) + : (1, "Error: uninstall: Release not loaded: my-release: release: not found") + }; + var stateManager = new InMemoryDeploymentStateManager(); + stateManager.SetSection("Helm:env", new JsonObject + { + ["ReleaseName"] = "my-release", + ["Namespace"] = "my-namespace" + }); + + var reporter = new TestPipelineActivityReporter(outputHelper); + var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish, + workspace.Path, + step: WellKnownPipelineSteps.Destroy); + + builder.Services.AddSingleton(); + builder.Services.AddSingleton(reporter); + builder.Services.AddSingleton(stateManager); + builder.Services.AddSingleton(fakeHelm); + builder.Services.Configure(o => o.SkipConfirmation = true); + + builder.AddKubernetesEnvironment("env"); + builder.AddContainer("api", "myimage"); + + using var app = builder.Build(); + await app.RunAsync(); + + Assert.Equal( + "uninstall my-release --namespace my-namespace --ignore-not-found", + Assert.Single( + fakeHelm.Arguments, + arguments => arguments.StartsWith("uninstall", StringComparison.OrdinalIgnoreCase))); + var stateSection = await stateManager.AcquireSectionAsync("Helm:env"); + Assert.Empty(stateSection.Data); + Assert.Contains( + reporter.CompletedSteps, + step => step is ("pipeline-execution", "Completed successfully", CompletionState.Completed)); + } + + [Fact] + public async Task DestroyExternalHelmChart_DoesNotIgnoreUnrelatedFailure() + { + using var workspace = TemporaryWorkspace.Create(outputHelper); + + var fakeHelm = new FakeHelmRunner + { + CommandResultFactory = arguments => arguments.StartsWith("uninstall", StringComparison.OrdinalIgnoreCase) + ? (1, "Error: Kubernetes cluster unreachable") + : (0, null) + }; + var stateManager = new InMemoryDeploymentStateManager(); + stateManager.SetSection("HelmChart:env:podinfo", new JsonObject + { + ["ReleaseName"] = "podinfo", + ["Namespace"] = "podinfo" + }); + + var reporter = new TestPipelineActivityReporter(outputHelper); + var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish, + workspace.Path, + step: WellKnownPipelineSteps.Destroy); + + builder.Services.AddSingleton(); + builder.Services.AddSingleton(reporter); + builder.Services.AddSingleton(stateManager); + builder.Services.AddSingleton(fakeHelm); + builder.Services.Configure(o => o.SkipConfirmation = true); + + builder.AddKubernetesEnvironment("env") + .AddHelmChart("podinfo", "oci://example.com/chart", "1.0.0") + .WithDestroy(); + + using var app = builder.Build(); + await app.RunAsync(); + + var uninstallArguments = Assert.Single( + fakeHelm.Arguments, + arguments => arguments.StartsWith("uninstall", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(" --ignore-not-found", uninstallArguments, StringComparison.Ordinal); + Assert.Equal( + "Step 'helm-uninstall-podinfo' failed: helm uninstall for chart 'podinfo' failed: " + + "Error: Kubernetes cluster unreachable", + reporter.CompletionMessage); + + var stateSection = await stateManager.AcquireSectionAsync("HelmChart:env:podinfo"); + Assert.Equal("podinfo", stateSection.Data["ReleaseName"]?.ToString()); + } } \ No newline at end of file diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/FakeHelmRunner.cs b/tests/Shared/FakeHelmRunner.cs similarity index 80% rename from tests/Aspire.Hosting.Kubernetes.Tests/FakeHelmRunner.cs rename to tests/Shared/FakeHelmRunner.cs index 721f025d98c..dd69f0765a8 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/FakeHelmRunner.cs +++ b/tests/Shared/FakeHelmRunner.cs @@ -1,7 +1,10 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -namespace Aspire.Hosting.Kubernetes.Tests; +using System.Collections.Concurrent; +using Aspire.Hosting.Kubernetes; + +namespace Aspire.Hosting.Tests; /// /// In-memory fake of for tests. Records arguments, @@ -19,8 +22,12 @@ internal sealed class FakeHelmRunner : IHelmRunner public string? LastArguments { get; private set; } + public ConcurrentQueue Arguments { get; } = []; + public int ExitCode { get; set; } + public Func? CommandResultFactory { get; set; } + /// /// Output emitted to onOutputData when arguments start with /// "version". Defaults to a recent stable Helm 4.x release so the @@ -43,6 +50,7 @@ public Task RunAsync( CancellationToken cancellationToken = default) { LastArguments = arguments; + Arguments.Enqueue(arguments); // Match any `helm version ...` probe (the validator passes // `version --short`). @@ -68,6 +76,12 @@ public Task RunAsync( WasUninstallCalled = true; } - return Task.FromResult(ExitCode); + var result = CommandResultFactory?.Invoke(arguments) ?? (ExitCode, null); + if (!string.IsNullOrEmpty(result.StandardError)) + { + onErrorData?.Invoke(result.StandardError); + } + + return Task.FromResult(result.ExitCode); } }