From 487a4097ab52f32679f0d7941ad0f6e419820cdb Mon Sep 17 00:00:00 2001 From: Sri Harsha Date: Fri, 2 Oct 2026 19:02:13 +0000 Subject: [PATCH 1/3] fix fixAzureLinuxV3Gen2Kata --- aks-node-controller/parser/helper_test.go | 54 ++++++++++- .../parser/templates/containerd_v2.toml.gtpl | 17 ++-- .../templates/containerd_v2_no_GPU.toml.gtpl | 13 +-- e2e/scenario/validators_kata.go | 70 +++++++++------ .../linux/cloud-init/artifacts/cse_config.sh | 55 ++++++++++++ pkg/agent/baker.go | 30 ++++--- pkg/agent/baker_test.go | 31 +++++++ pkg/agent/datamodel/types.go | 3 - pkg/agent/datamodel/types_test.go | 38 ++++++++ .../cloud-init/artifacts/cse_config_spec.sh | 90 +++++++++++++++++++ 10 files changed, 342 insertions(+), 59 deletions(-) diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index 434481a2327..e7c997eea75 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -523,7 +523,7 @@ func Test_getContainerdConfigV2(t *testing.T) { }, }, }, - want: base64.StdEncoding.EncodeToString([]byte(`version = 2 + want: base64.StdEncoding.EncodeToString([]byte(`version = 3 oom_score = -999 [plugins."io.containerd.cri.v1.images"] [plugins."io.containerd.cri.v1.images".pinned_images] @@ -559,7 +559,7 @@ oom_score = -999 }, noGpu: false, }, - want: base64.StdEncoding.EncodeToString([]byte(`version = 2 + want: base64.StdEncoding.EncodeToString([]byte(`version = 3 oom_score = -999 [plugins."io.containerd.cri.v1.images"] [plugins."io.containerd.cri.v1.images".pinned_images] @@ -594,7 +594,7 @@ oom_score = -999 }, noGpu: true, }, - want: base64.StdEncoding.EncodeToString([]byte(`version = 2 + want: base64.StdEncoding.EncodeToString([]byte(`version = 3 oom_score = -999 [plugins."io.containerd.cri.v1.images"] [plugins."io.containerd.cri.v1.images".pinned_images] @@ -665,6 +665,54 @@ oom_score = -999 } } +func Test_getContainerdConfigV2KataUsesNativePluginPaths(t *testing.T) { + config := &aksnodeconfigv1.Configuration{ + IsKata: true, + ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ + ContainerdVersion: "2.3.4", + }, + } + + tests := []struct { + name string + getConfig func(*aksnodeconfigv1.Configuration, string) string + }{ + { + name: "default", + getConfig: getContainerdConfigBase64, + }, + { + name: "no GPU", + getConfig: getNoGPUContainerdConfigBase64, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + decoded, err := base64.StdEncoding.DecodeString(test.getConfig(config, "2.3.4")) + if err != nil { + t.Fatalf("decoding containerd config: %v", err) + } + rendered := string(decoded) + if !strings.HasPrefix(rendered, "version = 3\n") { + t.Fatalf("containerd v2 config does not use schema 3:\n%s", rendered) + } + if strings.Contains(rendered, `io.containerd.grpc.v1.cri`) { + t.Fatalf("containerd v2 config contains the legacy CRI plugin path:\n%s", rendered) + } + if !strings.Contains(rendered, "disable_snapshot_annotations = false") { + t.Fatalf("containerd v2 Kata config does not disable snapshot annotations:\n%s", rendered) + } + for _, handler := range []string{"kata", "kata-v2"} { + expected := `plugins."io.containerd.cri.v1.runtime".containerd.runtimes.` + handler + `]` + if !strings.Contains(rendered, expected) { + t.Fatalf("containerd v2 config does not contain %q:\n%s", expected, rendered) + } + } + }) + } +} + func Test_getKubenetTemplate(t *testing.T) { tests := []struct { name string diff --git a/aks-node-controller/parser/templates/containerd_v2.toml.gtpl b/aks-node-controller/parser/templates/containerd_v2.toml.gtpl index c7c99583985..87f2e76daf6 100644 --- a/aks-node-controller/parser/templates/containerd_v2.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_v2.toml.gtpl @@ -1,4 +1,4 @@ -version = 2 +version = 3 oom_score = -999{{if getHasDataDir .KubeletConfig}} root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} {{- if .GetIsKata }} @@ -15,6 +15,9 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} enable_tar_index = false {{- end}} [plugins."io.containerd.cri.v1.images"] +{{- if .GetIsKata }} + disable_snapshot_annotations = false +{{- end}} {{- if .GetEnableArtifactStreaming }} snapshotter = "overlaybd" disable_snapshot_annotations = false @@ -66,29 +69,29 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} address = "/run/overlaybd-snapshotter/overlaybd.sock" {{- end}} {{- if .GetIsKata }} -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] +[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true snapshotter = "overlayfs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-v2] +[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-v2] runtime_path = "/usr/local/bin/containerd-shim-kata-v2-rs" runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true pod_annotations = ["io.katacontainers.snapshot-name"] snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-v2.options] + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-v2.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-azure-runtime-rs-v2.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" address = "/run/containerd/tardev-snapshotter.sock" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc] +[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-cc] snapshotter = "tardev" runtime_type = "io.containerd.kata-cc.v2" privileged_without_host_devices = true pod_annotations = ["io.katacontainers.*"] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc.options] + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-cc.options] ConfigPath = "/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml" {{- end}} diff --git a/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl b/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl index a0d5f78bd56..38af87dc3e3 100644 --- a/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl @@ -1,4 +1,4 @@ -version = 2 +version = 3 oom_score = -999{{if getHasDataDir .KubeletConfig}} root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} {{- if .GetIsKata }} @@ -15,6 +15,9 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} enable_tar_index = false {{- end}} [plugins."io.containerd.cri.v1.images"] +{{- if .GetIsKata }} + disable_snapshot_annotations = false +{{- end}} {{- if .GetEnableArtifactStreaming }} snapshotter = "overlaybd" disable_snapshot_annotations = false @@ -53,19 +56,19 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} address = "/run/overlaybd-snapshotter/overlaybd.sock" {{- end}} {{- if .GetIsKata }} -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] +[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true snapshotter = "overlayfs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-v2] +[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-v2] runtime_path = "/usr/local/bin/containerd-shim-kata-v2-rs" runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true pod_annotations = ["io.katacontainers.snapshot-name"] snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-v2.options] + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-v2.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-azure-runtime-rs-v2.toml" [proxy_plugins] [proxy_plugins.tardev] diff --git a/e2e/scenario/validators_kata.go b/e2e/scenario/validators_kata.go index c39cdc6b45f..b25a780a23c 100644 --- a/e2e/scenario/validators_kata.go +++ b/e2e/scenario/validators_kata.go @@ -41,35 +41,55 @@ const ( // cannot actually run it. var kataRuntimeHandlers = []string{kataRuntimeHandler, kataV2RuntimeHandler} -// ValidateKataContainerdConfig asserts that AgentBaker rendered a containerd configuration -// containing the Kata runtime handlers on a Kata-enabled VHD. +// ValidateKataContainerdConfig asserts that the normalized containerd configuration contains +// the Kata runtime handlers on a Kata-enabled VHD. // // This is the core regression check for the IsKata blocks of the containerd config templates in -// pkg/agent/baker.go. Note that AgentPoolProfile.IsContainerdV2Distro() returns false for every -// Kata distro (pkg/agent/datamodel/types.go), so Kata nodes are always rendered from -// containerdV1ConfigTemplate / containerdV1NoGPUConfigTemplate regardless of the underlying OS. -// The assertions below therefore target the containerd 1.x plugin paths that those templates -// emit. If Kata is ever promoted to the V2 templates, this validator should fail loudly rather -// than silently pass, which is why the plugin paths are asserted explicitly. +// pkg/agent/baker.go. Azure Linux 3 Kata uses the containerd 2.x split CRI plugin paths, while +// older Kata images continue to use the legacy combined CRI plugin. func ValidateKataContainerdConfig(ctx context.Context, s *Scenario) error { if err := assert.Equal(s.VHD.Distro.IsKataDistro(), true, "ValidateKataContainerdConfig requires a Kata distro, got %q", s.VHD.Distro); err != nil { return err } + execResult, err := execScriptOnVMForScenarioValidateExitCode(ctx, s, + "sudo cat "+containerdConfigPath, 0, "unable to read the containerd config on the node") + if err != nil { + return err + } + + // containerd config migrate may reserialize quoted TOML table components and values without + // changing their meaning. Remove both valid TOML quote styles before asserting content. + normalizedConfig := strings.NewReplacer(`"`, "", `'`, "").Replace(execResult.stdout) + + runtimePlugin := "io.containerd.grpc.v1.cri" + if s.VHD.Distro.IsAzureLinuxV3Distro() { + runtimePlugin = "io.containerd.cri.v1.runtime" + } + runtimeHandlersPath := `[plugins.` + runtimePlugin + `.containerd.runtimes.` + return errors.Join( // The standard "kata" runtime handler, backed by the kata v2 shim. - ValidateFileHasContent(ctx, s, containerdConfigPath, `[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]`), - ValidateFileHasContent(ctx, s, containerdConfigPath, `runtime_type = "io.containerd.kata.v2"`), - ValidateFileHasContent(ctx, s, containerdConfigPath, kataConfigPath), - ValidateFileHasContent(ctx, s, containerdConfigPath, `[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-v2]`), - ValidateFileHasContent(ctx, s, containerdConfigPath, `pod_annotations = ["io.katacontainers.snapshot-name"]`), - ValidateFileHasContent(ctx, s, containerdConfigPath, `runtime_path = "`+kataV2ShimBinaryPath+`"`), - ValidateFileHasContent(ctx, s, containerdConfigPath, kataV2ConfigPath), + assert.Contains(normalizedConfig, runtimeHandlersPath+`kata]`, + "expected the kata runtime handler in %s.\nConfig:\n%s", containerdConfigPath, execResult.stdout), + assert.Contains(normalizedConfig, `runtime_type = io.containerd.kata.v2`, + "expected the kata v2 shim runtime_type in %s.\nConfig:\n%s", containerdConfigPath, execResult.stdout), + assert.Contains(normalizedConfig, kataConfigPath, + "expected the kata config path in %s.\nConfig:\n%s", containerdConfigPath, execResult.stdout), + assert.Contains(normalizedConfig, runtimeHandlersPath+`kata-v2]`, + "expected the kata-v2 runtime handler in %s.\nConfig:\n%s", containerdConfigPath, execResult.stdout), + assert.Contains(normalizedConfig, `pod_annotations = [io.katacontainers.snapshot-name]`, + "expected the kata snapshot annotation in %s.\nConfig:\n%s", containerdConfigPath, execResult.stdout), + assert.Contains(normalizedConfig, `runtime_path = `+kataV2ShimBinaryPath, + "expected the kata-v2 shim path in %s.\nConfig:\n%s", containerdConfigPath, execResult.stdout), + assert.Contains(normalizedConfig, kataV2ConfigPath, + "expected the kata-v2 config path in %s.\nConfig:\n%s", containerdConfigPath, execResult.stdout), // Kata relies on snapshot annotations being forwarded to the snapshotter; the template sets // this explicitly under IsKata and disabling it breaks image pulling for Kata pods. - ValidateFileHasContent(ctx, s, containerdConfigPath, "disable_snapshot_annotations = false"), + assert.Contains(normalizedConfig, "disable_snapshot_annotations = false", + "expected snapshot annotations to be enabled in %s.\nConfig:\n%s", containerdConfigPath, execResult.stdout), ) } @@ -105,17 +125,9 @@ func ValidateKataErofsContainerdConfig(ctx context.Context, s *Scenario) error { // // Checking the file alone is not enough. Kata VHDs ship their own containerd build - CSE skips // installing one (see the "azurelinuxkata" entries in parts/common/components.json) - so the -// containerd major version on the node is decided by the image, not by AgentBaker, while the -// template AgentBaker renders is decided by the distro (IsContainerdV2Distro short-circuits to -// the v1 template for every Kata distro). The two can therefore disagree: AzureLinux V3 Kata -// currently boots containerd 2.x while being handed a containerd 1.x style config. -// -// That combination happens to work today because containerd 2.x migrates the legacy -// "io.containerd.grpc.v1.cri" runtime handlers onto the current -// "io.containerd.cri.v1.runtime" paths, but nothing guarantees it keeps doing so. This -// validator pins the property we actually care about: after containerd has parsed the config, -// the Kata handlers are present in the effective configuration and containerd raised no -// warnings while getting there. +// installed binary is the authority for the native config schema. This validator pins the +// property we actually care about: after containerd has parsed the normalized config, the Kata +// handlers are present in the effective configuration and containerd raised no warnings. func ValidateKataContainerdConfigDump(ctx context.Context, s *Scenario) error { // This must run on the node itself, not in a debug pod. The "debugnonhost" daemonset pods // used by execOnVMForScenarioOnUnprivilegedPod run a bare CBL-Mariner base image with no @@ -150,8 +162,8 @@ func ValidateKataContainerdConfigDump(ctx context.Context, s *Scenario) error { errs = append(errs, assert.Contains(normalizedDump, `runtime_type = "io.containerd.kata.v2"`, "expected the kata v2 shim runtime_type in the effective containerd config.\nDump:\n%s", dump)) - // A warning here means containerd did not fully understand the config we generated, e.g. it - // had to fall back on deprecated handling for the legacy plugin paths the Kata templates use. + // A warning here means the on-node normalization did not produce a native configuration for + // the installed containerd version. errs = append(errs, assert.NotContains(diagnostics, "level=warning", "containerd reported warnings while parsing the AgentBaker-generated config.\nstdout:\n%s\nstderr:\n%s", execResult.stdout, execResult.stderr)) diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 0d64344d7b4..5d1316f642b 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -531,7 +531,55 @@ EOF fi } +migrateContainerdConfigToInstalledVersion() { + local config_path="${1:-/etc/containerd/config.toml}" + local migrated_path="${config_path}.migrated" + local containerd_version_output + local containerd_version + + if ! containerd_version_output="$(containerd --version)"; then + echo "Failed to detect the installed containerd version" >&2 + return 1 + fi + + containerd_version="$(printf '%s\n' "${containerd_version_output}" | awk '{print $3}')" + containerd_version="${containerd_version#v}" + if [ -z "${containerd_version}" ]; then + echo "Unable to parse the installed containerd version from: ${containerd_version_output}" >&2 + return 1 + fi + + if ! semverCompare "${containerd_version}" "2.3.0"; then + return 0 + fi + + echo "Migrating containerd config to the schema supported by containerd ${containerd_version}" + rm -f "${migrated_path}" + if ! containerd --config "${config_path}" config migrate > "${migrated_path}"; then + rm -f "${migrated_path}" + echo "Failed to migrate containerd config for containerd ${containerd_version}" >&2 + return 1 + fi + if ! containerd --config "${migrated_path}" config dump > /dev/null; then + rm -f "${migrated_path}" + echo "Migrated containerd config is invalid for containerd ${containerd_version}" >&2 + return 1 + fi + if ! chmod 0644 "${migrated_path}"; then + rm -f "${migrated_path}" + echo "Failed to set permissions on the migrated containerd config" >&2 + return 1 + fi + if ! mv -f "${migrated_path}" "${config_path}"; then + rm -f "${migrated_path}" + echo "Failed to replace the containerd config with the migrated config" >&2 + return 1 + fi +} + ensureContainerd() { + local containerd_config_generated=false + mkdir -p "/etc/systemd/system/containerd.service.d" # Explicitly set LimitNOFILE=1048576 (the value that 'infinity' resolves to on Ubuntu 22.04) for both Ubuntu and Mariner/AzureLinux. # On Ubuntu 24.04 (Containerd 2.0), LimitNOFILE is removed upstream and systemd falls back to an implicit soft:hard limit @@ -559,16 +607,23 @@ EOF if [ "$?" -eq 0 ] && [ "${should_skip}" = "true" ]; then echo "Generating non-GPU containerd config for GPU node due to VM tags" echo "${CONTAINERD_CONFIG_NO_GPU_CONTENT}" | base64 -d > /etc/containerd/config.toml || exit $ERR_FILE_WATCH_TIMEOUT + containerd_config_generated=true else echo "Generating GPU containerd config..." echo "${CONTAINERD_CONFIG_CONTENT}" | base64 -d > /etc/containerd/config.toml || exit $ERR_FILE_WATCH_TIMEOUT + containerd_config_generated=true fi else echo "Generating containerd config..." echo "${CONTAINERD_CONFIG_CONTENT}" | base64 -d > /etc/containerd/config.toml || exit $ERR_FILE_WATCH_TIMEOUT + containerd_config_generated=true fi fi + if [ "${containerd_config_generated}" = "true" ]; then + migrateContainerdConfigToInstalledVersion /etc/containerd/config.toml || exit $ERR_CONTAINERD_VERSION_INVALID + fi + export -f should_e2e_mock_azure_china_cloud E2EMockAzureChinaCloud=$(should_e2e_mock_azure_china_cloud) if [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]; then diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index b8812c4c60a..2e644a979c8 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -2075,7 +2075,7 @@ root = "{{GetDataDir}}"{{- end}} ConfigPath = "/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml" {{- end}} ` - containerdV2ConfigTemplate ContainerdConfigTemplate = `version = 2 + containerdV2ConfigTemplate ContainerdConfigTemplate = `version = 3 oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} {{- if IsKata }} @@ -2092,6 +2092,9 @@ root = "{{GetDataDir}}"{{- end}} enable_tar_index = false {{- end}} [plugins."io.containerd.cri.v1.images"] +{{- if IsKata }} + disable_snapshot_annotations = false +{{- end}} {{- if IsArtifactStreamingEnabled }} snapshotter = "overlaybd" disable_snapshot_annotations = false @@ -2143,34 +2146,34 @@ root = "{{GetDataDir}}"{{- end}} address = "/run/overlaybd-snapshotter/overlaybd.sock" {{- end}} {{- if IsKata }} -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] +[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true snapshotter = "overlayfs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-v2] +[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-v2] runtime_path = "/usr/local/bin/containerd-shim-kata-v2-rs" runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true pod_annotations = ["io.katacontainers.snapshot-name"] snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-v2.options] + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-v2.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-azure-runtime-rs-v2.toml" [proxy_plugins] [proxy_plugins.tardev] type = "snapshot" address = "/run/containerd/tardev-snapshotter.sock" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc] +[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-cc] snapshotter = "tardev" runtime_type = "io.containerd.kata-cc.v2" privileged_without_host_devices = true pod_annotations = ["io.katacontainers.*"] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc.options] + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-cc.options] ConfigPath = "/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml" {{- end}} ` - containerdV2NoGPUConfigTemplate ContainerdConfigTemplate = `version = 2 + containerdV2NoGPUConfigTemplate ContainerdConfigTemplate = `version = 3 oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} {{- if IsKata }} @@ -2187,6 +2190,9 @@ root = "{{GetDataDir}}"{{- end}} enable_tar_index = false {{- end}} [plugins."io.containerd.cri.v1.images"] +{{- if IsKata }} + disable_snapshot_annotations = false +{{- end}} {{- if IsArtifactStreamingEnabled }} snapshotter = "overlaybd" disable_snapshot_annotations = false @@ -2225,19 +2231,19 @@ root = "{{GetDataDir}}"{{- end}} address = "/run/overlaybd-snapshotter/overlaybd.sock" {{- end}} {{- if IsKata }} -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] +[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true snapshotter = "overlayfs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-v2] +[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-v2] runtime_path = "/usr/local/bin/containerd-shim-kata-v2-rs" runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true pod_annotations = ["io.katacontainers.snapshot-name"] snapshotter = "erofs" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-v2.options] + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-v2.options] ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-azure-runtime-rs-v2.toml" [proxy_plugins] [proxy_plugins.tardev] diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index 764b253d47d..2f71be39bd3 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -47,6 +47,37 @@ health-check.localdns.local:53 { # KubeDNS overrides apply to DNS traffic from pods with dnsPolicy:ClusterFirst (referred to as KubeDNS traffic). ` +func TestContainerdV2TemplatesUseNativeKataPluginPaths(t *testing.T) { + tests := []struct { + name string + template ContainerdConfigTemplate + handlers []string + }{ + { + name: "default", + template: containerdV2ConfigTemplate, + handlers: []string{"kata", "kata-v2", "kata-cc"}, + }, + { + name: "no GPU", + template: containerdV2NoGPUConfigTemplate, + handlers: []string{"kata", "kata-v2"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + config := string(test.template) + require.True(t, strings.HasPrefix(config, "version = 3\n")) + require.NotContains(t, config, `io.containerd.grpc.v1.cri`) + require.Contains(t, config, "disable_snapshot_annotations = false") + for _, handler := range test.handlers { + require.Contains(t, config, `plugins."io.containerd.cri.v1.runtime".containerd.runtimes.`+handler+`]`) + } + }) + } +} + func TestRenderLinuxNodeCustomDataTemplateUsesBakerPlatformFunctions(t *testing.T) { template := []byte(`#cloud-config write_files: diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go index c271a86c94b..4409d897f05 100644 --- a/pkg/agent/datamodel/types.go +++ b/pkg/agent/datamodel/types.go @@ -1205,9 +1205,6 @@ func (a *AgentPoolProfile) Is2604VHDDistro() bool { } func (a *AgentPoolProfile) IsContainerdV2Distro() bool { - if a.Distro.IsKataDistro() { - return false - } return a.Distro.Is2604VHDDistro() || a.Distro.Is2404VHDDistro() || a.Distro.IsACLDistro() || a.Distro.IsAzureLinuxV3Distro() } diff --git a/pkg/agent/datamodel/types_test.go b/pkg/agent/datamodel/types_test.go index 5753b17141d..0ce6ed69f52 100644 --- a/pkg/agent/datamodel/types_test.go +++ b/pkg/agent/datamodel/types_test.go @@ -1096,6 +1096,44 @@ func TestAgentPoolProfileIs2604VHDDistro(t *testing.T) { } } +func TestAgentPoolProfileIsContainerdV2Distro(t *testing.T) { + tests := []struct { + name string + distro Distro + expected bool + }{ + { + name: "Azure Linux V3 Kata", + distro: AKSAzureLinuxV3Gen2Kata, + expected: true, + }, + { + name: "Azure Linux V2 Kata", + distro: AKSAzureLinuxV2Gen2Kata, + expected: false, + }, + { + name: "Ubuntu 24.04", + distro: AKSUbuntuContainerd2404, + expected: true, + }, + { + name: "Ubuntu 22.04", + distro: AKSUbuntuContainerd2204, + expected: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + profile := AgentPoolProfile{Distro: test.distro} + if actual := profile.IsContainerdV2Distro(); actual != test.expected { + t.Fatalf("IsContainerdV2Distro() = %t, want %t", actual, test.expected) + } + }) + } +} + func TestAgentPoolProfileIsAzureLinuxCgroupV2VHDDistro(t *testing.T) { cases := []struct { name string diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh index 1df096ea45c..06e69248f98 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh @@ -463,6 +463,96 @@ PubkeyAuthentication no" The status should be success End End + Describe 'migrateContainerdConfigToInstalledVersion' + setup_containerd_migration() { + TEST_CONTAINERD_CONFIG_DIR="$(mktemp -d)" + TEST_CONTAINERD_CONFIG="${TEST_CONTAINERD_CONFIG_DIR}/config.toml" + printf 'version = 3\n' > "${TEST_CONTAINERD_CONFIG}" + CONTAINERD_VERSION_OUTPUT='containerd github.com/containerd/containerd/v2 v2.3.4 deadbeef' + CONTAINERD_VERSION_EXIT=0 + CONTAINERD_MIGRATE_EXIT=0 + CONTAINERD_DUMP_EXIT=0 + CONTAINERD_MIGRATED_CONFIG='version = 4' + } + + cleanup_containerd_migration() { + rm -rf "${TEST_CONTAINERD_CONFIG_DIR}" + unset TEST_CONTAINERD_CONFIG_DIR TEST_CONTAINERD_CONFIG + unset CONTAINERD_VERSION_OUTPUT CONTAINERD_VERSION_EXIT + unset CONTAINERD_MIGRATE_EXIT CONTAINERD_DUMP_EXIT CONTAINERD_MIGRATED_CONFIG + } + + containerd() { + if [ "${1:-}" = "--version" ]; then + printf '%s\n' "${CONTAINERD_VERSION_OUTPUT}" + return "${CONTAINERD_VERSION_EXIT}" + fi + if [ "${3:-}" = "config" ] && [ "${4:-}" = "migrate" ]; then + printf '%s\n' "${CONTAINERD_MIGRATED_CONFIG}" + return "${CONTAINERD_MIGRATE_EXIT}" + fi + if [ "${3:-}" = "config" ] && [ "${4:-}" = "dump" ]; then + return "${CONTAINERD_DUMP_EXIT}" + fi + return 1 + } + + BeforeEach 'setup_containerd_migration' + AfterEach 'cleanup_containerd_migration' + + It 'leaves containerd 1.x configs unchanged' + CONTAINERD_VERSION_OUTPUT='containerd containerd.io 1.7.28 deadbeef' + + When call migrateContainerdConfigToInstalledVersion "${TEST_CONTAINERD_CONFIG}" + + The status should be success + The contents of file "${TEST_CONTAINERD_CONFIG}" should equal 'version = 3' + The path "${TEST_CONTAINERD_CONFIG}.migrated" should not be exist + End + + It 'leaves the native schema 3 config unchanged for containerd 2.2' + CONTAINERD_VERSION_OUTPUT='containerd github.com/containerd/containerd/v2 v2.2.4 deadbeef' + + When call migrateContainerdConfigToInstalledVersion "${TEST_CONTAINERD_CONFIG}" + + The status should be success + The contents of file "${TEST_CONTAINERD_CONFIG}" should equal 'version = 3' + The path "${TEST_CONTAINERD_CONFIG}.migrated" should not be exist + End + + It 'atomically installs the native config generated by containerd 2.x' + When call migrateContainerdConfigToInstalledVersion "${TEST_CONTAINERD_CONFIG}" + + The status should be success + The output should include 'Migrating containerd config to the schema supported by containerd 2.3.4' + The contents of file "${TEST_CONTAINERD_CONFIG}" should equal 'version = 4' + The path "${TEST_CONTAINERD_CONFIG}.migrated" should not be exist + End + + It 'keeps the original config when migration fails' + CONTAINERD_MIGRATE_EXIT=1 + + When call migrateContainerdConfigToInstalledVersion "${TEST_CONTAINERD_CONFIG}" + + The status should be failure + The output should include 'Migrating containerd config to the schema supported by containerd 2.3.4' + The contents of file "${TEST_CONTAINERD_CONFIG}" should equal 'version = 3' + The path "${TEST_CONTAINERD_CONFIG}.migrated" should not be exist + The error should include 'Failed to migrate containerd config for containerd 2.3.4' + End + + It 'keeps the original config when the migrated config is invalid' + CONTAINERD_DUMP_EXIT=1 + + When call migrateContainerdConfigToInstalledVersion "${TEST_CONTAINERD_CONFIG}" + + The status should be failure + The output should include 'Migrating containerd config to the schema supported by containerd 2.3.4' + The contents of file "${TEST_CONTAINERD_CONFIG}" should equal 'version = 3' + The path "${TEST_CONTAINERD_CONFIG}.migrated" should not be exist + The error should include 'Migrated containerd config is invalid for containerd 2.3.4' + End + End Describe 'ensureContainerd' It 'should not overwrite an existing NVIDIA containerd config' grep() { From 953a00a8a8080142f410d9c2495081815a91f932 Mon Sep 17 00:00:00 2001 From: Sri Harsha Date: Fri, 2 Oct 2026 15:20:57 -0700 Subject: [PATCH 2/3] fix: render native containerd 2.3 config schema Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- aks-node-controller/parser/helper.go | 22 ++++++++++++++- aks-node-controller/parser/helper_test.go | 27 +++++++++++++++++-- aks-node-controller/parser/parser_test.go | 1 + .../parser/templates/containerd_v2.toml.gtpl | 2 +- .../templates/containerd_v2_no_GPU.toml.gtpl | 2 +- 5 files changed, 49 insertions(+), 5 deletions(-) diff --git a/aks-node-controller/parser/helper.go b/aks-node-controller/parser/helper.go index bf8f25ffd62..ddc442831e2 100644 --- a/aks-node-controller/parser/helper.go +++ b/aks-node-controller/parser/helper.go @@ -89,6 +89,7 @@ func getFuncMapForContainerdConfigTemplate() template.FuncMap { "isKubernetesVersionGe": IsKubernetesVersionGe, "getHasDataDir": getHasDataDir, "getEnableNvidia": getEnableNvidia, + "getContainerdConfigVersion": func() int { return 3 }, } } @@ -220,14 +221,33 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat } } + renderTemplate, err := _template.Clone() + if err != nil { + return "", fmt.Errorf("error cloning containerd config template: %w", err) + } + configVersion := getContainerdConfigVersion(containerdVersion) + renderTemplate = renderTemplate.Funcs(template.FuncMap{ + "getContainerdConfigVersion": func() int { return configVersion }, + }) + var buffer bytes.Buffer - if err := _template.Execute(&buffer, aksnodeconfig); err != nil { + if err := renderTemplate.Execute(&buffer, aksnodeconfig); err != nil { return "", fmt.Errorf("error executing containerd config template for AKSNodeConfig: %w", err) } return buffer.String(), nil } +func getContainerdConfigVersion(containerdVersion string) int { + if IsKubernetesVersionGe(containerdVersion, "2.3.0") { + return 4 + } + if IsKubernetesVersionGe(containerdVersion, "2.0.0") { + return 3 + } + return 2 +} + // detectContainerdVersion runs "containerd --version" and parses the version string. // The expected output format is: "containerd " // e.g. "containerd containerd.io 1.7.22 c814c75..." or "containerd github.com/containerd/containerd/v2 v2.0.0 ..." diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index e7c997eea75..931bbee919e 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -694,8 +694,8 @@ func Test_getContainerdConfigV2KataUsesNativePluginPaths(t *testing.T) { t.Fatalf("decoding containerd config: %v", err) } rendered := string(decoded) - if !strings.HasPrefix(rendered, "version = 3\n") { - t.Fatalf("containerd v2 config does not use schema 3:\n%s", rendered) + if !strings.HasPrefix(rendered, "version = 4\n") { + t.Fatalf("containerd 2.3+ config does not use schema 4:\n%s", rendered) } if strings.Contains(rendered, `io.containerd.grpc.v1.cri`) { t.Fatalf("containerd v2 config contains the legacy CRI plugin path:\n%s", rendered) @@ -713,6 +713,29 @@ func Test_getContainerdConfigV2KataUsesNativePluginPaths(t *testing.T) { } } +func TestGetContainerdConfigVersion(t *testing.T) { + tests := []struct { + name string + containerdVersion string + want int + }{ + {name: "unknown version", containerdVersion: "", want: 2}, + {name: "containerd 1", containerdVersion: "1.7.22", want: 2}, + {name: "containerd 2.0", containerdVersion: "2.0.0", want: 3}, + {name: "containerd 2.2", containerdVersion: "2.2.4", want: 3}, + {name: "containerd 2.3", containerdVersion: "2.3.0", want: 4}, + {name: "containerd 2.3 patch", containerdVersion: "2.3.4", want: 4}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getContainerdConfigVersion(tt.containerdVersion); got != tt.want { + t.Fatalf("getContainerdConfigVersion(%q) = %d, want %d", tt.containerdVersion, got, tt.want) + } + }) + } +} + func Test_getKubenetTemplate(t *testing.T) { tests := []struct { name string diff --git a/aks-node-controller/parser/parser_test.go b/aks-node-controller/parser/parser_test.go index 9a2341f67ed..0e9879b3e9a 100644 --- a/aks-node-controller/parser/parser_test.go +++ b/aks-node-controller/parser/parser_test.go @@ -740,6 +740,7 @@ func TestBuildCSECmd_DetectsContainerdV2FromSystem(t *testing.T) { // Verify the v2 containerd config template was used (uses "io.containerd.cri.v1.images" path). containerdConfig, err := getBase64DecodedValue([]byte(vars["CONTAINERD_CONFIG_NO_GPU_CONTENT"])) require.NoError(t, err) + assert.True(t, strings.HasPrefix(containerdConfig, "version = 4\n"), containerdConfig) assert.Contains(t, containerdConfig, `plugins."io.containerd.cri.v1.images"`) assert.NotContains(t, containerdConfig, `plugins."io.containerd.grpc.v1.cri"`) } diff --git a/aks-node-controller/parser/templates/containerd_v2.toml.gtpl b/aks-node-controller/parser/templates/containerd_v2.toml.gtpl index 87f2e76daf6..acddf11054d 100644 --- a/aks-node-controller/parser/templates/containerd_v2.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_v2.toml.gtpl @@ -1,4 +1,4 @@ -version = 3 +version = {{getContainerdConfigVersion}} oom_score = -999{{if getHasDataDir .KubeletConfig}} root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} {{- if .GetIsKata }} diff --git a/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl b/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl index 38af87dc3e3..2862816c83d 100644 --- a/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl @@ -1,4 +1,4 @@ -version = 3 +version = {{getContainerdConfigVersion}} oom_score = -999{{if getHasDataDir .KubeletConfig}} root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} {{- if .GetIsKata }} From 31df3c1da50a65a4f3ec7db849adde5efef11346 Mon Sep 17 00:00:00 2001 From: Sri Harsha Date: Fri, 2 Oct 2026 15:43:38 -0700 Subject: [PATCH 3/3] fix: make containerd config schema version-aware Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- aks-node-controller/parser/helper.go | 71 +++++++--- aks-node-controller/parser/helper_test.go | 38 ++++-- aks-node-controller/parser/parser.go | 15 +- aks-node-controller/parser/parser_test.go | 32 +++++ .../parser/templates/containerd_v2.toml.gtpl | 2 +- .../templates/containerd_v2_no_GPU.toml.gtpl | 2 +- e2e/components/components_test.go | 4 + e2e/scenario/containerd_config_parity_test.go | 98 ++++++++++++++ e2e/scenario/node_config.go | 11 ++ e2e/scenario/scenario.go | 5 + e2e/scenario/validators_kata.go | 9 +- parts/common/components.json | 4 +- pkg/agent/baker.go | 38 +++--- pkg/agent/baker_test.go | 2 +- pkg/agent/bakerapi_test.go | 1 + pkg/agent/containerd_config.go | 92 +++++++++++++ pkg/agent/containerd_config_test.go | 128 ++++++++++++++++++ .../cloud-init/artifacts/cse_install_spec.sh | 7 +- .../cloud-init/artifacts/test_components.json | 2 +- 19 files changed, 496 insertions(+), 65 deletions(-) create mode 100644 e2e/scenario/containerd_config_parity_test.go create mode 100644 pkg/agent/containerd_config.go create mode 100644 pkg/agent/containerd_config_test.go diff --git a/aks-node-controller/parser/helper.go b/aks-node-controller/parser/helper.go index ddc442831e2..a854031ed3c 100644 --- a/aks-node-controller/parser/helper.go +++ b/aks-node-controller/parser/helper.go @@ -89,7 +89,7 @@ func getFuncMapForContainerdConfigTemplate() template.FuncMap { "isKubernetesVersionGe": IsKubernetesVersionGe, "getHasDataDir": getHasDataDir, "getEnableNvidia": getEnableNvidia, - "getContainerdConfigVersion": func() int { return 3 }, + "getContainerdConfigSchema": func() int { return 3 }, } } @@ -205,11 +205,16 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat return "", fmt.Errorf("AKSNodeConfig is nil") } + schema, err := containerdConfigSchema(containerdVersion) + if err != nil { + return "", err + } + // Select the appropriate containerd config template based on version and GPU presence. // Containerd 2.x uses different CRI plugin paths (io.containerd.cri.v1.images/runtime) // compared to containerd 1.x (io.containerd.grpc.v1.cri). var _template *template.Template - if isContainerdV2(containerdVersion) { + if schema >= 3 { _template = containerdV2ConfigTemplate if noGPU { _template = containerdV2ConfigNoGPUTemplate @@ -225,9 +230,8 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat if err != nil { return "", fmt.Errorf("error cloning containerd config template: %w", err) } - configVersion := getContainerdConfigVersion(containerdVersion) renderTemplate = renderTemplate.Funcs(template.FuncMap{ - "getContainerdConfigVersion": func() int { return configVersion }, + "getContainerdConfigSchema": func() int { return schema }, }) var buffer bytes.Buffer @@ -238,14 +242,29 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat return buffer.String(), nil } -func getContainerdConfigVersion(containerdVersion string) int { - if IsKubernetesVersionGe(containerdVersion, "2.3.0") { - return 4 +func semverGE(version, minimum string) bool { + actualVersion, err := semver.NewVersion(version) + if err != nil { + return false } - if IsKubernetesVersionGe(containerdVersion, "2.0.0") { - return 3 + minimumVersion, err := semver.NewVersion(minimum) + if err != nil { + return false + } + return !actualVersion.LessThan(minimumVersion) +} + +func containerdConfigSchema(version string) (int, error) { + switch { + case semverGE(version, "2.3.0"): + return 4, nil + case semverGE(version, "2.0.0"): + return 3, nil + case semverGE(version, "1.0.0"): + return 2, nil + default: + return 0, fmt.Errorf("unsupported or missing containerd version %q", version) } - return 2 } // detectContainerdVersion runs "containerd --version" and parses the version string. @@ -257,17 +276,33 @@ func detectContainerdVersion(ctx context.Context) (string, error) { if err != nil { return "", fmt.Errorf("running containerd --version: %w", err) } - return parseContainerdVersionOutput(string(out)), nil + version := parseContainerdVersionOutput(string(out)) + if version == "" { + return "", fmt.Errorf("parsing containerd version output %q", strings.TrimSpace(string(out))) + } + return version, nil } -// isContainerdV2 returns true if the containerd version string indicates a 2.x release. -// Containerd 2.x uses different CRI plugin paths (io.containerd.cri.v1.images and -// io.containerd.cri.v1.runtime) compared to 1.x (io.containerd.grpc.v1.cri). -func isContainerdV2(version string) bool { - if version == "" { - return false +func resolveContainerdConfigVersion(ctx context.Context, config *aksnodeconfigv1.Configuration) (string, error) { + detectedVersion, detectionErr := detectContainerdVersion(ctx) + if detectionErr == nil { + if _, err := containerdConfigSchema(detectedVersion); err != nil { + return "", fmt.Errorf("detected containerd version: %w", err) + } + return detectedVersion, nil + } + + configuredVersion := strings.TrimSpace(config.GetContainerdConfig().GetContainerdVersion()) + if configuredVersion != "" { + if _, err := containerdConfigSchema(configuredVersion); err != nil { + return "", fmt.Errorf("configured containerd version: %w", err) + } + return configuredVersion, nil + } + if config.GetIsKata() { + return "", fmt.Errorf("containerd version is required for Kata and runtime detection failed: %w", detectionErr) } - return IsKubernetesVersionGe(version, "2.0.0") + return "1.0.0", nil } func getIsMIGNode(gpuInstanceProfile string, migProfileLayout []string) bool { diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index 931bbee919e..01f82458661 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -497,7 +497,7 @@ oom_score = -999 } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := getContainerdConfigBase64(tt.args.aksnodeconfig, ""); got != tt.want { + if got := getContainerdConfigBase64(tt.args.aksnodeconfig, "1.7.22"); got != tt.want { t.Errorf("getContainerdConfig() = %v, want %v", got, tt.want) } }) @@ -713,24 +713,36 @@ func Test_getContainerdConfigV2KataUsesNativePluginPaths(t *testing.T) { } } -func TestGetContainerdConfigVersion(t *testing.T) { +func TestContainerdConfigSchema(t *testing.T) { tests := []struct { - name string - containerdVersion string - want int + name string + version string + want int + wantErr string }{ - {name: "unknown version", containerdVersion: "", want: 2}, - {name: "containerd 1", containerdVersion: "1.7.22", want: 2}, - {name: "containerd 2.0", containerdVersion: "2.0.0", want: 3}, - {name: "containerd 2.2", containerdVersion: "2.2.4", want: 3}, - {name: "containerd 2.3", containerdVersion: "2.3.0", want: 4}, - {name: "containerd 2.3 patch", containerdVersion: "2.3.4", want: 4}, + {name: "containerd 1", version: "1.7.22", want: 2}, + {name: "containerd 2.0", version: "2.0.0", want: 3}, + {name: "containerd 2.2", version: "2.2.4", want: 3}, + {name: "containerd 2.3", version: "2.3.0", want: 4}, + {name: "containerd 2.3 patch", version: "2.3.4", want: 4}, + {name: "missing version", wantErr: `unsupported or missing containerd version ""`}, + {name: "invalid version", version: "latest", wantErr: `unsupported or missing containerd version "latest"`}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := getContainerdConfigVersion(tt.containerdVersion); got != tt.want { - t.Fatalf("getContainerdConfigVersion(%q) = %d, want %d", tt.containerdVersion, got, tt.want) + got, err := containerdConfigSchema(tt.version) + if tt.wantErr != "" { + if err == nil || err.Error() != tt.wantErr { + t.Fatalf("containerdConfigSchema(%q) error = %v, want %q", tt.version, err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("containerdConfigSchema(%q) returned error: %v", tt.version, err) + } + if got != tt.want { + t.Fatalf("containerdConfigSchema(%q) = %d, want %d", tt.version, got, tt.want) } }) } diff --git a/aks-node-controller/parser/parser.go b/aks-node-controller/parser/parser.go index cd8124e01f5..66a00ad5bcf 100644 --- a/aks-node-controller/parser/parser.go +++ b/aks-node-controller/parser/parser.go @@ -31,7 +31,7 @@ func executeBootstrapTemplate(inputContract *aksnodeconfigv1.Configuration) (str } //nolint:funlen -func getCSEEnv(ctx context.Context, config *aksnodeconfigv1.Configuration, gpuConfig *gpu.GPUConfiguration) map[string]string { +func getCSEEnv(ctx context.Context, config *aksnodeconfigv1.Configuration, gpuConfig *gpu.GPUConfiguration) (map[string]string, error) { // streamingConnectionIdleTimeout was removed from KubeletConfiguration in k8s 1.34+. // Clear it from both KubeletFlags and KubeletConfigFileConfig so it doesn't appear // on the command line or in the marshaled config file JSON. @@ -44,7 +44,10 @@ func getCSEEnv(ctx context.Context, config *aksnodeconfigv1.Configuration, gpuCo } } - containerdVersion, _ := detectContainerdVersion(ctx) + containerdVersion, err := resolveContainerdConfigVersion(ctx, config) + if err != nil { + return nil, err + } cloudProviderSettings := getCloudProviderSettings(config) isMIGNode := getIsMIGNode(config.GetGpuConfig().GetGpuInstanceProfile(), config.GetGpuConfig().GetMigProfileLayout()) env := map[string]string{ @@ -225,7 +228,7 @@ func getCSEEnv(ctx context.Context, config *aksnodeconfigv1.Configuration, gpuCo for i, cert := range config.CustomCaCerts { env[fmt.Sprintf("CUSTOM_CA_CERT_%d", i)] = removeNewlines(cert) } - return env + return env, nil } type cloudProviderSettings struct { @@ -320,7 +323,11 @@ func BuildCSECmd(ctx context.Context, config *aksnodeconfigv1.Configuration, gpu // Convert to one-liner triggerBootstrapScript = strings.ReplaceAll(triggerBootstrapScript, "\n", " ") cmd := exec.CommandContext(ctx, "/bin/bash", "-c", triggerBootstrapScript) - env := mapToEnviron(getCSEEnv(ctx, config, gpuConfig)) + cseEnv, err := getCSEEnv(ctx, config, gpuConfig) + if err != nil { + return nil, fmt.Errorf("build CSE environment: %w", err) + } + env := mapToEnviron(cseEnv) cmd.Env = append(os.Environ(), env...) // append existing environment variables sort.Strings(cmd.Env) return cmd, nil diff --git a/aks-node-controller/parser/parser_test.go b/aks-node-controller/parser/parser_test.go index 0e9879b3e9a..248a1c2c538 100644 --- a/aks-node-controller/parser/parser_test.go +++ b/aks-node-controller/parser/parser_test.go @@ -770,3 +770,35 @@ func TestBuildCSECmd_FallsBackToV1WhenContainerdDetectionFails(t *testing.T) { assert.Contains(t, containerdConfig, `plugins."io.containerd.grpc.v1.cri"`) assert.NotContains(t, containerdConfig, `plugins."io.containerd.cri.v1.images"`) } + +func TestBuildCSECmd_KataUsesConfiguredContainerdVersionWhenDetectionFails(t *testing.T) { + tmpDir := t.TempDir() + t.Setenv("PATH", tmpDir) + + config := &aksnodeconfigv1.Configuration{ + IsKata: true, + ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ + ContainerdVersion: "2.3.4", + }, + } + + cmd, err := BuildCSECmd(context.TODO(), config, nil) + require.NoError(t, err) + vars := environToMap(cmd.Env) + containerdConfig, err := getBase64DecodedValue([]byte(vars["CONTAINERD_CONFIG_NO_GPU_CONTENT"])) + require.NoError(t, err) + assert.True(t, strings.HasPrefix(containerdConfig, "version = 4\n"), containerdConfig) + assert.Contains(t, containerdConfig, `plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata]`) + assert.Contains(t, containerdConfig, `plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-v2]`) +} + +func TestBuildCSECmd_KataRejectsMissingContainerdVersion(t *testing.T) { + tmpDir := t.TempDir() + t.Setenv("PATH", tmpDir) + + _, err := BuildCSECmd(context.TODO(), &aksnodeconfigv1.Configuration{ + IsKata: true, + ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{}, + }, nil) + require.ErrorContains(t, err, "containerd version is required for Kata and runtime detection failed") +} diff --git a/aks-node-controller/parser/templates/containerd_v2.toml.gtpl b/aks-node-controller/parser/templates/containerd_v2.toml.gtpl index acddf11054d..cdbe70b1e4a 100644 --- a/aks-node-controller/parser/templates/containerd_v2.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_v2.toml.gtpl @@ -1,4 +1,4 @@ -version = {{getContainerdConfigVersion}} +version = {{getContainerdConfigSchema}} oom_score = -999{{if getHasDataDir .KubeletConfig}} root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} {{- if .GetIsKata }} diff --git a/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl b/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl index 2862816c83d..63ce793662a 100644 --- a/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl @@ -1,4 +1,4 @@ -version = {{getContainerdConfigVersion}} +version = {{getContainerdConfigSchema}} oom_score = -999{{if getHasDataDir .KubeletConfig}} root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} {{- if .GetIsKata }} diff --git a/e2e/components/components_test.go b/e2e/components/components_test.go index 1f1c705c13b..c4164105133 100644 --- a/e2e/components/components_test.go +++ b/e2e/components/components_test.go @@ -55,6 +55,10 @@ func TestWs2022ServerCore(t *testing.T) { require.Len(t, serverCoreVersions, 3) } +func TestAzureLinuxKataContainerdPinned(t *testing.T) { + require.Equal(t, []string{"2.3.4"}, GetExpectedPackageVersions("containerd", "azurelinuxkata", "v3.0")) +} + func TestWs2022Nanoserver(t *testing.T) { serverCoreVersions, err := GetWindowsContainerImages("mcr.microsoft.com/windows/nanoserver:*", "2022-containerd") require.NoError(t, err) diff --git a/e2e/scenario/containerd_config_parity_test.go b/e2e/scenario/containerd_config_parity_test.go new file mode 100644 index 00000000000..2a511cf57df --- /dev/null +++ b/e2e/scenario/containerd_config_parity_test.go @@ -0,0 +1,98 @@ +package scenario + +import ( + "bytes" + "context" + "encoding/base64" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/Azure/agentbaker/aks-node-controller/parser" + "github.com/Azure/agentbaker/pkg/agent" + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/stretchr/testify/require" +) + +func TestKataContainerdConfigScriptfulScriptlessParity(t *testing.T) { + const containerdVersion = "2.3.4" + + nbc, err := baseTemplateLinux("eastus", "1.34.0", "amd64") + require.NoError(t, err) + nbc.ContainerdVersion = containerdVersion + nbc.AgentPoolProfile.Distro = datamodel.AKSAzureLinuxV3Gen2Kata + nbc.AgentPoolProfile.KubernetesConfig.ContainerdVersion = containerdVersion + nbc.ContainerService.Properties.AgentPoolProfiles[0].Distro = datamodel.AKSAzureLinuxV3Gen2Kata + nbc.ContainerService.Properties.AgentPoolProfiles[0].KubernetesConfig.ContainerdVersion = containerdVersion + + agentBaker, err := agent.NewAgentBaker() + require.NoError(t, err) + bootstrapping, err := agentBaker.GetNodeBootstrapping(context.Background(), nbc) + require.NoError(t, err) + scriptfulConfig := decodeContainerdConfigFromCSE(t, bootstrapping.CSE) + + nodeConfig, err := nbcToAKSNodeConfigV1(nbc) + require.NoError(t, err) + t.Setenv("PATH", t.TempDir()) + cmd, err := parser.BuildCSECmd(context.Background(), nodeConfig, nil) + require.NoError(t, err) + scriptlessConfig := decodeContainerdConfigFromEnv(t, cmd.Env) + + for _, rendered := range []string{scriptfulConfig, scriptlessConfig} { + require.True(t, strings.HasPrefix(rendered, "version = 4\n"), rendered) + require.NotContains(t, rendered, `io.containerd.grpc.v1.cri`) + require.Contains(t, rendered, `plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata]`) + require.Contains(t, rendered, `plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-v2]`) + } + + containerdBinary := os.Getenv("CONTAINERD_TEST_BINARY") + if containerdBinary == "" { + t.Log("CONTAINERD_TEST_BINARY is not set; skipping real containerd validation") + return + } + validateRenderedContainerdConfig(t, containerdBinary, "scriptful", scriptfulConfig) + validateRenderedContainerdConfig(t, containerdBinary, "scriptless", scriptlessConfig) +} + +func decodeContainerdConfigFromCSE(t *testing.T, cse string) string { + t.Helper() + match := regexp.MustCompile(`CONTAINERD_CONFIG_CONTENT="([^"]+)"`).FindStringSubmatch(cse) + require.Len(t, match, 2, cse) + return decodeContainerdConfig(t, match[1]) +} + +func decodeContainerdConfigFromEnv(t *testing.T, env []string) string { + t.Helper() + for _, entry := range env { + if encoded, found := strings.CutPrefix(entry, "CONTAINERD_CONFIG_CONTENT="); found { + return decodeContainerdConfig(t, encoded) + } + } + t.Fatal("CONTAINERD_CONFIG_CONTENT is missing from the CSE environment") + return "" +} + +func decodeContainerdConfig(t *testing.T, encoded string) string { + t.Helper() + decoded, err := base64.StdEncoding.DecodeString(encoded) + require.NoError(t, err) + return string(decoded) +} + +func validateRenderedContainerdConfig(t *testing.T, binary, name, rendered string) { + t.Helper() + configPath := filepath.Join(t.TempDir(), name+"-containerd.toml") + require.NoError(t, os.WriteFile(configPath, []byte(rendered), 0o600)) + + cmd := exec.Command(binary, "--config", configPath, "config", "dump") + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + require.NoError(t, cmd.Run(), stderr.String()) + require.Empty(t, stderr.String()) + require.True(t, strings.HasPrefix(stdout.String(), "version = 4\n"), stdout.String()) +} diff --git a/e2e/scenario/node_config.go b/e2e/scenario/node_config.go index f8f53457056..40948adcbf5 100644 --- a/e2e/scenario/node_config.go +++ b/e2e/scenario/node_config.go @@ -8,6 +8,7 @@ import ( aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1" "github.com/Masterminds/semver/v3" + "github.com/Azure/agentbaker/e2e/components" "github.com/Azure/agentbaker/e2e/config" "github.com/Azure/agentbaker/e2e/toolkit" "github.com/Azure/agentbaker/pkg/agent" @@ -155,6 +156,15 @@ func getBaseNBC(ctx context.Context, cluster *Cluster, vhd *config.Image) (*data nbc.ContainerService.Properties.HostedMasterProfile.FQDN = cluster.ClusterParams.FQDN nbc.ContainerService.Properties.AgentPoolProfiles[0].Distro = vhd.Distro nbc.AgentPoolProfile.Distro = vhd.Distro + if vhd.Distro == datamodel.AKSAzureLinuxV3Gen2Kata { + containerdVersions := components.GetExpectedPackageVersions("containerd", "azurelinuxkata", "v3.0") + if len(containerdVersions) != 1 || containerdVersions[0] == "" || containerdVersions[0] == "" { + return nil, fmt.Errorf("expected one pinned Azure Linux 3 Kata containerd version, got %v", containerdVersions) + } + nbc.ContainerdVersion = containerdVersions[0] + nbc.AgentPoolProfile.KubernetesConfig.ContainerdVersion = containerdVersions[0] + nbc.ContainerService.Properties.AgentPoolProfiles[0].KubernetesConfig.ContainerdVersion = containerdVersions[0] + } return nbc, nil } @@ -293,6 +303,7 @@ func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) (*aksno KubernetesVersion: cs.Properties.OrchestratorProfile.OrchestratorVersion, ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ ContainerdDownloadUrlBase: nbc.CloudSpecConfig.KubernetesSpecConfig.ContainerdDownloadURLBase, + ContainerdVersion: nbc.ContainerdVersion, }, OutboundCommand: `curl -v --insecure --proxy-insecure https://mcr.microsoft.com/v2/`, KubernetesCaCert: base64.StdEncoding.EncodeToString([]byte(cs.Properties.CertificateProfile.CaCertificate)), diff --git a/e2e/scenario/scenario.go b/e2e/scenario/scenario.go index 2647b5f5543..14b1c9e58b4 100644 --- a/e2e/scenario/scenario.go +++ b/e2e/scenario/scenario.go @@ -431,11 +431,16 @@ var _ = Register(&Scenario{ nbc.DisableUnattendedUpgrades = false }, Validator: func(ctx context.Context, s *Scenario) error { + containerdVersion, err := expectedPackageVersion("containerd", "azurelinuxkata", "v3.0") + if err != nil { + return err + } if err := errors.Join( ValidateKataContainerdConfig(ctx, s), ValidateKataErofsContainerdConfig(ctx, s), ValidateKataContainerdConfigDump(ctx, s), ValidateKataHostReadiness(ctx, s), + ValidateInstalledPackageVersion(ctx, s, "containerd2", containerdVersion), ); err != nil { return err } diff --git a/e2e/scenario/validators_kata.go b/e2e/scenario/validators_kata.go index b25a780a23c..13ef23a85b0 100644 --- a/e2e/scenario/validators_kata.go +++ b/e2e/scenario/validators_kata.go @@ -123,11 +123,10 @@ func ValidateKataErofsContainerdConfig(ctx context.Context, s *Scenario) error { // ValidateKataContainerdConfigDump asserts that containerd itself accepted the rendered // configuration and actually loaded the Kata runtime handlers. // -// Checking the file alone is not enough. Kata VHDs ship their own containerd build - CSE skips -// installing one (see the "azurelinuxkata" entries in parts/common/components.json) - so the -// installed binary is the authority for the native config schema. This validator pins the -// property we actually care about: after containerd has parsed the normalized config, the Kata -// handlers are present in the effective configuration and containerd raised no warnings. +// Checking the file alone is not enough. Azure Linux Kata pins containerd in components.json, +// but the installed binary remains the authority for the native config schema. This validator +// pins the property we actually care about: after containerd has parsed the normalized config, +// the Kata handlers are present in the effective configuration and containerd raised no warnings. func ValidateKataContainerdConfigDump(ctx context.Context, s *Scenario) error { // This must run on the node itself, not in a debug pod. The "debugnonhost" daemonset pods // used by execOnVMForScenarioOnUnprivilegedPod run a bare CBL-Mariner base image with no diff --git a/parts/common/components.json b/parts/common/components.json index 54982ebc33a..893d14e2df8 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -1171,8 +1171,8 @@ "v3.0": { "versionsV2": [ { - "renovateTag": "", - "latestVersion": "" + "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/base/x86_64/repodata, name=containerd2, os=azurelinux, release=3.0", + "latestVersion": "2.3.4" } ] } diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 2e644a979c8..9f13c5fffe7 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -753,6 +753,11 @@ func ValidateAndSetLinuxNodeBootstrappingConfigurationWithError(config *datamode if err := validateCustomLinuxOSConfig(config.AgentPoolProfile.GetCustomLinuxOSConfig()); err != nil { return err } + if config.AgentPoolProfile != nil { + if _, err := containerdConfigSchemaForConfig(config, config.AgentPoolProfile); err != nil { + return fmt.Errorf("validate containerd config: %w", err) + } + } if config.KubeletConfig == nil { return nil @@ -1247,25 +1252,26 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration "GetKubenetTemplate": func() string { return base64.StdEncoding.EncodeToString([]byte(kubenetCniTemplate)) }, + "GetContainerdConfigSchema": func() (int, error) { + return containerdConfigSchemaForConfig(config, profile) + }, "GetContainerdConfigContent": func() string { - output, err := containerdConfigFromTemplate(config, profile, func(profile *datamodel.AgentPoolProfile) ContainerdConfigTemplate { - if profile.IsContainerdV2Distro() { - return containerdV2ConfigTemplate - } - return containerdV1ConfigTemplate - }(profile)) + tmpl, err := selectContainerdConfigTemplate(config, profile, false) + if err != nil { + panic(err) + } + output, err := containerdConfigFromTemplate(config, profile, tmpl) if err != nil { panic(err) } return output }, "GetContainerdConfigNoGPUContent": func() string { - output, err := containerdConfigFromTemplate(config, profile, func(profile *datamodel.AgentPoolProfile) ContainerdConfigTemplate { - if profile.IsContainerdV2Distro() { - return containerdV2NoGPUConfigTemplate - } - return containerdV1NoGPUConfigTemplate - }(profile)) + tmpl, err := selectContainerdConfigTemplate(config, profile, true) + if err != nil { + panic(err) + } + output, err := containerdConfigFromTemplate(config, profile, tmpl) if err != nil { panic(err) } @@ -1972,7 +1978,7 @@ type ContainerdConfigTemplate string // duplicate them in CSE base64-encoded, and pick the right one. // they're identical except for GPU runtime class. const ( - containerdV1ConfigTemplate ContainerdConfigTemplate = `version = 2 + containerdV1ConfigTemplate ContainerdConfigTemplate = `version = {{GetContainerdConfigSchema}} oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} {{- if IsKata }} @@ -2075,7 +2081,7 @@ root = "{{GetDataDir}}"{{- end}} ConfigPath = "/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml" {{- end}} ` - containerdV2ConfigTemplate ContainerdConfigTemplate = `version = 3 + containerdV2ConfigTemplate ContainerdConfigTemplate = `version = {{GetContainerdConfigSchema}} oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} {{- if IsKata }} @@ -2173,7 +2179,7 @@ root = "{{GetDataDir}}"{{- end}} ConfigPath = "/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml" {{- end}} ` - containerdV2NoGPUConfigTemplate ContainerdConfigTemplate = `version = 3 + containerdV2NoGPUConfigTemplate ContainerdConfigTemplate = `version = {{GetContainerdConfigSchema}} oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} {{- if IsKata }} @@ -2251,7 +2257,7 @@ root = "{{GetDataDir}}"{{- end}} address = "/run/containerd/tardev-snapshotter.sock" {{- end}} ` - containerdV1NoGPUConfigTemplate ContainerdConfigTemplate = `version = 2 + containerdV1NoGPUConfigTemplate ContainerdConfigTemplate = `version = {{GetContainerdConfigSchema}} oom_score = -999{{if HasDataDir }} root = "{{GetDataDir}}"{{- end}} {{- if IsKata }} diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index 2f71be39bd3..dac01e6a658 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -68,7 +68,7 @@ func TestContainerdV2TemplatesUseNativeKataPluginPaths(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { config := string(test.template) - require.True(t, strings.HasPrefix(config, "version = 3\n")) + require.True(t, strings.HasPrefix(config, "version = {{GetContainerdConfigSchema}}\n")) require.NotContains(t, config, `io.containerd.grpc.v1.cri`) require.Contains(t, config, "disable_snapshot_annotations = false") for _, handler := range test.handlers { diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index 32998201e16..fbed8b02ff7 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -286,6 +286,7 @@ var _ = Describe("AgentBaker API implementation tests", func() { It("should not return an error for customized kata image", func() { config.AgentPoolProfile.Distro = datamodel.CustomizedImageKata + config.ContainerdVersion = "2.3.4" agentBaker, err := NewAgentBaker() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/agent/containerd_config.go b/pkg/agent/containerd_config.go new file mode 100644 index 00000000000..878525431d7 --- /dev/null +++ b/pkg/agent/containerd_config.go @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +package agent + +import ( + "fmt" + "strings" + + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/Masterminds/semver/v3" +) + +func semverGE(version, minimum string) bool { + actualVersion, err := semver.NewVersion(version) + if err != nil { + return false + } + minimumVersion, err := semver.NewVersion(minimum) + if err != nil { + return false + } + return !actualVersion.LessThan(minimumVersion) +} + +func containerdConfigSchema(version string) (int, error) { + switch { + case semverGE(version, "2.3.0"): + return 4, nil + case semverGE(version, "2.0.0"): + return 3, nil + case semverGE(version, "1.0.0"): + return 2, nil + default: + return 0, fmt.Errorf("unsupported or missing containerd version %q", version) + } +} + +func containerdVersionForConfig( + config *datamodel.NodeBootstrappingConfiguration, + profile *datamodel.AgentPoolProfile, +) (string, error) { + version := strings.TrimSpace(config.ContainerdVersion) + if version == "" && profile != nil && profile.KubernetesConfig != nil { + version = strings.TrimSpace(profile.KubernetesConfig.ContainerdVersion) + } + if version != "" { + if _, err := containerdConfigSchema(version); err != nil { + return "", err + } + return version, nil + } + if profile != nil && profile.Distro.IsKataDistro() { + return "", fmt.Errorf("containerd version is required for Kata distro %q", profile.Distro) + } + if profile != nil && profile.IsContainerdV2Distro() { + return "2.0.0", nil + } + return "1.0.0", nil +} + +func containerdConfigSchemaForConfig( + config *datamodel.NodeBootstrappingConfiguration, + profile *datamodel.AgentPoolProfile, +) (int, error) { + version, err := containerdVersionForConfig(config, profile) + if err != nil { + return 0, err + } + return containerdConfigSchema(version) +} + +func selectContainerdConfigTemplate( + config *datamodel.NodeBootstrappingConfiguration, + profile *datamodel.AgentPoolProfile, + noGPU bool, +) (ContainerdConfigTemplate, error) { + schema, err := containerdConfigSchemaForConfig(config, profile) + if err != nil { + return "", err + } + if schema >= 3 { + if noGPU { + return containerdV2NoGPUConfigTemplate, nil + } + return containerdV2ConfigTemplate, nil + } + if noGPU { + return containerdV1NoGPUConfigTemplate, nil + } + return containerdV1ConfigTemplate, nil +} diff --git a/pkg/agent/containerd_config_test.go b/pkg/agent/containerd_config_test.go new file mode 100644 index 00000000000..3a620023c61 --- /dev/null +++ b/pkg/agent/containerd_config_test.go @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +package agent + +import ( + "encoding/base64" + "strings" + "testing" + + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/stretchr/testify/require" +) + +func TestContainerdConfigSchema(t *testing.T) { + tests := []struct { + name string + version string + want int + wantErr string + }{ + {name: "containerd 1", version: "1.7.36", want: 2}, + {name: "containerd 2.2", version: "2.2.4", want: 3}, + {name: "containerd 2.3", version: "2.3.4", want: 4}, + {name: "missing version", wantErr: `unsupported or missing containerd version ""`}, + {name: "invalid version", version: "latest", wantErr: `unsupported or missing containerd version "latest"`}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := containerdConfigSchema(test.version) + if test.wantErr != "" { + require.EqualError(t, err, test.wantErr) + return + } + require.NoError(t, err) + require.Equal(t, test.want, got) + }) + } +} + +func TestScriptfulContainerdConfigUsesVersionedSchemaAndPluginPaths(t *testing.T) { + tests := []struct { + name string + version string + distro datamodel.Distro + wantSchema string + wantPluginPath string + unwantedPluginPath string + }{ + { + name: "containerd 1", + version: "1.7.36", + distro: datamodel.AKSUbuntuContainerd2204, + wantSchema: "version = 2\n", + wantPluginPath: `io.containerd.grpc.v1.cri`, + unwantedPluginPath: `io.containerd.cri.v1.runtime`, + }, + { + name: "containerd 2.2", + version: "2.2.4", + distro: datamodel.AKSAzureLinuxV3Gen2Kata, + wantSchema: "version = 3\n", + wantPluginPath: `io.containerd.cri.v1.runtime`, + unwantedPluginPath: `io.containerd.grpc.v1.cri`, + }, + { + name: "containerd 2.3", + version: "2.3.4", + distro: datamodel.AKSAzureLinuxV3Gen2Kata, + wantSchema: "version = 4\n", + wantPluginPath: `io.containerd.cri.v1.runtime`, + unwantedPluginPath: `io.containerd.grpc.v1.cri`, + }, + } + + for _, test := range tests { + for _, noGPU := range []bool{false, true} { + name := "default" + if noGPU { + name = "no GPU" + } + t.Run(test.name+"/"+name, func(t *testing.T) { + config := newNodeCustomDataRenderConfig(test.distro) + config.ContainerdVersion = test.version + config.AgentPoolProfile.KubernetesConfig = &datamodel.KubernetesConfig{ + ContainerRuntime: "containerd", + ContainerRuntimeConfig: map[string]string{}, + } + + tmpl, err := selectContainerdConfigTemplate(config, config.AgentPoolProfile, noGPU) + require.NoError(t, err) + encoded, err := containerdConfigFromTemplate(config, config.AgentPoolProfile, tmpl) + require.NoError(t, err) + rendered, err := base64.StdEncoding.DecodeString(encoded) + require.NoError(t, err) + configText := string(rendered) + + require.True(t, strings.HasPrefix(configText, test.wantSchema), configText) + require.Contains(t, configText, test.wantPluginPath) + require.NotContains(t, configText, test.unwantedPluginPath) + if test.distro.IsKataDistro() { + require.Contains(t, configText, `plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata]`) + require.Contains(t, configText, `plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-v2]`) + } + }) + } + } +} + +func TestKataContainerdConfigRequiresVersion(t *testing.T) { + config := newNodeCustomDataRenderConfig(datamodel.AKSAzureLinuxV3Gen2Kata) + config.AgentPoolProfile.KubernetesConfig = &datamodel.KubernetesConfig{ + ContainerRuntime: "containerd", + ContainerRuntimeConfig: map[string]string{}, + } + + err := ValidateAndSetLinuxNodeBootstrappingConfigurationWithError(config) + require.EqualError(t, err, `validate containerd config: containerd version is required for Kata distro "aks-azurelinux-v3-gen2-kata"`) +} + +func TestContainerdConfigRejectsUnsupportedVersion(t *testing.T) { + config := newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204) + config.ContainerdVersion = "latest" + + err := ValidateAndSetLinuxNodeBootstrappingConfigurationWithError(config) + require.EqualError(t, err, `validate containerd config: unsupported or missing containerd version "latest"`) +} diff --git a/spec/parts/linux/cloud-init/artifacts/cse_install_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_install_spec.sh index 3095fd7f1c9..033fa0b9e74 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_install_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_install_spec.sh @@ -83,14 +83,15 @@ Describe 'cse_install.sh' When call installContainerRuntime The output line 3 should equal "INFO: containerd package versions array is either empty or the first element is . Skipping containerd installation." End - # TODO(mheberling): In a ~month this will probably be removed when we use the standard containerd. - It 'skips the containerd installation for AzureLinux with Kata' + It 'returns expected output for successful installation of containerd in AzureLinux with Kata' UBUNTU_RELEASE="" # mocking Mariner doesn't have command `lsb_release -cs` OS="AZURELINUX" containerdPackage=$(readPackage "containerd") IS_KATA="true" When call installContainerRuntime - The output line 3 should equal "INFO: containerd package versions array is either empty or the first element is . Skipping containerd installation." + The variable packageVersion should equal "2.3.4-1.fake" + The output line 3 should equal "mock logs to events calling with AKS.CSE.installContainerRuntime.installStandaloneContainerd" + The output line 4 should equal "in installContainerRuntime - CONTAINERD_VERSION = 2.3.4-1.fake" End It 'returns expected output for successful installation of containerd in AzureLinux' UBUNTU_RELEASE="" # mocking AzureLinux doesn't have command `lsb_release -cs` diff --git a/spec/parts/linux/cloud-init/artifacts/test_components.json b/spec/parts/linux/cloud-init/artifacts/test_components.json index 01389f78a93..4d93f307ff3 100644 --- a/spec/parts/linux/cloud-init/artifacts/test_components.json +++ b/spec/parts/linux/cloud-init/artifacts/test_components.json @@ -193,7 +193,7 @@ "versionsV2": [ { "renovateTag": "", - "latestVersion": "" + "latestVersion": "2.3.4-1.fake" } ] }