fix fixAzureLinuxV3Gen2Kata

aks-node-controller/parser/helper.go

@@ -89,6 +89,7 @@ func getFuncMapForContainerdConfigTemplate() template.FuncMap {
 		"isKubernetesVersionGe":            IsKubernetesVersionGe,
 		"getHasDataDir":                    getHasDataDir,
 		"getEnableNvidia":                  getEnableNvidia,
+		"getContainerdConfigSchema":        func() int { return 3 },
 	}
 }
 
@@ -204,11 +205,16 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat
 		return "", fmt.Errorf("AKSNodeConfig is nil")
 	}
 
+	schema, err := containerdConfigSchema(containerdVersion)
+	if err != nil {
+		return "", err
+	}
+
 	// Select the appropriate containerd config template based on version and GPU presence.
 	// Containerd 2.x uses different CRI plugin paths (io.containerd.cri.v1.images/runtime)
 	// compared to containerd 1.x (io.containerd.grpc.v1.cri).
 	var _template *template.Template
-	if isContainerdV2(containerdVersion) {
+	if schema >= 3 {
 		_template = containerdV2ConfigTemplate
 		if noGPU {
 			_template = containerdV2ConfigNoGPUTemplate
@@ -220,14 +226,47 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat
 		}
 	}
 
+	renderTemplate, err := _template.Clone()
+	if err != nil {
+		return "", fmt.Errorf("error cloning containerd config template: %w", err)
+	}
+	renderTemplate = renderTemplate.Funcs(template.FuncMap{
+		"getContainerdConfigSchema": func() int { return schema },
+	})
+
 	var buffer bytes.Buffer
-	if err := _template.Execute(&buffer, aksnodeconfig); err != nil {
+	if err := renderTemplate.Execute(&buffer, aksnodeconfig); err != nil {
 		return "", fmt.Errorf("error executing containerd config template for AKSNodeConfig: %w", err)
 	}
 
 	return buffer.String(), nil
 }
 
+func semverGE(version, minimum string) bool {
+	actualVersion, err := semver.NewVersion(version)
+	if err != nil {
+		return false
+	}
+	minimumVersion, err := semver.NewVersion(minimum)
+	if err != nil {
+		return false
+	}
+	return !actualVersion.LessThan(minimumVersion)
+}
+
+func containerdConfigSchema(version string) (int, error) {
+	switch {
+	case semverGE(version, "2.3.0"):
+		return 4, nil
+	case semverGE(version, "2.0.0"):
+		return 3, nil
+	case semverGE(version, "1.0.0"):
+		return 2, nil
+	default:
+		return 0, fmt.Errorf("unsupported or missing containerd version %q", version)
+	}
+}
+
 // detectContainerdVersion runs "containerd --version" and parses the version string.
 // The expected output format is: "containerd <source> <version> <commit>"
 // e.g. "containerd containerd.io 1.7.22 c814c75..." or "containerd github.com/containerd/containerd/v2 v2.0.0 ..."
@@ -237,17 +276,33 @@ func detectContainerdVersion(ctx context.Context) (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("running containerd --version: %w", err)
 	}
-	return parseContainerdVersionOutput(string(out)), nil
+	version := parseContainerdVersionOutput(string(out))
+	if version == "" {
+		return "", fmt.Errorf("parsing containerd version output %q", strings.TrimSpace(string(out)))
+	}
+	return version, nil
 }
 
-// isContainerdV2 returns true if the containerd version string indicates a 2.x release.
-// Containerd 2.x uses different CRI plugin paths (io.containerd.cri.v1.images and
-// io.containerd.cri.v1.runtime) compared to 1.x (io.containerd.grpc.v1.cri).
-func isContainerdV2(version string) bool {
-	if version == "" {
-		return false
+func resolveContainerdConfigVersion(ctx context.Context, config *aksnodeconfigv1.Configuration) (string, error) {
+	detectedVersion, detectionErr := detectContainerdVersion(ctx)
+	if detectionErr == nil {
+		if _, err := containerdConfigSchema(detectedVersion); err != nil {
+			return "", fmt.Errorf("detected containerd version: %w", err)
+		}
+		return detectedVersion, nil
+	}
+
+	configuredVersion := strings.TrimSpace(config.GetContainerdConfig().GetContainerdVersion())
+	if configuredVersion != "" {
+		if _, err := containerdConfigSchema(configuredVersion); err != nil {
+			return "", fmt.Errorf("configured containerd version: %w", err)
+		}
+		return configuredVersion, nil
+	}
+	if config.GetIsKata() {
+		return "", fmt.Errorf("containerd version is required for Kata and runtime detection failed: %w", detectionErr)
 	}
-	return IsKubernetesVersionGe(version, "2.0.0")
+	return "1.0.0", nil
 }
 
 func getIsMIGNode(gpuInstanceProfile string, migProfileLayout []string) bool {

aks-node-controller/parser/helper_test.go

@@ -497,7 +497,7 @@ oom_score = -999
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := getContainerdConfigBase64(tt.args.aksnodeconfig, ""); got != tt.want {
+			if got := getContainerdConfigBase64(tt.args.aksnodeconfig, "1.7.22"); got != tt.want {
 				t.Errorf("getContainerdConfig() = %v, want %v", got, tt.want)
 			}
 		})
@@ -523,7 +523,7 @@ func Test_getContainerdConfigV2(t *testing.T) {
 					},
 				},
 			},
-			want: base64.StdEncoding.EncodeToString([]byte(`version = 2
+			want: base64.StdEncoding.EncodeToString([]byte(`version = 3
 oom_score = -999
 [plugins."io.containerd.cri.v1.images"]
   [plugins."io.containerd.cri.v1.images".pinned_images]
@@ -559,7 +559,7 @@ oom_score = -999
 				},
 				noGpu: false,
 			},
-			want: base64.StdEncoding.EncodeToString([]byte(`version = 2
+			want: base64.StdEncoding.EncodeToString([]byte(`version = 3
 oom_score = -999
 [plugins."io.containerd.cri.v1.images"]
   [plugins."io.containerd.cri.v1.images".pinned_images]
@@ -594,7 +594,7 @@ oom_score = -999
 				},
 				noGpu: true,
 			},
-			want: base64.StdEncoding.EncodeToString([]byte(`version = 2
+			want: base64.StdEncoding.EncodeToString([]byte(`version = 3
 oom_score = -999
 [plugins."io.containerd.cri.v1.images"]
   [plugins."io.containerd.cri.v1.images".pinned_images]
@@ -665,6 +665,89 @@ oom_score = -999
 	}
 }
 
+func Test_getContainerdConfigV2KataUsesNativePluginPaths(t *testing.T) {
+	config := &aksnodeconfigv1.Configuration{
+		IsKata: true,
+		ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{
+			ContainerdVersion: "2.3.4",
+		},
+	}
+
+	tests := []struct {
+		name      string
+		getConfig func(*aksnodeconfigv1.Configuration, string) string
+	}{
+		{
+			name:      "default",
+			getConfig: getContainerdConfigBase64,
+		},
+		{
+			name:      "no GPU",
+			getConfig: getNoGPUContainerdConfigBase64,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			decoded, err := base64.StdEncoding.DecodeString(test.getConfig(config, "2.3.4"))
+			if err != nil {
+				t.Fatalf("decoding containerd config: %v", err)
+			}
+			rendered := string(decoded)
+			if !strings.HasPrefix(rendered, "version = 4\n") {
+				t.Fatalf("containerd 2.3+ config does not use schema 4:\n%s", rendered)
+			}
+			if strings.Contains(rendered, `io.containerd.grpc.v1.cri`) {
+				t.Fatalf("containerd v2 config contains the legacy CRI plugin path:\n%s", rendered)
+			}
+			if !strings.Contains(rendered, "disable_snapshot_annotations = false") {
+				t.Fatalf("containerd v2 Kata config does not disable snapshot annotations:\n%s", rendered)
+			}
+			for _, handler := range []string{"kata", "kata-v2"} {
+				expected := `plugins."io.containerd.cri.v1.runtime".containerd.runtimes.` + handler + `]`
+				if !strings.Contains(rendered, expected) {
+					t.Fatalf("containerd v2 config does not contain %q:\n%s", expected, rendered)
+				}
+			}
+		})
+	}
+}
+
+func TestContainerdConfigSchema(t *testing.T) {
+	tests := []struct {
+		name    string
+		version string
+		want    int
+		wantErr string
+	}{
+		{name: "containerd 1", version: "1.7.22", want: 2},
+		{name: "containerd 2.0", version: "2.0.0", want: 3},
+		{name: "containerd 2.2", version: "2.2.4", want: 3},
+		{name: "containerd 2.3", version: "2.3.0", want: 4},
+		{name: "containerd 2.3 patch", version: "2.3.4", want: 4},
+		{name: "missing version", wantErr: `unsupported or missing containerd version ""`},
+		{name: "invalid version", version: "latest", wantErr: `unsupported or missing containerd version "latest"`},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := containerdConfigSchema(tt.version)
+			if tt.wantErr != "" {
+				if err == nil || err.Error() != tt.wantErr {
+					t.Fatalf("containerdConfigSchema(%q) error = %v, want %q", tt.version, err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("containerdConfigSchema(%q) returned error: %v", tt.version, err)
+			}
+			if got != tt.want {
+				t.Fatalf("containerdConfigSchema(%q) = %d, want %d", tt.version, got, tt.want)
+			}
+		})
+	}
+}
+
 func Test_getKubenetTemplate(t *testing.T) {
 	tests := []struct {
 		name string

aks-node-controller/parser/parser.go

@@ -31,7 +31,7 @@ func executeBootstrapTemplate(inputContract *aksnodeconfigv1.Configuration) (str
 }
 
 //nolint:funlen
-func getCSEEnv(ctx context.Context, config *aksnodeconfigv1.Configuration, gpuConfig *gpu.GPUConfiguration) map[string]string {
+func getCSEEnv(ctx context.Context, config *aksnodeconfigv1.Configuration, gpuConfig *gpu.GPUConfiguration) (map[string]string, error) {
 	// streamingConnectionIdleTimeout was removed from KubeletConfiguration in k8s 1.34+.
 	// Clear it from both KubeletFlags and KubeletConfigFileConfig so it doesn't appear
 	// on the command line or in the marshaled config file JSON.
@@ -44,7 +44,10 @@ func getCSEEnv(ctx context.Context, config *aksnodeconfigv1.Configuration, gpuCo
 		}
 	}
 
-	containerdVersion, _ := detectContainerdVersion(ctx)
+	containerdVersion, err := resolveContainerdConfigVersion(ctx, config)
+	if err != nil {
+		return nil, err
+	}
 	cloudProviderSettings := getCloudProviderSettings(config)
 	isMIGNode := getIsMIGNode(config.GetGpuConfig().GetGpuInstanceProfile(), config.GetGpuConfig().GetMigProfileLayout())
 	env := map[string]string{
@@ -225,7 +228,7 @@ func getCSEEnv(ctx context.Context, config *aksnodeconfigv1.Configuration, gpuCo
 	for i, cert := range config.CustomCaCerts {
 		env[fmt.Sprintf("CUSTOM_CA_CERT_%d", i)] = removeNewlines(cert)
 	}
-	return env
+	return env, nil
 }
 
 type cloudProviderSettings struct {
@@ -320,7 +323,11 @@ func BuildCSECmd(ctx context.Context, config *aksnodeconfigv1.Configuration, gpu
 	// Convert to one-liner
 	triggerBootstrapScript = strings.ReplaceAll(triggerBootstrapScript, "\n", " ")
 	cmd := exec.CommandContext(ctx, "/bin/bash", "-c", triggerBootstrapScript)
-	env := mapToEnviron(getCSEEnv(ctx, config, gpuConfig))
+	cseEnv, err := getCSEEnv(ctx, config, gpuConfig)
+	if err != nil {
+		return nil, fmt.Errorf("build CSE environment: %w", err)
+	}
+	env := mapToEnviron(cseEnv)
 	cmd.Env = append(os.Environ(), env...) // append existing environment variables
 	sort.Strings(cmd.Env)
 	return cmd, nil

aks-node-controller/parser/parser_test.go

@@ -740,6 +740,7 @@ func TestBuildCSECmd_DetectsContainerdV2FromSystem(t *testing.T) {
 	// Verify the v2 containerd config template was used (uses "io.containerd.cri.v1.images" path).
 	containerdConfig, err := getBase64DecodedValue([]byte(vars["CONTAINERD_CONFIG_NO_GPU_CONTENT"]))
 	require.NoError(t, err)
+	assert.True(t, strings.HasPrefix(containerdConfig, "version = 4\n"), containerdConfig)
 	assert.Contains(t, containerdConfig, `plugins."io.containerd.cri.v1.images"`)
 	assert.NotContains(t, containerdConfig, `plugins."io.containerd.grpc.v1.cri"`)
 }
@@ -769,3 +770,35 @@ func TestBuildCSECmd_FallsBackToV1WhenContainerdDetectionFails(t *testing.T) {
 	assert.Contains(t, containerdConfig, `plugins."io.containerd.grpc.v1.cri"`)
 	assert.NotContains(t, containerdConfig, `plugins."io.containerd.cri.v1.images"`)
 }
+
+func TestBuildCSECmd_KataUsesConfiguredContainerdVersionWhenDetectionFails(t *testing.T) {
+	tmpDir := t.TempDir()
+	t.Setenv("PATH", tmpDir)
+
+	config := &aksnodeconfigv1.Configuration{
+		IsKata: true,
+		ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{
+			ContainerdVersion: "2.3.4",
+		},
+	}
+
+	cmd, err := BuildCSECmd(context.TODO(), config, nil)
+	require.NoError(t, err)
+	vars := environToMap(cmd.Env)
+	containerdConfig, err := getBase64DecodedValue([]byte(vars["CONTAINERD_CONFIG_NO_GPU_CONTENT"]))
+	require.NoError(t, err)
+	assert.True(t, strings.HasPrefix(containerdConfig, "version = 4\n"), containerdConfig)
+	assert.Contains(t, containerdConfig, `plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata]`)
+	assert.Contains(t, containerdConfig, `plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-v2]`)
+}
+
+func TestBuildCSECmd_KataRejectsMissingContainerdVersion(t *testing.T) {
+	tmpDir := t.TempDir()
+	t.Setenv("PATH", tmpDir)
+
+	_, err := BuildCSECmd(context.TODO(), &aksnodeconfigv1.Configuration{
+		IsKata:           true,
+		ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{},
+	}, nil)
+	require.ErrorContains(t, err, "containerd version is required for Kata and runtime detection failed")
+}

aks-node-controller/parser/templates/containerd_v2.toml.gtpl

@@ -1,4 +1,4 @@
-version = 2
+version = {{getContainerdConfigSchema}}
 oom_score = -999{{if getHasDataDir .KubeletConfig}}
 root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
 {{- if .GetIsKata }}
@@ -15,6 +15,9 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
   enable_tar_index = false
 {{- end}}
 [plugins."io.containerd.cri.v1.images"]
+{{- if .GetIsKata }}
+  disable_snapshot_annotations = false
+{{- end}}
 {{- if .GetEnableArtifactStreaming }}
   snapshotter = "overlaybd"
   disable_snapshot_annotations = false
@@ -66,29 +69,29 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
     address = "/run/overlaybd-snapshotter/overlaybd.sock"
 {{- end}}
 {{- if .GetIsKata }}
-[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
+[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata]
   runtime_type = "io.containerd.kata.v2"
   privileged_without_host_devices = true
   snapshotter = "overlayfs"
-  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options]
+  [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata.options]
     ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml"
-[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-v2]
+[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-v2]
   runtime_path = "/usr/local/bin/containerd-shim-kata-v2-rs"
   runtime_type = "io.containerd.kata.v2"
   privileged_without_host_devices = true
   pod_annotations = ["io.katacontainers.snapshot-name"]
   snapshotter = "erofs"
-  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-v2.options]
+  [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-v2.options]
     ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-azure-runtime-rs-v2.toml"
 [proxy_plugins]
   [proxy_plugins.tardev]
     type = "snapshot"
     address = "/run/containerd/tardev-snapshotter.sock"
-[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc]
+[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-cc]
   snapshotter = "tardev"
   runtime_type = "io.containerd.kata-cc.v2"
   privileged_without_host_devices = true
   pod_annotations = ["io.katacontainers.*"]
-  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc.options]
+  [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-cc.options]
     ConfigPath = "/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml"
 {{- end}}