Merge pull request #18009 from justinsb/aiconform_custom_tests

tests/ai-conformance: create first tests for AI Conformance

Author: k8s-ci-robot

tests/e2e/scenarios/ai-conformance/run-test.sh

@@ -216,7 +216,8 @@ echo "Waiting for Sample Workload to Complete..."
 kubectl wait --for=condition=complete job/test-gpu-pod --timeout=5m || true
 kubectl logs job/test-gpu-pod || echo "Failed to get logs"
 
-# Note: The actual AI conformance test suite (e.g., k8s-ai-conformance binary)
-# would be executed here. For this scenario, we establish the compliant environment.
-
 echo "AI Conformance Environment Setup Complete."
+
+# Now run the actual AI conformance tests
+cd "${REPO_ROOT}/tests/e2e/scenarios/ai-conformance/validators"
+go test -v ./... -timeout=60m

tests/e2e/scenarios/ai-conformance/validators/accelerators/dra_support/apiversions_test.go

@@ -0,0 +1,57 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dra_support
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/kops/tests/e2e/scenarios/ai-conformance/validators"
+)
+
+// TestAcceleratorsDRASupport corresponds to the accelerators/dra_support scenario
+func TestAcceleratorsDRASupport(t *testing.T) {
+	// Description:
+	//   Support Dynamic Resource Allocation (DRA) APIs to enable more flexible and fine-grained resource requests beyond simple counts.
+	h := validators.NewValidatorHarness(t)
+
+	h.Logf("# DRA API Availability")
+
+	gv := schema.GroupVersion{Group: "resource.k8s.io", Version: "v1"}
+
+	// Check resource.k8s.io/v1 is registered
+	h.Logf("## Checking for DRA API version v1")
+	{
+		result := h.ShellExec(fmt.Sprintf("kubectl api-versions | grep %s", gv.Group+"/"+gv.Version))
+		if !strings.Contains(result.Stdout(), "resource.k8s.io/v1\n") {
+			h.Fatalf("Expected DRA API version group %s version %s, but it was not found", gv.Group, gv.Version)
+		}
+		h.Success("DRA API version %s is available.", gv.String())
+	}
+
+	// Check all expected DRA API resources are registered
+	for _, resource := range []string{"deviceclasses", "resourceclaims", "resourceclaimtemplates", "resourceslices"} {
+		h.Logf("## Checking for %s", resource)
+		result := h.ShellExec(fmt.Sprintf("kubectl api-resources --api-group=%s | grep %s", gv.Group, resource))
+		if !strings.Contains(result.Stdout(), "resource.k8s.io/v1") {
+			h.Fatalf("Expected DRA API resource %s to be available in group %s version %s, but it was not found", resource, gv.Group, gv.Version)
+		}
+		h.Success("DRA API resource %s is available.", resource)
+	}
+}

tests/e2e/scenarios/ai-conformance/validators/accelerators/dra_support/dra_cuda_test.go

@@ -0,0 +1,54 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dra_support
+
+import (
+	"fmt"
+	"testing"
+
+	"k8s.io/kops/tests/e2e/scenarios/ai-conformance/validators"
+)
+
+// TestDRAWorks is an additional test to verify that DRA is not only available, but also functional.
+// This is not currently required by AI conformance.
+func TestDRAWorks(t *testing.T) {
+	h := validators.NewValidatorHarness(t)
+
+	h.Logf("# DRA functional tests")
+
+	h.Logf("## Listing device classes")
+	deviceClasses := h.ListDeviceClasses()
+	for _, deviceClass := range deviceClasses {
+		h.Logf("* %s", deviceClass.Name())
+	}
+
+	h.Logf("## Listing resource slices")
+	resourceSlices := h.ListResourceSlices()
+	for _, resourceSlice := range resourceSlices {
+		h.Logf("* %s", resourceSlice.Name())
+	}
+
+	if !h.HasDeviceClass("gpu.nvidia.com") {
+		t.Skipf("gpu.nvidia.com device class not found; skipping")
+	}
+
+	h.Logf("## Run cuda-smoketest")
+	ns := "default"
+	h.ShellExec(fmt.Sprintf("kubectl apply --namespace %s -f testdata/cuda-smoketest.yaml", ns))
+	h.ShellExec(fmt.Sprintf("kubectl wait --for=condition=complete --namespace %s job/cuda-smoketest --timeout=5m", ns))
+	h.ShellExec(fmt.Sprintf("kubectl logs --namespace %s job/cuda-smoketest", ns))
+}

tests/e2e/scenarios/ai-conformance/validators/accelerators/dra_support/testdata/cuda-smoketest.yaml

@@ -0,0 +1,39 @@
+# Create a ResourceClaim and Job to test DRA
+apiVersion: resource.k8s.io/v1
+kind: ResourceClaim
+metadata:
+  name: cuda-smoketest
+spec:
+  devices:
+    requests:
+    - name: single-gpu
+      exactly:
+        deviceClassName: gpu.nvidia.com
+        allocationMode: ExactCount
+        count: 1
+
+---
+# TODO: Use Deployment+HPA
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: cuda-smoketest
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      tolerations:
+      - key: "nvidia.com/gpu"
+        operator: "Exists"
+        effect: "NoSchedule"
+      containers:
+      - name: test
+        image: nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda12.5.0-ubuntu22.04
+        command: ["/bin/sh", "-c"]
+        args: ["/cuda-samples/vectorAdd"]
+        resources:
+          claims:
+          - name: gpu
+      resourceClaims:
+      - name: gpu
+        resourceClaimName: cuda-smoketest

tests/e2e/scenarios/ai-conformance/validators/exec.go

@@ -0,0 +1,74 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validators
+
+import (
+	"bytes"
+	"os/exec"
+)
+
+// ShellExec executes the given shell command and returns the result. If the command fails, it fails the test.
+func (h *ValidatorHarness) ShellExec(shellCommand string) *CommandResult {
+	ctx := h.Context()
+	cmd := exec.CommandContext(ctx, "sh", "-c", shellCommand)
+	var stdout bytes.Buffer
+	cmd.Stdout = &stdout
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+
+	h.Logf("ShellExec(%q)", shellCommand)
+	err := cmd.Run()
+
+	result := &CommandResult{
+		stdout: stdout.String(),
+		stderr: stderr.String(),
+		err:    err,
+	}
+
+	h.output.OnShellExec(shellCommand, result)
+
+	if err != nil {
+		h.Logf("Command failed: %q", shellCommand)
+		h.Logf("Stdout: %s", result.Stdout())
+		h.Logf("Stderr: %s", result.Stderr())
+		h.Fatalf("Command execution %q failed with error: %v", shellCommand, err)
+	}
+
+	return result
+}
+
+// CommandResult encapsulates the result of executing a shell command, including stdout, stderr, and any error that occurred.
+type CommandResult struct {
+	stdout string
+	stderr string
+	err    error
+}
+
+// Stdout returns the standard output of the command execution.
+func (r *CommandResult) Stdout() string {
+	return r.stdout
+}
+
+// Stderr returns the standard error output of the command execution.
+func (r *CommandResult) Stderr() string {
+	return r.stderr
+}
+
+// Err returns the error that occurred during command execution, if any.
+func (r *CommandResult) Err() error {
+	return r.err
+}

tests/e2e/scenarios/ai-conformance/validators/harness.go

@@ -0,0 +1,72 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validators
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+// ValidatorHarness provides a common context and utilities for AI conformance validation.
+type ValidatorHarness struct {
+	t *testing.T
+
+	output OutputSink
+
+	dynamicClient dynamic.Interface
+	restConfig    *rest.Config
+}
+
+// NewValidatorHarness creates a new ValidatorHarness.
+func NewValidatorHarness(t *testing.T) *ValidatorHarness {
+	h := &ValidatorHarness{t: t}
+
+	h.output = createMarkdownOutput(t)
+	h.t.Cleanup(func() {
+		if err := h.output.Close(); err != nil {
+			h.t.Errorf("failed to close output: %v", err)
+		}
+	})
+
+	// use the current context in kubeconfig
+	kubeconfig := os.Getenv("KUBECONFIG")
+	if kubeconfig == "" {
+		home, err := os.UserHomeDir()
+		if err != nil {
+			h.Fatalf("failed to get user home directory: %v", err)
+		}
+		kubeconfig = filepath.Join(home, ".kube", "config")
+	}
+	restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+	if err != nil {
+		h.Fatalf("failed to build get kubeconfig: %v", err)
+	}
+	h.restConfig = restConfig
+
+	return h
+}
+
+// Context returns the context associated with the test, which can be used for command execution and API calls.
+func (h *ValidatorHarness) Context() context.Context {
+	return h.t.Context()
+}