Skip to content

Commit 9dfefe9

Browse files
justinsbjustinsb
committed
tests/ai-conformance: simple test and harness for DRA compliance
1 parent 6e37f93 commit 9dfefe9

6 files changed

Lines changed: 340 additions & 3 deletions

File tree

‎tests/e2e/scenarios/ai-conformance/run-test.sh‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,8 @@ echo "Waiting for Sample Workload to Complete..."
233233
kubectl wait --for=condition=complete job/test-gpu-pod --timeout=5m || true
234234
kubectl logs job/test-gpu-pod || echo "Failed to get logs"
235235

236-
# Note: The actual AI conformance test suite (e.g., k8s-ai-conformance binary)
237-
# would be executed here. For this scenario, we establish the compliant environment.
238-
239236
echo "AI Conformance Environment Setup Complete."
237+
238+
# Now run the actual AI conformance tests
239+
cd ${REPO_ROOT}/tests/e2e/scenarios/ai-conformance/validators
240+
go test -v ./... -timeout=60m

‎tests/e2e/scenarios/ai-conformance/validators/accelerators/dra_support/apiversions_test.go‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
Copyright The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package dra_support
18+
19+
import (
20+
"fmt"
21+
"strings"
22+
"testing"
23+
24+
"k8s.io/apimachinery/pkg/runtime/schema"
25+
"k8s.io/kops/tests/e2e/scenarios/ai-conformance/validators"
26+
)
27+
28+
func TestAPIVersions(t *testing.T) {
29+
// Support Dynamic Resource Allocation (DRA) APIs to enable more flexible and fine-grained resource requests beyond simple counts.
30+
h := validators.NewValidatorHarness(t)
31+
32+
h.Logf("# DRA API Availability")
33+
34+
gv := schema.GroupVersion{Group: "resource.k8s.io", Version: "v1"}
35+
36+
// Check resource.k8s.io/v1 is registered
37+
h.Logf("## Checking for DRA API version v1")
38+
{
39+
result := h.ShellExec(fmt.Sprintf("kubectl api-versions | grep %s", gv.Group+"/"+gv.Version))
40+
if !strings.Contains(result.Stdout(), "resource.k8s.io/v1\n") {
41+
h.Fatalf("Expected DRA API version group %s version %s, but it was not found", gv.Group, gv.Version)
42+
}
43+
h.Success("DRA API version %s is available.", gv.String())
44+
}
45+
46+
// Check all expected DRA API resources are registered
47+
for _, resource := range []string{"deviceclasses", "resourceclaims", "resourceclaimtemplates", "resourceslices"} {
48+
h.Logf("## Checking for %s", resource)
49+
result := h.ShellExec(fmt.Sprintf("kubectl api-resources --api-group=%s | grep %s", gv.Group, resource))
50+
if !strings.Contains(result.Stdout(), "resource.k8s.io/v1") {
51+
h.Fatalf("Expected DRA API resource %s to be available in group %s version %s, but it was not found", resource, gv.Group, gv.Version)
52+
}
53+
h.Success("DRA API resource %s is available.", resource)
54+
}
55+
}

‎tests/e2e/scenarios/ai-conformance/validators/exec.go‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
Copyright The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package validators
18+
19+
import (
20+
"bytes"
21+
"os/exec"
22+
)
23+
24+
func (h *ValidatorHarness) ShellExec(shellCommand string) *CommandResult {
25+
ctx := h.Context()
26+
cmd := exec.CommandContext(ctx, "sh", "-c", shellCommand)
27+
var stdout bytes.Buffer
28+
cmd.Stdout = &stdout
29+
var stderr bytes.Buffer
30+
cmd.Stderr = &stderr
31+
32+
err := cmd.Run()
33+
34+
result := &CommandResult{
35+
stdout: stdout.String(),
36+
stderr: stderr.String(),
37+
err: err,
38+
}
39+
40+
h.output.OnShellExec(shellCommand, result)
41+
42+
if err != nil {
43+
h.Logf("Command failed: %q", shellCommand)
44+
h.Logf("Stdout: %s", result.Stdout())
45+
h.Logf("Stderr: %s", result.Stderr())
46+
h.Fatalf("Command execution %q failed with error: %v", shellCommand, err)
47+
}
48+
49+
return result
50+
}
51+
52+
type CommandResult struct {
53+
stdout string
54+
stderr string
55+
err error
56+
}
57+
58+
func (r *CommandResult) Stdout() string {
59+
return r.stdout
60+
}
61+
62+
func (r *CommandResult) Stderr() string {
63+
return r.stderr
64+
}
65+
66+
func (r *CommandResult) Err() error {
67+
return r.err
68+
}

‎tests/e2e/scenarios/ai-conformance/validators/harness.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
Copyright The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package validators
18+
19+
import (
20+
"context"
21+
"testing"
22+
)
23+
24+
type ValidatorHarness struct {
25+
t *testing.T
26+
27+
output OutputSink
28+
}
29+
30+
func NewValidatorHarness(t *testing.T) *ValidatorHarness {
31+
h := &ValidatorHarness{t: t}
32+
33+
h.output = createMarkdownOutput(t)
34+
h.t.Cleanup(func() {
35+
if err := h.output.Close(); err != nil {
36+
h.t.Errorf("failed to close output: %v", err)
37+
}
38+
})
39+
return h
40+
}
41+
42+
func (h *ValidatorHarness) Context() context.Context {
43+
return h.t.Context()
44+
}

‎tests/e2e/scenarios/ai-conformance/validators/markdown.go‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
/*
2+
Copyright The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package validators
18+
19+
import (
20+
"fmt"
21+
"os"
22+
"path/filepath"
23+
"runtime"
24+
"testing"
25+
)
26+
27+
// MarkdownOutput is an implementation of OutputSink that writes output in Markdown format to a file.
28+
type MarkdownOutput struct {
29+
f *os.File
30+
t *testing.T
31+
}
32+
33+
func (o *MarkdownOutput) WriteText(text string) {
34+
o.printf("%s", text)
35+
}
36+
37+
func (o *MarkdownOutput) Success(text string) {
38+
o.printf("✓ %s\n", text)
39+
}
40+
41+
func (o *MarkdownOutput) Close() error {
42+
return o.f.Close()
43+
}
44+
45+
func (o *MarkdownOutput) OnShellExec(command string, results *CommandResult) {
46+
o.printf("```bash\n> %s\n", command)
47+
o.printf("%s", results.Stdout())
48+
o.printf("%s", results.Stderr())
49+
50+
if results.Err() != nil {
51+
o.printf("Error:\n```\n%v\n```\n", results.Err())
52+
}
53+
54+
o.printf("```\n")
55+
}
56+
57+
func (o *MarkdownOutput) printf(format string, args ...interface{}) {
58+
s := fmt.Sprintf(format, args...)
59+
_, err := fmt.Fprintln(o.f, s)
60+
if err != nil {
61+
o.t.Fatalf("failed to write to markdown file: %v", err)
62+
}
63+
}
64+
65+
func createMarkdownOutput(t *testing.T) OutputSink {
66+
// Get file path and other info from the current caller frame (depth 0)
67+
_, testFilename, _, ok := runtime.Caller(2)
68+
if !ok {
69+
t.Fatal("Could not get test caller")
70+
}
71+
_, baseFilename, _, ok := runtime.Caller(1)
72+
if !ok {
73+
t.Fatal("Could not get test caller")
74+
}
75+
76+
baseDir := filepath.Dir(baseFilename)
77+
78+
testRelativePath, err := filepath.Rel(baseDir, testFilename)
79+
if err != nil {
80+
t.Fatalf("failed to get relative path: %v", err)
81+
}
82+
83+
testRelativeDir := filepath.Dir(testRelativePath)
84+
85+
artifactsDir := os.Getenv("ARTIFACTS")
86+
if artifactsDir == "" {
87+
artifactsDir = "_artifacts"
88+
}
89+
outputBase := filepath.Join(artifactsDir, "ai-conformance")
90+
91+
outputPath := filepath.Join(outputBase, testRelativeDir, t.Name()+".md")
92+
93+
if err := os.MkdirAll(filepath.Dir(outputPath), 0755); err != nil {
94+
t.Fatalf("failed to create output directory %v: %v", filepath.Dir(outputPath), err)
95+
}
96+
97+
outputFile, err := os.Create(outputPath)
98+
if err != nil {
99+
t.Fatalf("failed to create markdown file: %v", err)
100+
}
101+
output := &MarkdownOutput{f: outputFile, t: t}
102+
return output
103+
}

‎tests/e2e/scenarios/ai-conformance/validators/output.go‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/*
2+
Copyright The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package validators
18+
19+
import (
20+
"fmt"
21+
"io"
22+
)
23+
24+
// OutputSink is an interface for writing output from the validators.
25+
type OutputSink interface {
26+
// WriteText writes a text string to the output sink.
27+
WriteText(text string)
28+
29+
// OnShellExec is called when a shell command is executed, with the command, its stdout, stderr, and any error that occurred.
30+
OnShellExec(command string, results *CommandResult)
31+
32+
// Success indicates a successful check, allowing the output sink to format it accordingly.
33+
Success(text string)
34+
35+
// Close closes the output sink and releases any resources.
36+
io.Closer
37+
}
38+
39+
// Logf is like t.Logf, but also writes to the sinks.
40+
func (h *ValidatorHarness) Logf(format string, args ...interface{}) {
41+
s := fmt.Sprintf(format, args...)
42+
43+
h.output.WriteText(s)
44+
h.t.Logf(format, args...)
45+
}
46+
47+
// Log is like t.Log, but also writes to the sinks.
48+
func (h *ValidatorHarness) Log(s string) {
49+
h.output.WriteText(s)
50+
h.t.Log(s)
51+
}
52+
53+
// Fatalf is like t.Fatalf, but also writes to the sinks.
54+
func (h *ValidatorHarness) Fatalf(format string, args ...interface{}) {
55+
s := fmt.Sprintf(format, args...)
56+
57+
h.output.WriteText("FATAL: " + s)
58+
h.t.Fatalf(format, args...)
59+
}
60+
61+
// Success is like Logf, but indicates a successful check.
62+
func (h *ValidatorHarness) Success(format string, args ...interface{}) {
63+
s := fmt.Sprintf(format, args...)
64+
h.output.Success(s)
65+
h.t.Logf("SUCCESS: "+format, args...)
66+
}

0 commit comments

Comments
 (0)