Skip to content

Commit 01a8139

Browse files
authored
Merge pull request #18084 from justinsb/write_aiconformance_report
[aiconformance]: Write conformance report to file
2 parents 0ed329a + 1f034e1 commit 01a8139

16 files changed

Lines changed: 371 additions & 22 deletions

File tree

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
/*
2+
Copyright The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package aiconformance

tests/e2e/scenarios/ai-conformance/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ require (
66
github.com/yuin/goldmark v1.7.16
77
k8s.io/apimachinery v0.35.2
88
k8s.io/client-go v0.35.2
9+
sigs.k8s.io/yaml v1.6.0
910
)
1011

1112
require (
@@ -32,5 +33,4 @@ require (
3233
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
3334
sigs.k8s.io/randfill v1.0.0 // indirect
3435
sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
35-
sigs.k8s.io/yaml v1.6.0 // indirect
3636
)

tests/e2e/scenarios/ai-conformance/run-test.sh

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,9 @@ if [[ "${AVAILABILITY}" == "false" ]]; then
5151
fi
5252
rm -f "${SCENARIO_ROOT}/tools/check-aws-availability/check-aws-availability"
5353

54+
# Put kOps onto PATH
5455
kops-acquire-latest
56+
cp "${KOPS}" "${BIN_DIR}/kops"
5557

5658
# Cluster Configuration
5759
# - Networking: Cilium with Gateway API enabled
@@ -72,7 +74,7 @@ kops-up
7274

7375
# Now add an instance group for GPU nodes with the appropriate labels for NVIDIA DRA
7476
# TODO: find zones, match images, etc. rather than hard-coding
75-
${KOPS} create --name "${CLUSTER_NAME}" -f - <<EOF
77+
kops create --name "${CLUSTER_NAME}" -f - <<EOF
7678
apiVersion: kops.k8s.io/v1alpha2
7779
kind: InstanceGroup
7880
metadata:
@@ -92,7 +94,7 @@ spec:
9294
- us-east-2c
9395
EOF
9496

95-
${KOPS} update cluster --name "${CLUSTER_NAME}" --yes --admin
97+
kops update cluster --name "${CLUSTER_NAME}" --yes --admin
9698

9799
echo "Listing node with their labels..."
98100
kubectl get nodes --show-labels
@@ -218,7 +220,7 @@ echo "Verifying Cluster and Components"
218220
echo "----------------------------------------------------------------"
219221

220222
# Wait for kOps validation
221-
"${KOPS}" validate cluster --wait=15m
223+
kops validate cluster --wait=15m
222224

223225
# Verify Components
224226
echo "Verifying NVIDIA Device Plugin..."
@@ -351,3 +353,7 @@ echo "AI Conformance Environment Setup Complete."
351353
# Now run the actual AI conformance tests
352354
cd "${REPO_ROOT}/tests/e2e/scenarios/ai-conformance/validators"
353355
go test -v ./... -timeout=60m
356+
357+
# Compile and write the conformance report
358+
cd "${REPO_ROOT}/tests/e2e/scenarios/ai-conformance"
359+
go run ./tools/write-conformance-report
Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
/*
2+
Copyright The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package main
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"os"
23+
"os/exec"
24+
"path/filepath"
25+
"strings"
26+
27+
"k8s.io/kops/tests/e2e/scenarios/ai-conformance/validators/conformance"
28+
)
29+
30+
func main() {
31+
ctx := context.Background()
32+
if err := run(ctx); err != nil {
33+
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
34+
os.Exit(1)
35+
}
36+
}
37+
38+
func run(ctx context.Context) error {
39+
artifactsDir := os.Getenv("ARTIFACTS")
40+
if artifactsDir == "" {
41+
artifactsDir = "_artifacts"
42+
}
43+
reportPath := filepath.Join(artifactsDir, "ai-conformance.yaml")
44+
45+
kubernetesVersion := ""
46+
kopsVersion := ""
47+
48+
if kubernetesVersion == "" {
49+
v, err := getKubernetesVersion(ctx)
50+
if err != nil {
51+
return fmt.Errorf("getting kubernetes version: %v", err)
52+
}
53+
kubernetesVersion = v
54+
}
55+
56+
if kopsVersion == "" {
57+
v, err := getKopsVersion(ctx)
58+
if err != nil {
59+
return fmt.Errorf("getting kOps version: %v", err)
60+
}
61+
kopsVersion = v
62+
}
63+
64+
metadata := conformance.Metadata{
65+
KubernetesVersion: kubernetesVersion,
66+
PlatformName: "kOps",
67+
PlatformVersion: kopsVersion,
68+
VendorName: "kOps Project",
69+
WebsiteURL: "https://kops.sigs.k8s.io/",
70+
RepoURL: "https://github.com/kubernetes/kops",
71+
DocumentationURL: "https://kops.sigs.k8s.io/",
72+
ProductLogoURL: "https://github.com/kubernetes/kops/blob/master/docs/img/logo.png",
73+
Description: "Kubernetes Operations (kOps) - Production Grade k8s Installation, Upgrades and Management",
74+
ContactEmailAddress: "sig-cluster-lifecycle@kubernetes.io",
75+
}
76+
77+
if err := conformance.WriteReport(artifactsDir, metadata); err != nil {
78+
panic(fmt.Sprintf("Failed to write conformance report: %v", err))
79+
}
80+
81+
fmt.Printf("Conformance report written to: %s\n", reportPath)
82+
return nil
83+
}
84+
85+
// getKubernetesVersion retrieves the Kubernetes version by running "kubectl version" and parsing the output.
86+
func getKubernetesVersion(ctx context.Context) (string, error) {
87+
cmd := exec.CommandContext(ctx, "kubectl", "version")
88+
output, err := cmd.Output()
89+
if err != nil {
90+
return "", fmt.Errorf("failed to execute kubectl version: %v", err)
91+
}
92+
lines := strings.Split(string(output), "\n")
93+
for _, line := range lines {
94+
suffix, ok := strings.CutPrefix(line, "Server Version:")
95+
if ok {
96+
return strings.TrimSpace(suffix), nil
97+
}
98+
}
99+
return "", fmt.Errorf("server version not found in kubectl output: %q", string(output))
100+
}
101+
102+
// getKopsVersion retrieves the kOps version by running "kops version --short".
103+
func getKopsVersion(ctx context.Context) (string, error) {
104+
cmd := exec.CommandContext(ctx, "kops", "version", "--short")
105+
output, err := cmd.Output()
106+
if err != nil {
107+
return "", fmt.Errorf("failed to execute kops version: %v", err)
108+
}
109+
return strings.TrimSpace(string(output)), nil
110+
}

tests/e2e/scenarios/ai-conformance/validators/accelerators/dra_support/apiversions_test.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,4 +54,8 @@ func TestAcceleratorsDRASupport(t *testing.T) {
5454
}
5555
h.Success("DRA API resource %s is available.", resource)
5656
}
57+
58+
if h.AllPassed() {
59+
h.RecordConformance("accelerators", "dra_support")
60+
}
5761
}

tests/e2e/scenarios/ai-conformance/validators/accelerators/driver_runtime_management/driver_runtime_management_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ func TestAcceleratorsDriverRuntimeManagement(t *testing.T) {
132132

133133
// Record conformance
134134
if h.AllPassed() {
135-
h.RecordConformance("accelerators/driver_runtime_management")
135+
h.RecordConformance("accelerators", "driver_runtime_management")
136136
h.Success("Driver Runtime Management conformance test PASSED")
137137
}
138138
}
Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
Copyright The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package conformance
18+
19+
import (
20+
"fmt"
21+
"os"
22+
"path/filepath"
23+
24+
"sigs.k8s.io/yaml"
25+
)
26+
27+
// Report is the conformance report data, including metadata and test results.
28+
type Report struct {
29+
Metadata Metadata `json:"metadata"`
30+
Spec map[string][]Info `json:"spec"`
31+
}
32+
33+
// Metadata is the header for the conformance report, including Kubernetes version, platform information, vendor details, and contact information.
34+
type Metadata struct {
35+
KubernetesVersion string `json:"kubernetesVersion,omitempty"`
36+
PlatformName string `json:"platformName,omitempty"`
37+
PlatformVersion string `json:"platformVersion,omitempty"`
38+
VendorName string `json:"vendorName,omitempty"`
39+
WebsiteURL string `json:"websiteUrl,omitempty"`
40+
RepoURL string `json:"repoUrl,omitempty"`
41+
DocumentationURL string `json:"documentationUrl,omitempty"`
42+
ProductLogoURL string `json:"productLogoUrl,omitempty"`
43+
Description string `json:"description,omitempty"`
44+
ContactEmailAddress string `json:"contactEmailAddress,omitempty"`
45+
}
46+
47+
// WriteReport scans the artifacts directory for attestation files and writes
48+
// the final conformance report to the artifacts directory.
49+
func WriteReport(artifactsDir string, metadata Metadata) error {
50+
report := &Report{
51+
Metadata: metadata,
52+
Spec: make(map[string][]Info),
53+
}
54+
55+
// Walk the artifactsDir tree, looking for "ai-conformance.yaml"
56+
if err := filepath.WalkDir(artifactsDir, func(path string, d os.DirEntry, err error) error {
57+
if err != nil {
58+
return fmt.Errorf("walking artifacts directory %q: %w", artifactsDir, err)
59+
}
60+
if d.IsDir() {
61+
return nil
62+
}
63+
if d.Name() != "ai-conformance.yaml" {
64+
return nil
65+
}
66+
67+
b, err := os.ReadFile(path)
68+
if err != nil {
69+
return fmt.Errorf("reading attestation file %q: %w", path, err)
70+
}
71+
72+
var attestation Attestation
73+
if err := yaml.Unmarshal(b, &attestation); err != nil {
74+
return fmt.Errorf("parsing attestation file %q: %w", path, err)
75+
}
76+
77+
report.Spec[attestation.Section] = append(report.Spec[attestation.Section], attestation.Info)
78+
return nil
79+
}); err != nil {
80+
return fmt.Errorf("scanning artifacts directory for attestations: %w", err)
81+
}
82+
83+
if err := os.MkdirAll(artifactsDir, 0755); err != nil {
84+
return fmt.Errorf("creating artifacts directory %q: %w", artifactsDir, err)
85+
}
86+
87+
reportPath := filepath.Join(artifactsDir, "ai-conformance.yaml")
88+
b, err := yaml.Marshal(report)
89+
if err != nil {
90+
return fmt.Errorf("marshaling conformance report: %w", err)
91+
}
92+
if err := os.WriteFile(reportPath, b, 0644); err != nil {
93+
return fmt.Errorf("writing conformance report to %q: %w", reportPath, err)
94+
}
95+
96+
return nil
97+
}

0 commit comments

Comments
 (0)