Skip to content

Commit 9a8c64f

Browse files
authored
Merge pull request #18063 from justinsb/aiconformance_gateway_api
[aiconformance]: Validate networking/ai_inference requirements
2 parents 80f3199 + d096fd3 commit 9a8c64f

File tree

10 files changed

+534
-14
lines changed

10 files changed

+534
-14
lines changed

tests/e2e/scenarios/ai-conformance/run-test.sh

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@ set -o nounset
1919
set -o pipefail
2020

2121
REPO_ROOT=$(git rev-parse --show-toplevel)
22+
23+
BIN_DIR="${REPO_ROOT}/.build/bin"
24+
mkdir -p "${BIN_DIR}"
25+
export PATH="${BIN_DIR}:$PATH"
26+
2227
source "${REPO_ROOT}"/tests/e2e/scenarios/lib/common.sh
2328

2429
# AI Conformance requirements:
@@ -190,7 +195,23 @@ helm upgrade -i kuberay-operator kuberay/kuberay-operator \
190195

191196
# Kueue
192197
echo "Installing Kueue..."
193-
kubectl apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.8/manifests.yaml
198+
helm install kueue https://github.com/kubernetes-sigs/kueue/releases/download/v0.16.2/kueue-0.16.2.tgz \
199+
--namespace kueue-system \
200+
--create-namespace \
201+
--wait --timeout 300s
202+
203+
# Gateway API
204+
kubectl kustomize "github.com/kubernetes-sigs/gateway-api/config/crd?ref=v1.5.0" | kubectl apply -f -
205+
206+
# Gateway API Implenmentation - Istio
207+
helm repo add istio https://istio-release.storage.googleapis.com/charts
208+
helm repo update
209+
210+
wget https://github.com/istio/istio/releases/download/1.29.1/istioctl-1.29.1-linux-amd64.tar.gz -O "${BIN_DIR}/istioctl.tar.gz"
211+
tar -xzf "${BIN_DIR}/istioctl.tar.gz" -C "${BIN_DIR}"
212+
rm "${BIN_DIR}/istioctl.tar.gz"
213+
214+
istioctl install --set profile=minimal -y
194215

195216
echo "----------------------------------------------------------------"
196217
echo "Verifying Cluster and Components"

tests/e2e/scenarios/ai-conformance/validators/harness.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@ type ValidatorHarness struct {
4242

4343
// testNamespace is a per-test namespace to use for creating resources. It is lazily initialized when TestNamespace() is called.
4444
testNamespace string
45+
46+
// objectIDs tracks the Kubernetes objects that have been created or observed during the test. This can be used for cleanup or reporting.
47+
objectIDs []*KubeObjectID
4548
}
4649

4750
// NewValidatorHarness creates a new ValidatorHarness.

tests/e2e/scenarios/ai-conformance/validators/kube.go

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -206,10 +206,21 @@ func (h *ValidatorHarness) TestNamespace() string {
206206
}
207207

208208
// ApplyManifest applies a Kubernetes manifest from the given file path to the specified namespace.
209+
// It returns the list of objects found in the manifest.
209210
// We use kubectl so that the output is clear and in theory someone could run the same commands themselves to debug.
210-
func (h *ValidatorHarness) ApplyManifest(namespace string, manifestPath string) {
211-
h.Logf("Applying manifest %q to namespace %q", manifestPath, namespace)
212-
h.ShellExec(fmt.Sprintf("kubectl apply -n %s -f %s", namespace, manifestPath))
211+
func (h *ValidatorHarness) ApplyManifest(defaultNamespace string, manifestPath string) []*KubeObjectID {
212+
h.Logf("Applying manifest %q to namespace %q", manifestPath, defaultNamespace)
213+
214+
objects, err := h.parseManifestObjects(manifestPath, defaultNamespace)
215+
if err != nil {
216+
h.Fatalf("failed to parse manifest %s: %v", manifestPath, err)
217+
}
218+
219+
h.objectIDs = append(h.objectIDs, objects...)
220+
221+
h.ShellExec(fmt.Sprintf("kubectl apply -n %s -f %s", defaultNamespace, manifestPath))
222+
223+
return objects
213224
}
214225

215226
// dumpNamespaceResources dumps key resources from the namespace to the artifacts directory for debugging.
@@ -226,17 +237,20 @@ func (h *ValidatorHarness) dumpNamespaceResources(ctx context.Context, ns string
226237
return
227238
}
228239

229-
resourceTypes := []string{
230-
"pods",
231-
"jobs",
232-
"deployments",
233-
"statefulsets",
234-
"services",
235-
"events",
240+
resourceTypes := make(map[string]bool)
241+
for _, objectID := range h.objectIDs {
242+
gvk := objectID.GVK()
243+
id := fmt.Sprintf("%s.%s", gvk.Kind, gvk.Group)
244+
resourceTypes[id] = true
236245
}
237246

238-
for _, resourceType := range resourceTypes {
239-
if err := h.dumpResource(ctx, ns, resourceType, filepath.Join(clusterInfoDir, resourceType+".yaml")); err != nil {
247+
// Always include Events, Pods: they are usually not in the manifest, but are often critical for understanding failures.
248+
resourceTypes["Events"] = true
249+
resourceTypes["Pods"] = true
250+
251+
for resourceType := range resourceTypes {
252+
filename := strings.ToLower(resourceType) + ".yaml"
253+
if err := h.dumpResource(ctx, ns, resourceType, filepath.Join(clusterInfoDir, filename)); err != nil {
240254
h.Logf("failed to dump resource %s: %v", resourceType, err)
241255
}
242256
}
Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
/*
2+
Copyright The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package validators
18+
19+
import (
20+
"bufio"
21+
"bytes"
22+
"encoding/json"
23+
"fmt"
24+
"io"
25+
"os"
26+
"strings"
27+
28+
"k8s.io/apimachinery/pkg/runtime/schema"
29+
"k8s.io/apimachinery/pkg/util/yaml"
30+
)
31+
32+
// KubeObjectID represents a Kubernetes object parsed from a manifest.
33+
type KubeObjectID struct {
34+
h *ValidatorHarness
35+
36+
gvk schema.GroupVersionKind
37+
name string
38+
namespace string
39+
}
40+
41+
// GVK returns the GroupVersionKind of the object.
42+
func (o *KubeObjectID) GVK() schema.GroupVersionKind {
43+
return o.gvk
44+
}
45+
46+
// Name returns the name of the object.
47+
func (o *KubeObjectID) Name() string {
48+
return o.name
49+
}
50+
51+
// Namespace returns the namespace of the object.
52+
func (o *KubeObjectID) Namespace() string {
53+
return o.namespace
54+
}
55+
56+
// KubectlWaitOption configures the behavior of KubectlWait.
57+
type KubectlWaitOption func(*kubectlWaitOptions)
58+
59+
type kubectlWaitOptions struct {
60+
timeout string
61+
}
62+
63+
// WithTimeout sets the timeout for kubectl wait.
64+
func WithTimeout(timeout string) KubectlWaitOption {
65+
return func(o *kubectlWaitOptions) {
66+
o.timeout = timeout
67+
}
68+
}
69+
70+
// KubectlWait waits for the object to become healthy using kubectl wait.
71+
// The wait condition is determined by the object's kind.
72+
// Objects that don't have a meaningful wait condition are skipped.
73+
func (o *KubeObjectID) KubectlWait(opts ...KubectlWaitOption) {
74+
condition := waitConditionForKind(o.gvk.Kind)
75+
if condition == "" {
76+
o.h.Errorf("No wait condition for %s/%s, cannot wait", o.gvk.Kind, o.name)
77+
return
78+
}
79+
80+
options := &kubectlWaitOptions{timeout: "300s"}
81+
for _, opt := range opts {
82+
opt(options)
83+
}
84+
85+
resourceType := kubectlResourceType(o.gvk)
86+
o.h.ShellExec(fmt.Sprintf("kubectl wait -n %s %s %s/%s --timeout=%s",
87+
o.namespace, condition, resourceType, o.name, options.timeout))
88+
}
89+
90+
// waitConditionForKind returns the kubectl wait --for condition appropriate for the given kind.
91+
// Returns empty string for kinds that don't have a meaningful wait condition.
92+
func waitConditionForKind(kind string) string {
93+
switch kind {
94+
case "HTTPRoute":
95+
return "--for=jsonpath='{.status.parents[0].conditions[?(@.type==\"Accepted\")].status}'=True"
96+
case "Gateway":
97+
return "--for=condition=Programmed"
98+
case "Deployment":
99+
return "--for=condition=Available"
100+
case "Pod":
101+
return "--for=condition=Ready"
102+
case "Job":
103+
return "--for=condition=Complete"
104+
default:
105+
return ""
106+
}
107+
}
108+
109+
// kubectlResourceType returns the kubectl resource type string for a GVK.
110+
// For core API resources, this is the lowercase kind.
111+
// For other groups, this is "kind.group" (lowercased).
112+
func kubectlResourceType(gvk schema.GroupVersionKind) string {
113+
kind := strings.ToLower(gvk.Kind)
114+
if gvk.Group == "" {
115+
return kind
116+
}
117+
return kind + "." + gvk.Group
118+
}
119+
120+
// parseManifestObjects parses a multi-document YAML manifest file and returns the objects found.
121+
func (h *ValidatorHarness) parseManifestObjects(manifestPath string, defaultNamespace string) ([]*KubeObjectID, error) {
122+
data, err := os.ReadFile(manifestPath)
123+
if err != nil {
124+
return nil, fmt.Errorf("reading manifest %s: %w", manifestPath, err)
125+
}
126+
127+
var objects []*KubeObjectID
128+
reader := yaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(data)))
129+
for {
130+
doc, err := reader.Read()
131+
if err == io.EOF {
132+
break
133+
}
134+
if err != nil {
135+
return nil, fmt.Errorf("reading YAML document from %s: %w", manifestPath, err)
136+
}
137+
138+
// Skip empty documents
139+
doc = bytes.TrimSpace(doc)
140+
if len(doc) == 0 {
141+
continue
142+
}
143+
144+
obj, err := h.parseMinimalObject(doc, defaultNamespace)
145+
if err != nil {
146+
return nil, fmt.Errorf("parsing object from %s: %w", manifestPath, err)
147+
}
148+
if obj != nil {
149+
objects = append(objects, obj)
150+
}
151+
}
152+
153+
return objects, nil
154+
}
155+
156+
// parseMinimalObject extracts GVK and name from a YAML document without full deserialization.
157+
func (h *ValidatorHarness) parseMinimalObject(doc []byte, defaultNamespace string) (*KubeObjectID, error) {
158+
// Use the YAML-to-JSON utility to decode into a map
159+
jsonData, err := yaml.ToJSON(doc)
160+
if err != nil {
161+
return nil, fmt.Errorf("converting YAML to JSON: %w", err)
162+
}
163+
164+
// Quick parse using the unstructured decoder
165+
var raw map[string]interface{}
166+
if err := json.Unmarshal(jsonData, &raw); err != nil {
167+
return nil, fmt.Errorf("parsing JSON: %w", err)
168+
}
169+
170+
apiVersion, _ := raw["apiVersion"].(string)
171+
kind, _ := raw["kind"].(string)
172+
if apiVersion == "" || kind == "" {
173+
return nil, nil
174+
}
175+
176+
metadata, _ := raw["metadata"].(map[string]interface{})
177+
name, _ := metadata["name"].(string)
178+
namespace, _ := metadata["namespace"].(string)
179+
180+
if namespace == "" {
181+
namespace = defaultNamespace
182+
}
183+
gv, err := schema.ParseGroupVersion(apiVersion)
184+
if err != nil {
185+
return nil, fmt.Errorf("parsing apiVersion %q: %w", apiVersion, err)
186+
}
187+
188+
return &KubeObjectID{
189+
h: h,
190+
gvk: gv.WithKind(kind),
191+
name: name,
192+
namespace: namespace,
193+
}, nil
194+
}

tests/e2e/scenarios/ai-conformance/validators/markdown.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ type MarkdownOutput struct {
3232

3333
// WriteText writes the given plain text to the markdown file.
3434
func (o *MarkdownOutput) WriteText(text string) {
35+
o.printf("\n")
3536
o.printf("%s", text)
3637
}
3738

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
/*
2+
Copyright The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package ai_inference
18+
19+
import (
20+
"testing"
21+
22+
"k8s.io/kops/tests/e2e/scenarios/ai-conformance/validators"
23+
)
24+
25+
// TestNetworking_AIInference corresponds to the networking/ai_inference conformance requirement,
26+
// tested using the KubeRay operator as an example of a complex AI operator with a CRD.
27+
func TestNetworking_AIInference(t *testing.T) {
28+
// Description:
29+
// Support the Kubernetes Gateway API with an implementation for advanced traffic management for inference services,
30+
// which enables capabilities like weighted traffic splitting,
31+
// header-based routing (for OpenAI protocol headers),
32+
// and optional integration with service meshes."
33+
34+
h := validators.NewValidatorHarness(t)
35+
36+
h.Logf("# Gateway API support for AI inference")
37+
38+
h.Run("weighted-traffic-splitting", func(h *validators.ValidatorHarness) {
39+
h.Logf("## Verify Weighted Traffic Splitting")
40+
ns := h.TestNamespace()
41+
42+
objects := h.ApplyManifest(ns, "testdata/weighted-traffic-splitting.yaml")
43+
44+
for _, obj := range objects {
45+
if obj.GVK().Kind == "HTTPRoute" {
46+
h.Logf("Waiting for HTTPRoute %s to be accepted", obj.Name())
47+
obj.KubectlWait()
48+
}
49+
}
50+
})
51+
52+
h.Run("header-based-routing", func(h *validators.ValidatorHarness) {
53+
h.Logf("## Verify Header Based Routing")
54+
ns := h.TestNamespace()
55+
56+
objects := h.ApplyManifest(ns, "testdata/header-based-routing.yaml")
57+
58+
for _, obj := range objects {
59+
if obj.GVK().Kind == "HTTPRoute" {
60+
h.Logf("Waiting for HTTPRoute %s to be accepted", obj.Name())
61+
obj.KubectlWait()
62+
}
63+
}
64+
})
65+
66+
if h.AllPassed() {
67+
h.RecordConformance("networking/ai_inference")
68+
}
69+
}

0 commit comments

Comments
 (0)