[DO NOT MERGE] Perf test for pod-with-pvc

clusterloader2/pkg/framework/multiclient.go

@@ -78,6 +78,8 @@ func NewMultiDynamicClient(kubeconfigPath string, number int) (*MultiDynamicClie
 		if err != nil {
 			return nil, fmt.Errorf("config prepare failed: %v", err)
 		}
+		conf.QPS = 500
+		conf.Burst = 500
 		if number < 1 {
 			return nil, fmt.Errorf("incorrect clients number")
 		}

clusterloader2/testing/scheduler-throughput/config-pvc.yaml

@@ -0,0 +1,36 @@
+{{$totalSchedulerThroughputPods := DefaultParam .CL2_SCHEDULER_THROUGHPUT_PODS 200000}}
+{{$defaultQps := DefaultParam .CL2_DEFAULT_QPS  1000}}
+{{$defaultBurst := DefaultParam .CL2_DEFAULT_BURST 10000}}
+{{$uniformQps := DefaultParam .CL2_UNIFORM_QPS 1000}}
+
+{{$SCHEDULER_THROUGHPUT_THRESHOLD := DefaultParam .CL2_SCHEDULER_THROUGHPUT_THRESHOLD 400}}
+
+name: direct-scheduler-throughput
+namespace:
+  prefix: pvc-sched-test
+  number: 1
+  enableExistingNamespaces: true
+  deleteAutomanagedNamespaces: false
+tuningSets:
+# default is a tuningset that is meant to be used when we don't have any specific requirements on pace of operations.
+- name: default
+  globalQPSLoad:
+    qps: {{$defaultQps}}
+    burst: {{$defaultBurst}}
+- name: UniformQPS
+  qpsLoad:
+    qps: {{$uniformQps}}
+    burst: {{$defaultBurst}}
+steps:
+- name: create PVCs
+  phases:
+  - namespaceRange:
+      min: 1
+      max: 1
+    replicasPerNamespace: {{$totalSchedulerThroughputPods}}
+    tuningSet: UniformQPS
+    objectBundle:
+    - basename: direct-scheduler-throughput-pod-pvc
+      objectTemplatePath: pvc-default.yaml
+      templateFillMap:
+        Group: direct-scheduler-throughput

clusterloader2/testing/scheduler-throughput/config.yaml

@@ -1,12 +1,15 @@
-{{$totalSchedulerThroughputPods := DefaultParam .CL2_SCHEDULER_THROUGHPUT_PODS 5000}}
-{{$defaultQps := DefaultParam .CL2_DEFAULT_QPS  500}}
-{{$defaultBurst := DefaultParam .CL2_DEFAULT_BURST 1000}}
-{{$uniformQps := DefaultParam .CL2_UNIFORM_QPS 500}}
+{{$totalSchedulerThroughputPods := DefaultParam .CL2_SCHEDULER_THROUGHPUT_PODS 200000}}
+{{$defaultQps := DefaultParam .CL2_DEFAULT_QPS  1000}}
+{{$defaultBurst := DefaultParam .CL2_DEFAULT_BURST 10000}}
+{{$uniformQps := DefaultParam .CL2_UNIFORM_QPS 1000}}
 
 {{$SCHEDULER_THROUGHPUT_THRESHOLD := DefaultParam .CL2_SCHEDULER_THROUGHPUT_THRESHOLD 400}}
 
 name: direct-scheduler-throughput
 namespace:
+  prefix: pvc-sched-test
+  enableExistingNamespaces: true
+  deleteAutomanagedNamespaces: false
   number: 1
 tuningSets:
 # default is a tuningset that is meant to be used when we don't have any specific requirements on pace of operations.
@@ -17,6 +20,7 @@ tuningSets:
 - name: UniformQPS
   qpsLoad:
     qps: {{$uniformQps}}
+    burst: {{$defaultBurst}}
 steps:
 - name: Creating scheduler throughput measurements
   measurements:
@@ -51,7 +55,7 @@ steps:
     Method: WaitForRunningPods
     Params:
       action: gather
-      timeout: 5m
+      timeout: 20m
       desiredPodCount: {{$totalSchedulerThroughputPods}}
       labelSelector: group = direct-scheduler-throughput
 - name: Collecting scheduler throughput measurements

clusterloader2/testing/scheduler-throughput/create_pv.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+
+import argparse
+import concurrent.futures
+import threading
+import time
+from kubernetes import client, config
+
+BATCH_SIZE = 1000
+PARALLEL = 20
+PREFIX = "direct-scheduler-throughput-pod"
+STORAGE_CLASS = "sched-test"
+
+config.load_kube_config()
+
+_local = threading.local()
+
+
+def get_core_api():
+    if not hasattr(_local, "core_v1"):
+        _local.core_v1 = client.CoreV1Api()
+    return _local.core_v1
+
+
+def create_pv(i, namespace):
+    api = get_core_api()
+    pv = client.V1PersistentVolume(
+        metadata=client.V1ObjectMeta(name=f"{PREFIX}-pv-{i}"),
+        spec=client.V1PersistentVolumeSpec(
+            capacity={"storage": "1Gi"},
+            access_modes=["ReadWriteOnce"],
+            storage_class_name=STORAGE_CLASS,
+            volume_mode="Filesystem",
+            persistent_volume_reclaim_policy="Delete",
+            claim_ref=client.V1ObjectReference(
+                namespace=namespace,
+                name=f"{PREFIX}-pvc-{i}",
+            ),
+            host_path=client.V1HostPathVolumeSource(
+                path="/tmp/sched-test",
+                type="DirectoryOrCreate",
+            ),
+        ),
+    )
+    try:
+        api.create_persistent_volume(pv)
+    except client.exceptions.ApiException as e:
+        if e.status == 409:
+            pass
+        else:
+            raise
+
+
+def run_batch(start, end, namespace):
+    for i in range(start, end + 1):
+        create_pv(i, namespace)
+    print(f"Batch {start}-{end} done ({end - start + 1} PVs)")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--total", type=int, default=100_000)
+    parser.add_argument("--start", type=int, default=1, help="Resume from this index (inclusive)")
+    parser.add_argument("--parallel", type=int, default=PARALLEL)
+    parser.add_argument("--batch-size", type=int, default=BATCH_SIZE)
+    parser.add_argument("--namespace", type=str, default="default", help="Namespace for claimRef")
+    args = parser.parse_args()
+
+    start_idx = args.start
+    end_idx = args.total
+    batch_size = args.batch_size
+    total_to_create = end_idx - start_idx + 1
+
+    batches = []
+    for s in range(start_idx, end_idx + 1, batch_size):
+        e = min(s + batch_size - 1, end_idx)
+        batches.append((s, e))
+
+    print(f"Creating PVs {start_idx}-{end_idx} ({total_to_create} PVs) with {args.parallel} workers...")
+    start_time = time.time()
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=args.parallel) as pool:
+        futures = [pool.submit(run_batch, s, e, args.namespace) for s, e in batches]
+        concurrent.futures.wait(futures)
+        for f in futures:
+            f.result()
+
+    elapsed = time.time() - start_time
+    print(f"Done in {elapsed:.1f}s ({total_to_create / elapsed:.0f} PVs/s)")
+
+
+if __name__ == "__main__":
+    main()

clusterloader2/testing/scheduler-throughput/pod-default.yaml

@@ -1,10 +1,18 @@
 apiVersion: v1
 kind: Pod
 metadata:
-  generateName: pod-churn-
+  name: pod-churn-{{.Index}}
   labels:
     group: {{.Group}}  
 spec:
+  volumes:
+    - name: storage
+      persistentVolumeClaim:
+        claimName: direct-scheduler-throughput-pod-pvc-{{.Index}}
   containers:
   - image: registry.k8s.io/pause:3.9
-    name: pause
+    name: pause
+    volumeMounts:
+        - name: storage
+          mountPath: /home/node
+          subPath: home

clusterloader2/testing/scheduler-throughput/pvc-default.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: direct-scheduler-throughput-pod-pvc-{{.Index}}
+spec:
+  accessModes: [ReadWriteOnce]
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: sched-test
+  volumeName: direct-scheduler-throughput-pod-pv-{{.Index}}
+  volumeMode: Filesystem