Move Prometheus to addon nodes in GCE

AWS is missing firewall rules to allow Prometheus to scrape control
plane.

Author: serathius

tests/e2e/scenarios/scalability/pre-test.sh

@@ -20,16 +20,11 @@ set -x
 # We need an instance group with a large single single node for addons such as prometheus, exec-service, etc.
 # In kubeup we used to call it heapster due to name of first addon, but now we call it addons.
 
-if [[ "${CLOUD_PROVIDER}" == "aws" ]]; then
-  kops create instancegroup addons --edit=false --role node --zone us-east-2b
-  kops edit instancegroup addons --set=spec.machineType="${ADDONS_NODE_SIZE:-c7a.8xlarge}" \
-    --set=spec.maxSize=1 --set=spec.minSize=1 --set=spec.image="ssm:/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id"
-elif [[ "${CLOUD_PROVIDER}" == "gce" ]]; then
+if [[ "${CLOUD_PROVIDER}" == "gce" ]]; then
   kops create instancegroup addons --edit=false --role node --zone us-east1-b
   kops edit instancegroup addons --set=spec.machineType="${ADDONS_NODE_SIZE:-c3-standard-22}" \
     --set=spec.maxSize=1 --set=spec.minSize=1 --set=spec.rootVolume.type=hyperdisk-balanced \
     --set=spec.image="${INSTANCE_IMAGE:-ubuntu-os-cloud/ubuntu-2404-noble-amd64-v20251001}"
+  kops update cluster --yes
+  kops validate cluster --wait 10m
 fi
-
-kops update cluster --yes
-kops validate cluster --wait 10m

tests/e2e/scenarios/scalability/run-test.sh

@@ -127,6 +127,8 @@ KOPS_FEATURE_FLAGS="EtcdEventsHTTP,${KOPS_FEATURE_FLAGS:-}"
 
 # AWS ONLY feature flags
 if [[ "${CLOUD_PROVIDER}" == "aws" ]]; then
+  # AWS doesn't run dedicated addons node
+  export CL2_PROMETHEUS_TOLERATE_MASTER="true"
   # Enable creating a single nodes instance group
   KOPS_FEATURE_FLAGS="AWSSingleNodesInstanceGroup,${KOPS_FEATURE_FLAGS:-}"
   create_args+=("--set spec.etcdClusters[*].etcdMembers[*].volumeIOPS=10000")
@@ -188,8 +190,6 @@ fi
 # this is used as a label to select kube-proxy pods on kops for kube-proxy service
 # used by CL2 Prometheus here https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/pkg/prometheus/manifests/default/kube-proxy-service.yaml#L2
 export PROMETHEUS_KUBE_PROXY_SELECTOR_KEY="k8s-app"
-export PROMETHEUS_SCRAPE_APISERVER_ONLY="true"
-export CL2_PROMETHEUS_TOLERATE_MASTER="true"
 export ETCD_PORT="4001" # we want cl2 to use this port for etcd instead of 2379
 if [[ "${CLOUD_PROVIDER}" == "aws" && "${SCALE_SCENARIO}" == "performance" ]]; then
   # CL2 uses KUBE_SSH_KEY_PATH path to ssh to instances for scraping metrics