Merge pull request #3802 from matix522/master

feat: Add an ability to measure clusters with dns only control plane …

Author: k8s-ci-robot

clusterloader2/cmd/clusterloader.go

@@ -91,6 +91,7 @@ func initClusterFlags() {
 	// TODO(#595): Change the name of the MASTER_IP and MASTER_INTERNAL_IP flags and vars to plural
 	flags.StringSliceEnvVar(&clusterLoaderConfig.ClusterConfig.MasterIPs, "masterip", "MASTER_IP", nil /*defaultValue*/, "Hostname/IP of the master node, supports multiple values when separated by commas")
 	flags.StringSliceEnvVar(&clusterLoaderConfig.ClusterConfig.MasterInternalIPs, "master-internal-ip", "MASTER_INTERNAL_IP", nil /*defaultValue*/, "Cluster internal/private IP of the master vm, supports multiple values when separated by commas")
+	flags.StringEnvVar(&clusterLoaderConfig.ClusterConfig.MasterDNSEndpoint, "master-endpoint", "MASTER_DNS_ENDPOINT", "", "Endpoint of the master node, exclusive with --masterip and --master-internal-ips")
 	flags.BoolEnvVar(&clusterLoaderConfig.ClusterConfig.APIServerPprofByClientEnabled, "apiserver-pprof-by-client-enabled", "APISERVER_PPROF_BY_CLIENT_ENABLED", true, "Whether apiserver pprof endpoint can be accessed by Kubernetes client.")
 	flags.BoolVar(&clusterLoaderConfig.ClusterConfig.SkipClusterVerification, "skip-cluster-verification", false, "Whether to skip the cluster verification, which expects at least one schedulable node in the cluster")
 
@@ -167,6 +168,23 @@ func completeConfig(m *framework.MultiClientSet) error {
 		clusterLoaderConfig.ClusterConfig.Nodes = nodes
 		klog.V(0).Infof("ClusterConfig.Nodes set to %v", nodes)
 	}
+	if clusterLoaderConfig.ClusterConfig.MasterDNSEndpoint == "" {
+		err := completeIpMasterConfig(m)
+		if err != nil {
+			return err
+		}
+	}
+
+	if !clusterLoaderConfig.ClusterConfig.Provider.Features().SupportAccessAPIServerPprofEndpoint {
+		clusterLoaderConfig.ClusterConfig.APIServerPprofByClientEnabled = false
+	}
+	if clusterLoaderConfig.ClusterConfig.K8SClientsNumber == 0 {
+		clusterLoaderConfig.ClusterConfig.K8SClientsNumber = getClientsNumber(clusterLoaderConfig.ClusterConfig.Nodes)
+	}
+	return nil
+}
+
+func completeIpMasterConfig(m *framework.MultiClientSet) error {
 	if clusterLoaderConfig.ClusterConfig.MasterName == "" {
 		masterName, err := util.GetMasterName(m.GetClient())
 		if err == nil {
@@ -194,13 +212,6 @@ func completeConfig(m *framework.MultiClientSet) error {
 			klog.Errorf("Getting master internal ip error: %v", err)
 		}
 	}
-
-	if !clusterLoaderConfig.ClusterConfig.Provider.Features().SupportAccessAPIServerPprofEndpoint {
-		clusterLoaderConfig.ClusterConfig.APIServerPprofByClientEnabled = false
-	}
-	if clusterLoaderConfig.ClusterConfig.K8SClientsNumber == 0 {
-		clusterLoaderConfig.ClusterConfig.K8SClientsNumber = getClientsNumber(clusterLoaderConfig.ClusterConfig.Nodes)
-	}
 	return nil
 }
 
@@ -288,6 +299,15 @@ func main() {
 		klog.Exitf("Parsing flags error: %v", errList.String())
 	}
 
+	klog.V(2).Infof("KubeConfigPath: %v", clusterLoaderConfig.ClusterConfig.KubeConfigPath)
+	if clusterLoaderConfig.ClusterConfig.KubeConfigPath != "" {
+		content, err := ioutil.ReadFile(clusterLoaderConfig.ClusterConfig.KubeConfigPath)
+		if err != nil {
+			klog.Errorf("Error reading kubeconfig: %v", err)
+		} else {
+			klog.V(2).Infof("KubeConfig content:\n%s", string(content))
+		}
+	}
 	mclient, err := framework.NewMultiClientSet(clusterLoaderConfig.ClusterConfig.KubeConfigPath, 1)
 	if err != nil {
 		klog.Exitf("Client creation error: %v", err)

clusterloader2/pkg/config/cluster.go

@@ -46,6 +46,7 @@ type ClusterConfig struct {
 	MasterIPs           []string
 	MasterInternalIPs   []string
 	MasterName          string
+	MasterDNSEndpoint   string
 	// Deprecated: use NamespaceConfig.DeleteStaleNamespaces instead.
 	DeleteStaleNamespaces bool
 	// TODO(#1696): Clean up after removing automanagedNamespaces

clusterloader2/pkg/measurement/common/profile.go

@@ -67,6 +67,9 @@ func (p *profileMeasurement) populateProfileConfig(config *measurement.Config) e
 	}
 	p.config.provider = config.ClusterFramework.GetClusterConfig().Provider
 	p.config.hosts = config.ClusterFramework.GetClusterConfig().MasterIPs
+	if masterDnsEnpoint := config.ClusterFramework.GetClusterConfig().MasterDNSEndpoint; masterDnsEnpoint != "" {
+		p.config.hosts = append(p.config.hosts, masterDnsEnpoint)
+	}
 	return nil
 }
 

clusterloader2/pkg/prometheus/manifests/master-ip/master-service.yaml

@@ -1,5 +1,6 @@
 {{$PROMETHEUS_SCRAPE_NODE_EXPORTER := DefaultParam .PROMETHEUS_SCRAPE_NODE_EXPORTER false}}
 {{$PROMETHEUS_SCRAPE_APISERVER_ONLY := DefaultParam .PROMETHEUS_SCRAPE_APISERVER_ONLY false}}
+{{$PROMETHEUS_MASTER_DNS_ENDPOINT := DefaultParam .PROMETHEUS_MASTER_DNS_ENDPOINT ""}}
 
 # Service object for the kubelet running on master node.
 apiVersion: v1
@@ -10,8 +11,13 @@ metadata:
   labels:
     k8s-app: master
 spec:
+  {{if eq $PROMETHEUS_MASTER_DNS_ENDPOINT ""}}
   type: ClusterIP
   clusterIP: None
+  {{else}}
+  type: ExternalName
+  externalName: {{ $PROMETHEUS_MASTER_DNS_ENDPOINT }}
+  {{end}}
   ports:
     - name: apiserver
       port: 443

clusterloader2/pkg/prometheus/manifests/master-ip/master-serviceMonitor.yaml

@@ -75,4 +75,5 @@ spec:
       name: prometheus-token
       key: token
   selector:
-    k8s-app: master
+    matchLabels:
+      k8s-app: master

clusterloader2/pkg/prometheus/prometheus.go

@@ -156,10 +156,14 @@ func NewController(clusterLoaderConfig *config.ClusterLoaderConfig) (pc *Control
 	if errList != nil {
 		return nil, errList
 	}
-	mapping["MasterIps"], err = getMasterIps(clusterLoaderConfig.ClusterConfig, clusterLoaderConfig.PrometheusConfig.ScrapeMastersWithPublicIPs)
-	if err != nil {
-		klog.Warningf("Couldn't get master ip, will ignore manifests requiring it: %v", err)
-		delete(mapping, "MasterIps")
+	if clusterLoaderConfig.ClusterConfig.MasterDNSEndpoint != "" {
+		klog.Infof("Cluster uses DNS master enpoint will not provide MasterIps")
+	} else {
+		mapping["MasterIps"], err = getMasterIps(clusterLoaderConfig.ClusterConfig, clusterLoaderConfig.PrometheusConfig.ScrapeMastersWithPublicIPs)
+		if err != nil {
+			klog.Warningf("Couldn't get master ip, will ignore manifests requiring it: %v", err)
+			delete(mapping, "MasterIps")
+		}
 	}
 	if _, exists := mapping["PROMETHEUS_SCRAPE_APISERVER_ONLY"]; !exists {
 		mapping["PROMETHEUS_SCRAPE_APISERVER_ONLY"] = clusterLoaderConfig.ClusterConfig.Provider.Features().ShouldPrometheusScrapeApiserverOnly
@@ -200,6 +204,9 @@ func NewController(clusterLoaderConfig *config.ClusterLoaderConfig) (pc *Control
 	} else {
 		clusterLoaderConfig.PrometheusConfig.ScrapeNetworkPolicies = mapping["PROMETHEUS_SCRAPE_KUBE_NETWORK_POLICIES"].(bool)
 	}
+	if _, exists := mapping["PROMETHEUS_MASTER_DNS_ENDPOINT"]; !exists {
+		mapping["PROMETHEUS_MASTER_DNS_ENDPOINT"] = clusterLoaderConfig.ClusterConfig.MasterDNSEndpoint
+	}
 	mapping["PROMETHEUS_SCRAPE_NODE_LOCAL_DNS"] = clusterLoaderConfig.PrometheusConfig.ScrapeNodeLocalDNS
 	mapping["PROMETHEUS_SCRAPE_KUBE_STATE_METRICS"] = clusterLoaderConfig.PrometheusConfig.ScrapeKubeStateMetrics
 	mapping["PROMETHEUS_SCRAPE_METRICS_SERVER_METRICS"] = clusterLoaderConfig.PrometheusConfig.ScrapeMetricsServerMetrics
@@ -639,6 +646,9 @@ func dumpAdditionalLogsOnPrometheusSetupFailure(k8sClient kubernetes.Interface)
 }
 
 func getMasterIps(clusterConfig config.ClusterConfig, usePublicIPs bool) ([]string, error) {
+	if clusterConfig.MasterDNSEndpoint != "" {
+		return nil, fmt.Errorf("cluster has master DNS endpoint %s, don't use master IPs", clusterConfig.MasterDNSEndpoint)
+	}
 	if usePublicIPs {
 		if len(clusterConfig.MasterIPs) == 0 {
 			return nil, fmt.Errorf("requested to use public IPs, however no publics IPs are provided")

clusterloader2/pkg/util/cluster.go

@@ -72,17 +72,17 @@ func LogClusterNodes(c clientset.Interface) error {
 	}
 	klog.V(2).Infof("Listing cluster nodes:")
 	for i := range nodeList {
-		var internalIP, externalIP string
+		var internalIPs, externalIPs []string
 		isSchedulable := IsNodeSchedulableAndUntainted(&nodeList[i])
 		for _, address := range nodeList[i].Status.Addresses {
 			if address.Type == corev1.NodeInternalIP {
-				internalIP = address.Address
+				internalIPs = append(internalIPs, address.Address)
 			}
 			if address.Type == corev1.NodeExternalIP {
-				externalIP = address.Address
+				externalIPs = append(externalIPs, address.Address)
 			}
 		}
-		klog.V(2).Infof("Name: %v, internalIP: %v, externalIP: %v, isSchedulable: %v", nodeList[i].ObjectMeta.Name, internalIP, externalIP, isSchedulable)
+		klog.V(2).Infof("Name: %v, internalIPs: %v, externalIPs: %v, isSchedulable: %v", nodeList[i].ObjectMeta.Name, internalIPs, externalIPs, isSchedulable)
 	}
 	return nil
 }

clusterloader2/run-e2e.sh

@@ -82,11 +82,11 @@ fi
 # Create a dedicated service account for cluster-loader.
 cluster_loader_sa_exists=$(kubectl --kubeconfig "${KUBECONFIG}" get serviceaccount cluster-loader --ignore-not-found | wc -l)
 if [[ "$cluster_loader_sa_exists" -eq 0 ]]; then
-	kubectl --kubeconfig "${KUBECONFIG}" create serviceaccount cluster-loader
+   kubectl --kubeconfig "${KUBECONFIG}" create serviceaccount cluster-loader
 fi
 cluster_loader_crb_exists=$(kubectl --kubeconfig "${KUBECONFIG}" get clusterrolebinding cluster-loader --ignore-not-found | wc -l)
 if [[ "$cluster_loader_crb_exists" -eq 0 ]]; then
-	kubectl --kubeconfig "${KUBECONFIG}" create clusterrolebinding cluster-loader --clusterrole=cluster-admin --serviceaccount=default:cluster-loader
+   kubectl --kubeconfig "${KUBECONFIG}" create clusterrolebinding cluster-loader --clusterrole=cluster-admin --serviceaccount=default:cluster-loader
 fi
 
 
@@ -95,13 +95,18 @@ kubeconfig=$(mktemp)
 server=$(kubectl --kubeconfig "${KUBECONFIG}" config view -o jsonpath='{.clusters[0].cluster.server}')
 ca=$(kubectl --kubeconfig "${KUBECONFIG}" get configmap kube-root-ca.crt -o jsonpath='{.data.ca\.crt}' | base64 -w 0)
 token=$(kubectl --kubeconfig "${KUBECONFIG}" --duration=8760h create token cluster-loader)
+ca_data=""
+
+if [[ "${MASTER_DNS_ENDPOINT:-}" == "" ]]; then
+  ca_data="    certificate-authority-data: ${ca}"
+fi
 echo "
 apiVersion: v1
 kind: Config
 clusters:
 - name: default-cluster
   cluster:
-    certificate-authority-data: ${ca}
+${ca_data}
     server: ${server}
 contexts:
 - name: default-context