Skip to content

Commit dd9aa2b

Browse files
matix522matix522
committed
feat: Add an ability to measure clusters with dns only control plane access
This update allows ClusterLoader2 to target clusters using a DNS endpoint for the control plane. It includes support for the MASTER_DNS_ENDPOINT environment variable and relevant configuration changes. Changed logging to show all of the Node adresses to allow for dualstack clusters.
1 parent f624f5b commit dd9aa2b

File tree

8 files changed

+65
-19
lines changed

8 files changed

+65
-19
lines changed

‎clusterloader2/cmd/clusterloader.go‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ func initClusterFlags() {
9191
// TODO(#595): Change the name of the MASTER_IP and MASTER_INTERNAL_IP flags and vars to plural
9292
flags.StringSliceEnvVar(&clusterLoaderConfig.ClusterConfig.MasterIPs, "masterip", "MASTER_IP", nil /*defaultValue*/, "Hostname/IP of the master node, supports multiple values when separated by commas")
9393
flags.StringSliceEnvVar(&clusterLoaderConfig.ClusterConfig.MasterInternalIPs, "master-internal-ip", "MASTER_INTERNAL_IP", nil /*defaultValue*/, "Cluster internal/private IP of the master vm, supports multiple values when separated by commas")
94+
flags.StringEnvVar(&clusterLoaderConfig.ClusterConfig.MasterDNSEndpoint, "master-endpoint", "MASTER_DNS_ENDPOINT", "", "Endpoint of the master node, exclusive with --masterip and --master-internal-ips")
9495
flags.BoolEnvVar(&clusterLoaderConfig.ClusterConfig.APIServerPprofByClientEnabled, "apiserver-pprof-by-client-enabled", "APISERVER_PPROF_BY_CLIENT_ENABLED", true, "Whether apiserver pprof endpoint can be accessed by Kubernetes client.")
9596
flags.BoolVar(&clusterLoaderConfig.ClusterConfig.SkipClusterVerification, "skip-cluster-verification", false, "Whether to skip the cluster verification, which expects at least one schedulable node in the cluster")
9697

@@ -167,6 +168,23 @@ func completeConfig(m *framework.MultiClientSet) error {
167168
clusterLoaderConfig.ClusterConfig.Nodes = nodes
168169
klog.V(0).Infof("ClusterConfig.Nodes set to %v", nodes)
169170
}
171+
if clusterLoaderConfig.ClusterConfig.MasterDNSEndpoint == "" {
172+
err := completeIpMasterConfig(m)
173+
if err != nil {
174+
return err
175+
}
176+
}
177+
178+
if !clusterLoaderConfig.ClusterConfig.Provider.Features().SupportAccessAPIServerPprofEndpoint {
179+
clusterLoaderConfig.ClusterConfig.APIServerPprofByClientEnabled = false
180+
}
181+
if clusterLoaderConfig.ClusterConfig.K8SClientsNumber == 0 {
182+
clusterLoaderConfig.ClusterConfig.K8SClientsNumber = getClientsNumber(clusterLoaderConfig.ClusterConfig.Nodes)
183+
}
184+
return nil
185+
}
186+
187+
func completeIpMasterConfig(m *framework.MultiClientSet) error {
170188
if clusterLoaderConfig.ClusterConfig.MasterName == "" {
171189
masterName, err := util.GetMasterName(m.GetClient())
172190
if err == nil {
@@ -194,13 +212,6 @@ func completeConfig(m *framework.MultiClientSet) error {
194212
klog.Errorf("Getting master internal ip error: %v", err)
195213
}
196214
}
197-
198-
if !clusterLoaderConfig.ClusterConfig.Provider.Features().SupportAccessAPIServerPprofEndpoint {
199-
clusterLoaderConfig.ClusterConfig.APIServerPprofByClientEnabled = false
200-
}
201-
if clusterLoaderConfig.ClusterConfig.K8SClientsNumber == 0 {
202-
clusterLoaderConfig.ClusterConfig.K8SClientsNumber = getClientsNumber(clusterLoaderConfig.ClusterConfig.Nodes)
203-
}
204215
return nil
205216
}
206217

@@ -288,6 +299,15 @@ func main() {
288299
klog.Exitf("Parsing flags error: %v", errList.String())
289300
}
290301

302+
klog.V(2).Infof("KubeConfigPath: %v", clusterLoaderConfig.ClusterConfig.KubeConfigPath)
303+
if clusterLoaderConfig.ClusterConfig.KubeConfigPath != "" {
304+
content, err := ioutil.ReadFile(clusterLoaderConfig.ClusterConfig.KubeConfigPath)
305+
if err != nil {
306+
klog.Errorf("Error reading kubeconfig: %v", err)
307+
} else {
308+
klog.V(2).Infof("KubeConfig content:\n%s", string(content))
309+
}
310+
}
291311
mclient, err := framework.NewMultiClientSet(clusterLoaderConfig.ClusterConfig.KubeConfigPath, 1)
292312
if err != nil {
293313
klog.Exitf("Client creation error: %v", err)

‎clusterloader2/pkg/config/cluster.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ type ClusterConfig struct {
4646
MasterIPs []string
4747
MasterInternalIPs []string
4848
MasterName string
49+
MasterDNSEndpoint string
4950
// Deprecated: use NamespaceConfig.DeleteStaleNamespaces instead.
5051
DeleteStaleNamespaces bool
5152
// TODO(#1696): Clean up after removing automanagedNamespaces

‎clusterloader2/pkg/measurement/common/profile.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@ func (p *profileMeasurement) populateProfileConfig(config *measurement.Config) e
6767
}
6868
p.config.provider = config.ClusterFramework.GetClusterConfig().Provider
6969
p.config.hosts = config.ClusterFramework.GetClusterConfig().MasterIPs
70+
if masterDnsEnpoint := config.ClusterFramework.GetClusterConfig().MasterDNSEndpoint; masterDnsEnpoint != "" {
71+
p.config.hosts = append(p.config.hosts, masterDnsEnpoint)
72+
}
7073
return nil
7174
}
7275

‎clusterloader2/pkg/prometheus/manifests/master-ip/master-service.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{{$PROMETHEUS_SCRAPE_NODE_EXPORTER := DefaultParam .PROMETHEUS_SCRAPE_NODE_EXPORTER false}}
22
{{$PROMETHEUS_SCRAPE_APISERVER_ONLY := DefaultParam .PROMETHEUS_SCRAPE_APISERVER_ONLY false}}
3+
{{$PROMETHEUS_MASTER_DNS_ENDPOINT := DefaultParam .PROMETHEUS_MASTER_DNS_ENDPOINT ""}}
34

45
# Service object for the kubelet running on master node.
56
apiVersion: v1
@@ -10,8 +11,13 @@ metadata:
1011
labels:
1112
k8s-app: master
1213
spec:
14+
{{if eq $PROMETHEUS_MASTER_DNS_ENDPOINT ""}}
1315
type: ClusterIP
1416
clusterIP: None
17+
{{else}}
18+
type: ExternalName
19+
externalName: {{ $PROMETHEUS_MASTER_DNS_ENDPOINT }}
20+
{{end}}
1521
ports:
1622
- name: apiserver
1723
port: 443

‎clusterloader2/pkg/prometheus/manifests/master-ip/master-serviceMonitor.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,4 +75,5 @@ spec:
7575
name: prometheus-token
7676
key: token
7777
selector:
78-
k8s-app: master
78+
matchLabels:
79+
k8s-app: master

‎clusterloader2/pkg/prometheus/prometheus.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -156,10 +156,14 @@ func NewController(clusterLoaderConfig *config.ClusterLoaderConfig) (pc *Control
156156
if errList != nil {
157157
return nil, errList
158158
}
159-
mapping["MasterIps"], err = getMasterIps(clusterLoaderConfig.ClusterConfig, clusterLoaderConfig.PrometheusConfig.ScrapeMastersWithPublicIPs)
160-
if err != nil {
161-
klog.Warningf("Couldn't get master ip, will ignore manifests requiring it: %v", err)
162-
delete(mapping, "MasterIps")
159+
if clusterLoaderConfig.ClusterConfig.MasterDNSEndpoint != "" {
160+
klog.Infof("Cluster uses DNS master enpoint will not provide MasterIps")
161+
} else {
162+
mapping["MasterIps"], err = getMasterIps(clusterLoaderConfig.ClusterConfig, clusterLoaderConfig.PrometheusConfig.ScrapeMastersWithPublicIPs)
163+
if err != nil {
164+
klog.Warningf("Couldn't get master ip, will ignore manifests requiring it: %v", err)
165+
delete(mapping, "MasterIps")
166+
}
163167
}
164168
if _, exists := mapping["PROMETHEUS_SCRAPE_APISERVER_ONLY"]; !exists {
165169
mapping["PROMETHEUS_SCRAPE_APISERVER_ONLY"] = clusterLoaderConfig.ClusterConfig.Provider.Features().ShouldPrometheusScrapeApiserverOnly
@@ -200,6 +204,9 @@ func NewController(clusterLoaderConfig *config.ClusterLoaderConfig) (pc *Control
200204
} else {
201205
clusterLoaderConfig.PrometheusConfig.ScrapeNetworkPolicies = mapping["PROMETHEUS_SCRAPE_KUBE_NETWORK_POLICIES"].(bool)
202206
}
207+
if _, exists := mapping["PROMETHEUS_MASTER_DNS_ENDPOINT"]; !exists {
208+
mapping["PROMETHEUS_MASTER_DNS_ENDPOINT"] = clusterLoaderConfig.ClusterConfig.MasterDNSEndpoint
209+
}
203210
mapping["PROMETHEUS_SCRAPE_NODE_LOCAL_DNS"] = clusterLoaderConfig.PrometheusConfig.ScrapeNodeLocalDNS
204211
mapping["PROMETHEUS_SCRAPE_KUBE_STATE_METRICS"] = clusterLoaderConfig.PrometheusConfig.ScrapeKubeStateMetrics
205212
mapping["PROMETHEUS_SCRAPE_METRICS_SERVER_METRICS"] = clusterLoaderConfig.PrometheusConfig.ScrapeMetricsServerMetrics
@@ -639,6 +646,9 @@ func dumpAdditionalLogsOnPrometheusSetupFailure(k8sClient kubernetes.Interface)
639646
}
640647

641648
func getMasterIps(clusterConfig config.ClusterConfig, usePublicIPs bool) ([]string, error) {
649+
if clusterConfig.MasterDNSEndpoint != "" {
650+
return nil, fmt.Errorf("cluster has master DNS endpoint %s, don't use master IPs", clusterConfig.MasterDNSEndpoint)
651+
}
642652
if usePublicIPs {
643653
if len(clusterConfig.MasterIPs) == 0 {
644654
return nil, fmt.Errorf("requested to use public IPs, however no publics IPs are provided")

‎clusterloader2/pkg/util/cluster.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,17 +74,17 @@ func LogClusterNodes(c clientset.Interface) error {
7474
}
7575
klog.V(2).Infof("Listing cluster nodes:")
7676
for i := range nodeList {
77-
var internalIP, externalIP string
77+
var internalIPs, externalIPs []string
7878
isSchedulable := IsNodeSchedulableAndUntainted(&nodeList[i])
7979
for _, address := range nodeList[i].Status.Addresses {
8080
if address.Type == corev1.NodeInternalIP {
81-
internalIP = address.Address
81+
internalIPs = append(internalIPs, address.Address)
8282
}
8383
if address.Type == corev1.NodeExternalIP {
84-
externalIP = address.Address
84+
externalIPs = append(externalIPs, address.Address)
8585
}
8686
}
87-
klog.V(2).Infof("Name: %v, internalIP: %v, externalIP: %v, isSchedulable: %v", nodeList[i].ObjectMeta.Name, internalIP, externalIP, isSchedulable)
87+
klog.V(2).Infof("Name: %v, internalIPs: %v, externalIPs: %v, isSchedulable: %v", nodeList[i].ObjectMeta.Name, internalIPs, externalIPs, isSchedulable)
8888
}
8989
return nil
9090
}

‎clusterloader2/run-e2e.sh‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -82,11 +82,11 @@ fi
8282
# Create a dedicated service account for cluster-loader.
8383
cluster_loader_sa_exists=$(kubectl --kubeconfig "${KUBECONFIG}" get serviceaccount cluster-loader --ignore-not-found | wc -l)
8484
if [[ "$cluster_loader_sa_exists" -eq 0 ]]; then
85-
kubectl --kubeconfig "${KUBECONFIG}" create serviceaccount cluster-loader
85+
kubectl --kubeconfig "${KUBECONFIG}" create serviceaccount cluster-loader
8686
fi
8787
cluster_loader_crb_exists=$(kubectl --kubeconfig "${KUBECONFIG}" get clusterrolebinding cluster-loader --ignore-not-found | wc -l)
8888
if [[ "$cluster_loader_crb_exists" -eq 0 ]]; then
89-
kubectl --kubeconfig "${KUBECONFIG}" create clusterrolebinding cluster-loader --clusterrole=cluster-admin --serviceaccount=default:cluster-loader
89+
kubectl --kubeconfig "${KUBECONFIG}" create clusterrolebinding cluster-loader --clusterrole=cluster-admin --serviceaccount=default:cluster-loader
9090
fi
9191

9292

@@ -95,13 +95,18 @@ kubeconfig=$(mktemp)
9595
server=$(kubectl --kubeconfig "${KUBECONFIG}" config view -o jsonpath='{.clusters[0].cluster.server}')
9696
ca=$(kubectl --kubeconfig "${KUBECONFIG}" get configmap kube-root-ca.crt -o jsonpath='{.data.ca\.crt}' | base64 -w 0)
9797
token=$(kubectl --kubeconfig "${KUBECONFIG}" --duration=8760h create token cluster-loader)
98+
ca_data=""
99+
100+
if [[ "${MASTER_DNS_ENDPOINT:-}" == "" ]]; then
101+
ca_data=" certificate-authority-data: ${ca}"
102+
fi
98103
echo "
99104
apiVersion: v1
100105
kind: Config
101106
clusters:
102107
- name: default-cluster
103108
cluster:
104-
certificate-authority-data: ${ca}
109+
${ca_data}
105110
server: ${server}
106111
contexts:
107112
- name: default-context

0 commit comments

Comments
 (0)