Merge pull request #18149 from hakman/azure-terraform

azure: Add experimental Terraform support

Author: k8s-ci-robot

cmd/kops/integration_test.go

@@ -323,6 +323,13 @@ func TestMinimal_v1_34(t *testing.T) {
 		runTestTerraformAWS(t)
 }
 
+// TestMinimalAzure runs the test on a minimum Azure configuration.
+func TestMinimalAzure(t *testing.T) {
+	newIntegrationTest("minimal-azure.example.com", "minimal_azure").
+		withVersion("v1alpha3").
+		runTestTerraformAzure(t)
+}
+
 // TestMinimal_NoneDNS runs the test on a minimum configuration with --dns=none
 func TestMinimal_NoneDNS(t *testing.T) {
 	newIntegrationTest("minimal.example.com", "minimal-dns-none").
@@ -1736,6 +1743,111 @@ func (i *integrationTest) runTestTerraformGCE(t *testing.T) {
 	i.runTest(t, ctx, h, expectedFilenames, "", "", nil)
 }
 
+func (i *integrationTest) runTestTerraformAzure(t *testing.T) {
+	t.Setenv("AZURE_STORAGE_ACCOUNT", "teststorage")
+	t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")
+
+	featureflag.ParseFlags("+Azure,+AzureTerraform")
+	defer featureflag.ParseFlags("-Azure,-AzureTerraform")
+
+	ctx := testcontext.ForTest(t)
+	h := testutils.NewIntegrationTestHarness(t)
+	defer h.Close()
+
+	h.MockKopsVersion("1.34.0-beta.1")
+	h.SetupMockAzure()
+
+	var stdout bytes.Buffer
+
+	i.srcDir = updateClusterTestBase + i.srcDir
+	inputYAML := "in-" + i.version + ".yaml"
+
+	factory := i.setupCluster(t, ctx, inputYAML, stdout)
+
+	options := &UpdateClusterOptions{}
+	options.InitDefaults()
+	options.Target = "terraform"
+	options.OutDir = path.Join(h.TempDir, "out")
+	options.RunTasksOptions.MaxTaskDuration = 30 * time.Second
+	options.CreateKubecfg = false
+	options.ClusterName = i.clusterName
+
+	updateClusterResults, err := RunUpdateCluster(ctx, factory, &stdout, options)
+	if err != nil {
+		t.Fatalf("error running update cluster %q: %v", i.clusterName, err)
+	}
+
+	for key, task := range updateClusterResults.TaskMap {
+		if _, err := json.Marshal(task); err != nil {
+			t.Errorf("unable to marshal task %q of type %T to json: %v", key, task, err)
+		}
+	}
+
+	files, err := os.ReadDir(path.Join(h.TempDir, "out"))
+	if err != nil {
+		t.Fatalf("failed to read dir: %v", err)
+	}
+
+	var fileNames []string
+	for _, f := range files {
+		fileNames = append(fileNames, f.Name())
+	}
+	sort.Strings(fileNames)
+	if actual, expected := strings.Join(fileNames, ","), "data,kubernetes.tf"; actual != expected {
+		t.Fatalf("unexpected files.  actual=%q, expected=%q, test=%q", actual, expected, "kubernetes.tf")
+	}
+
+	actualTF, err := os.ReadFile(path.Join(h.TempDir, "out", "kubernetes.tf"))
+	if err != nil {
+		t.Fatalf("unexpected error reading actual terraform output: %v", err)
+	}
+	golden.AssertMatchesFile(t, string(actualTF), path.Join(i.srcDir, "kubernetes.tf"))
+
+	actualDataDir := filepath.Join(h.TempDir, "out", "data")
+	actualDataFiles, err := os.ReadDir(actualDataDir)
+	if err != nil {
+		t.Fatalf("failed to read data dir %q: %v", actualDataDir, err)
+	}
+
+	var actualDataFilenames []string
+	for _, f := range actualDataFiles {
+		actualDataFilenames = append(actualDataFilenames, f.Name())
+	}
+	sort.Strings(actualDataFilenames)
+
+	expectedDataDir := filepath.Join(i.srcDir, "data")
+	expectedDataFilenames := actualDataFilenames
+	if !golden.UpdateExpectedOutput() {
+		expectedDataFiles, err := os.ReadDir(expectedDataDir)
+		if err != nil {
+			t.Fatalf("failed to read data dir %q: %v", expectedDataDir, err)
+		}
+		expectedDataFilenames = make([]string, 0, len(expectedDataFiles))
+		for _, f := range expectedDataFiles {
+			expectedDataFilenames = append(expectedDataFilenames, f.Name())
+		}
+		sort.Strings(expectedDataFilenames)
+	}
+
+	for _, filename := range expectedDataFilenames {
+		expectedPath := filepath.Join(expectedDataDir, filename)
+		actualPath := filepath.Join(actualDataDir, filename)
+		actualDataContent, err := os.ReadFile(actualPath)
+		if err != nil {
+			t.Errorf("failed to read actual data file %q: %v", actualPath, err)
+			continue
+		}
+		golden.AssertMatchesFile(t, string(actualDataContent), expectedPath)
+	}
+
+	if !reflect.DeepEqual(actualDataFilenames, expectedDataFilenames) {
+		actual := strings.Join(actualDataFilenames, "\n")
+		expected := strings.Join(expectedDataFilenames, "\n")
+		t.Log(diff.FormatDiff(actual, expected))
+		t.Error("unexpected data files.")
+	}
+}
+
 func (i *integrationTest) runTestTerraformHetzner(t *testing.T) {
 	t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")
 

docs/getting_started/azure.md

@@ -6,6 +6,7 @@
 
 * Create, update and delete clusters
 * Create, edit and delete instance groups
+* Generate Terraform configuration with `--target=terraform` (experimental, requires `AzureTerraform` feature flag)
 * ...
 
 ## Requirements
@@ -83,7 +84,6 @@ kOps for Azure currently does not support the following features:
 * Azure Disk volumes
 * Azure Load Balancer
 * Autoscaling (using Cluster Autoscaler or Karpenter)
-* Terraform support
 * Multi-master clusters
 * ...
 

pkg/featureflag/featureflag.go

@@ -80,6 +80,8 @@ var (
 	ClusterAddons = new("ClusterAddons", Bool(false))
 	// Azure toggles the Azure support.
 	Azure = new("Azure", Bool(false))
+	// AzureTerraform toggles the Azure terraform support.
+	AzureTerraform = new("AzureTerraform", Bool(false))
 	// APIServerNodes enables ability to provision nodes that only run the kube-apiserver.
 	APIServerNodes = new("APIServerNodes", Bool(false))
 	// UseAddonOperators activates experimental addon operator support

pkg/model/azuremodel/network.go

@@ -285,6 +285,15 @@ func (b *NetworkModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 	}
 	c.AddTask(ngwTask)
 
+	rtTask := &azuretasks.RouteTable{
+		Name:          fi.PtrTo(b.NameForRouteTable()),
+		Lifecycle:     b.Lifecycle,
+		ResourceGroup: b.LinkToResourceGroup(),
+		Tags:          map[string]*string{},
+		Shared:        fi.PtrTo(b.Cluster.IsSharedAzureRouteTable()),
+	}
+	c.AddTask(rtTask)
+
 	for _, subnetSpec := range b.Cluster.Spec.Networking.Subnets {
 		subnetTask := &azuretasks.Subnet{
 			Name:                 fi.PtrTo(subnetSpec.Name),
@@ -293,21 +302,13 @@ func (b *NetworkModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 			VirtualNetwork:       b.LinkToVirtualNetwork(),
 			NatGateway:           ngwTask,
 			NetworkSecurityGroup: nsgTask,
+			RouteTable:           rtTask,
 			CIDR:                 fi.PtrTo(subnetSpec.CIDR),
 			Shared:               fi.PtrTo(b.Cluster.SharedVPC()),
 		}
 		c.AddTask(subnetTask)
 	}
 
-	rtTask := &azuretasks.RouteTable{
-		Name:          fi.PtrTo(b.NameForRouteTable()),
-		Lifecycle:     b.Lifecycle,
-		ResourceGroup: b.LinkToResourceGroup(),
-		Tags:          map[string]*string{},
-		Shared:        fi.PtrTo(b.Cluster.IsSharedAzureRouteTable()),
-	}
-	c.AddTask(rtTask)
-
 	return nil
 }
 

pkg/testutils/integrationtestharness.go

@@ -59,6 +59,7 @@ import (
 	"k8s.io/kops/pkg/pki"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+	"k8s.io/kops/upup/pkg/fi/cloudup/azuretasks"
 	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
 	"k8s.io/kops/util/pkg/vfs"
 )
@@ -133,6 +134,10 @@ func (h *IntegrationTestHarness) Close() {
 	}
 }
 
+func (h *IntegrationTestHarness) SetupMockAzure() *azuretasks.MockAzureCloud {
+	return azuretasks.InstallMockAzureCloud("eastus", "sub-123", "minimal-azure.example.com")
+}
+
 func (h *IntegrationTestHarness) SetupMockAWS() *awsup.MockAWSCloud {
 	ctx := context.TODO()
 	cloud := awsup.InstallMockAWSCloud("us-test-1", "abc")

tests/integration/update_cluster/minimal_azure/data/azurerm_linux_virtual_machine_scale_set_control-plane-eastus-1.masters.minimal-azure.example.com_user_data

@@ -0,0 +1,134 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.34.0-beta.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.34.0-beta.1/nodeup-linux-amd64
+NODEUP_HASH_AMD64=c86e072f622b91546b7b3f3cb1a0f8a131e48b966ad018a0ac1520ceedf37725
+NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.34.0-beta.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.34.0-beta.1/nodeup-linux-arm64
+NODEUP_HASH_ARM64=64a9a9510538a449e85d05e13e3cd98b80377d68a673447c26821d40f00f0075
+
+export AZURE_STORAGE_ACCOUNT=teststorage
+
+
+
+
+sysctl -w net.core.rmem_max=16777216 || true
+sysctl -w net.core.wmem_max=16777216 || true
+sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true
+sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true
+
+
+function ensure-install-dir() {
+  INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
+  mkdir -p ${INSTALL_DIR}/bin
+  mkdir -p ${INSTALL_DIR}/conf
+  cd ${INSTALL_DIR}
+}
+
+# Retry a download until we get it. args: name, sha, urls
+download-or-bust() {
+  echo "== Downloading $1 with hash $2 from $3 =="
+  local -r file="$1"
+  local -r hash="$2"
+  local -a urls
+  IFS=, read -r -a urls <<< "$3"
+
+  if [[ -f "${file}" ]]; then
+    if ! validate-hash "${file}" "${hash}"; then
+      rm -f "${file}"
+    else
+      return 0
+    fi
+  fi
+
+  while true; do
+    for url in "${urls[@]}"; do
+      commands=(
+        "curl -f --compressed -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --compression=auto -O ${file} --connect-timeout=20 --tries=6 --wait=10"
+        "curl -f -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget -O ${file} --connect-timeout=20 --tries=6 --wait=10"
+      )
+      for cmd in "${commands[@]}"; do
+        echo "== Downloading ${url} using ${cmd} =="
+        if ! (${cmd} "${url}"); then
+          echo "== Failed to download ${url} using ${cmd} =="
+          continue
+        fi
+        if ! validate-hash "${file}" "${hash}"; then
+          echo "== Failed to validate hash for ${url} =="
+          rm -f "${file}"
+        else
+          echo "== Downloaded ${url} with hash ${hash} =="
+          return 0
+        fi
+      done
+    done
+
+    echo "== All downloads failed; sleeping before retrying =="
+    sleep 60
+  done
+}
+
+validate-hash() {
+  local -r file="$1"
+  local -r expected="$2"
+  local actual
+
+  actual=$(sha256sum "${file}" | awk '{ print $1 }') || true
+  if [[ "${actual}" != "${expected}" ]]; then
+    echo "== File ${file} is corrupted; hash ${actual} doesn't match expected ${expected} =="
+    return 1
+  fi
+}
+
+function download-release() {
+  case "$(uname -m)" in
+  x86_64*|i?86_64*|amd64*)
+    NODEUP_URL="${NODEUP_URL_AMD64}"
+    NODEUP_HASH="${NODEUP_HASH_AMD64}"
+    ;;
+  aarch64*|arm64*)
+    NODEUP_URL="${NODEUP_URL_ARM64}"
+    NODEUP_HASH="${NODEUP_HASH_ARM64}"
+    ;;
+  *)
+    echo "Unsupported host arch: $(uname -m)" >&2
+    exit 1
+    ;;
+  esac
+
+  cd ${INSTALL_DIR}/bin
+  download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}"
+
+  chmod +x nodeup
+
+  echo "== Running nodeup =="
+  # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+  ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
+}
+
+####################################################################################
+
+/bin/systemd-machine-id-setup || echo "== Failed to initialize the machine ID; ensure machine-id configured =="
+
+echo "== nodeup node config starting =="
+ensure-install-dir
+
+cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
+CloudProvider: azure
+ClusterName: minimal-azure.example.com
+ConfigBase: memfs://tests/minimal-azure.example.com
+InstanceGroupName: control-plane-eastus-1
+InstanceGroupRole: ControlPlane
+NodeupConfigHash: gcFFJQgFVVQBfA4OkUuHIEgG9bNZ02Sp+LIMsgJ/UEY=
+
+__EOF_KUBE_ENV
+
+download-release
+echo "== nodeup node config done =="