CU13 preparedness

DaniBunny · DaniBunny · commit 2a523877cefb · 2021-09-27T13:52:34.000-07:00
diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/README.md
@@ -1,4 +1,4 @@
-# Create a Kubernetes cluster using Kubeadm on Ubuntu 16.04 LTS or 18.04 LTS
+# Create a Kubernetes cluster using Kubeadm on Ubuntu 20.04 LTS
 
 
 ## __[ubuntu](ubuntu/)__
diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc-ad.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc-ad.sh
@@ -49,8 +49,8 @@ export DEBIAN_FRONTEND=noninteractive
 
 # Kube version.
 #
-KUBE_DPKG_VERSION=1.16.3-00
-KUBE_VERSION=1.16.3
+KUBE_DPKG_VERSION=1.20.7-00
+KUBE_VERSION=1.20.7
 
 # Wait for 5 minutes for the cluster to be ready.
 #
@@ -61,7 +61,7 @@ RETRY_INTERVAL=5
 #
 export DOCKER_REGISTRY="mcr.microsoft.com"
 export DOCKER_REPOSITORY="mssql/bdc"
-export DOCKER_TAG="2019-CU8-ubuntu-16.04"
+export DOCKER_TAG="2019-CU13-ubuntu-20.04"
 
 # Variables used for azdata cluster creation.
 #
diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh
@@ -34,8 +34,8 @@ export DEBIAN_FRONTEND=noninteractive
 
 # Kube version.
 #
-KUBE_DPKG_VERSION=1.18.3-00
-KUBE_VERSION=1.18.3
+KUBE_DPKG_VERSION=1.20.7-00
+KUBE_VERSION=1.20.7
 
 # Wait for 5 minutes for the cluster to be ready.
 #
@@ -46,7 +46,7 @@ RETRY_INTERVAL=5
 #
 export DOCKER_REGISTRY="mcr.microsoft.com"
 export DOCKER_REPOSITORY="mssql/bdc"
-export DOCKER_TAG="2019-CU8-ubuntu-16.04"
+export DOCKER_TAG="2019-CU13-ubuntu-20.04"
 
 # Variables used for azdata cluster creation.
 #
diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/README.md
@@ -1,6 +1,6 @@
-# Create a Kubernetes cluster using Kubeadm on Ubuntu 16.04 LTS or 18.04 LTS
+# Create a Kubernetes cluster using Kubeadm on Ubuntu 20.04 LTS
 
-In this example, we will deploy Kubernetes over multiple Linux machines (physical or virtualized) using kubeadm utility. These instructions have been tested primarily with Ubuntu 16.04 LTS & 18.04 LTS versions.
+In this example, we will deploy Kubernetes over multiple Linux machines (physical or virtualized) using kubeadm utility. These instructions have been tested primarily with Ubuntu20.04 LTS version.
 
 ## Pre-requisites
 
diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-master.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-master.sh
@@ -2,7 +2,7 @@
 
 # Initialize a kubernetes cluster on the current node.
 #
-KUBE_VERSION=1.16.2
+KUBE_VERSION=1.20.7
 sudo kubeadm init --pod-network-cidr=10.244.0.0/16 --kubernetes-version=$KUBE_VERSION
 mkdir -p $HOME/.kube
 sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-prereqs.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-prereqs.sh
@@ -10,7 +10,7 @@ cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
 deb http://apt.kubernetes.io/ kubernetes-xenial main
 EOF
 
-KUBE_DPKG_VERSION=1.16.2-00
+KUBE_DPKG_VERSION=1.20.7-00
 apt-get update
 apt-get install -y ebtables ethtool
 apt-get install -y docker.io
diff --git a/samples/features/sql-big-data-cluster/deployment/platform-ops/bdc-aks-ops/deploy-bdc-aks.sh b/samples/features/sql-big-data-cluster/deployment/platform-ops/bdc-aks-ops/deploy-bdc-aks.sh
@@ -19,7 +19,7 @@ done
 azdata bdc config init --source aks-dev-test --target bdc-aks --force
 
 #Configurations for BDC deployment
-azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.docker.imageTag=2019-CU12-ubuntu-20.04"
+azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.docker.imageTag=2019-CU13-ubuntu-20.04"
 azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.storage.data.className=default"
 azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.storage.logs.className=default"
 
diff --git a/samples/features/sql-big-data-cluster/deployment/platform-ops/deploy-bdc-aks.sh b/samples/features/sql-big-data-cluster/deployment/platform-ops/deploy-bdc-aks.sh
@@ -19,7 +19,7 @@ done
 azdata bdc config init --source aks-dev-test --target bdc-aks --force
 
 #Configurations for BDC deployment
-azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.docker.imageTag=2019-CU12-ubuntu-20.04"
+azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.docker.imageTag=2019-CU13-ubuntu-20.04"
 azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.storage.data.className=default"
 azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.storage.logs.className=default"
 
diff --git a/samples/features/sql-big-data-cluster/deployment/private-aks/scripts/deploy-bdc-private-aks.sh b/samples/features/sql-big-data-cluster/deployment/private-aks/scripts/deploy-bdc-private-aks.sh
@@ -18,7 +18,7 @@ done
 azdata bdc config init --source aks-dev-test --target private-bdc-aks --force
 
 #Configurations for BDC deployment
-azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.docker.imageTag=2019-CU12-ubuntu-20.04"
+azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.docker.imageTag=2019-CU13-ubuntu-20.04"
 azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.storage.data.className=default"
 azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.storage.logs.className=default"
 
diff --git a/samples/features/sql-big-data-cluster/spark/config-install/installpackage_Spark.ipynb b/samples/features/sql-big-data-cluster/spark/config-install/installpackage_Spark.ipynb
@@ -17,19 +17,21 @@
         {
             "cell_type": "markdown",
             "source": [
-                "<p align=\"center\">\n",
-                "<img src =\"https://raw.githubusercontent.com/microsoft/azuredatastudio/master/src/sql/media/microsoft_logo_gray.svg?sanitize=true\" width=\"250\" align=\"center\">\n",
-                "</p>\n",
-                "\n",
-                "# **Spark Package Management in SQL Server 2019 Big Data Clusters**\n",
-                "This guide covers installing packages and submitting jobs to a SQL Server 2019 Big Data Cluster using Spark.\n",
-                "* Built-In Tools\n",
-                "* Install Packages from a Maven Repository onto the Spark Cluster at Runtime\n",
-                "* Import .jar from HDFS for use at runtime\n",
-                "* Import .jar at runtime through Azure Data Studio notebook cell configuration\n",
-                "* Install Python Packages at Runtime for use with PySpark \n",
-                "* Submit local .jar or python file\n",
-                "<!-- <span style=\"color:red\"><font size=\"3\">Please press the \"Run Cells\" button to run the notebook</font></span> -->"
+                "<p align=\"center\">\r\n",
+                "<img src =\"https://raw.githubusercontent.com/microsoft/azuredatastudio/master/src/sql/media/microsoft_logo_gray.svg?sanitize=true\" width=\"250\" align=\"center\">\r\n",
+                "</p>\r\n",
+                "\r\n",
+                "# **Spark Package Management in SQL Server 2019 Big Data Clusters**\r\n",
+                "This guide covers installing packages and submitting jobs to a SQL Server 2019 Big Data Cluster using Spark.\r\n",
+                "* Built-In Tools\r\n",
+                "* Install Packages from a Maven Repository onto the Spark Cluster at Runtime\r\n",
+                "* Import .jar from HDFS for use at runtime\r\n",
+                "* Import .jar at runtime through Azure Data Studio notebook cell configuration\r\n",
+                "* Install Python Packages at Runtime for use with PySpark \r\n",
+                "* Submit local .jar or python file\r\n",
+                "<!-- <span style=\"color:red\"><font size=\"3\">Please press the \"Run Cells\" button to run the notebook</font></span> -->\r\n",
+                "\r\n",
+                "For more information on package managament, refer to [Spark library management](https://docs.microsoft.com/sql/big-data-cluster/spark-install-packages?view=sql-server-ver15)"
             ],
             "metadata": {
                 "azdata_cell_guid": "cbc8ced8-8931-4302-b252-7e7e478a16d4"
@@ -38,13 +40,10 @@
         {
             "cell_type": "markdown",
             "source": [
-                "# Built-in Tools\n",
-                "* Spark and Hadoop base packages\n",
-                "* Python 3.5 and Python 2.7\n",
-                "* Pandas, Sklearn, Numpy, and other data processing packages.\n",
-                "* R and MRO packages\n",
-                "* Sparklyr\n",
-                ""
+                "# Built-in Tools\r\n",
+                "* Spark and Hadoop base packages\r\n",
+                "* Python 3.8 with PySpark and Pandas, Sklearn, Numpy, and other data processing libraries.\r\n",
+                "* R 3.5 with Spark.R, sparklyr and MRO packages\r\n"
             ],
             "metadata": {
                 "azdata_cell_guid": "2fc8a069-115e-4d9b-bedc-5c55f79466b1"
@@ -59,8 +58,7 @@
                 "```\r\n",
                 "%%configure -f \\\r\n",
                 "{\"conf\": {\"spark.jars.packages\": \"com.microsoft.azure:azure-eventhubs-spark_2.11:2.3.1\"}}\r\n",
-                "```\r\n",
-                ""
+                "```\r\n"
             ],
             "metadata": {
                 "azdata_cell_guid": "a0fecc05-f094-4dda-9afe-0de8ddad87eb"
@@ -76,8 +74,7 @@
                 "```\n",
                 "%%configure -f\n",
                 "{\"conf\": {\"spark.jars\": \"/jar/mycodeJar.jar\"}}\n",
-                "```\n",
-                ""
+                "```\n"
             ],
             "metadata": {
                 "azdata_cell_guid": "c5e65fa2-faf0-4e22-aac1-69d7ff8c9989"
@@ -91,8 +88,7 @@
                 "```\n",
                 "%%configure -f\n",
                 "{\"conf\": {\"spark.jars\": \"/jar/mycodeJar.jar\"}}\n",
-                "```\n",
-                ""
+                "```\n"
             ],
             "metadata": {
                 "azdata_cell_guid": "6fc4085f-e142-4355-b215-148dbf6c5b86"
@@ -140,12 +136,11 @@
                 "\r\n",
                 "* [Submit Spark jobs on SQL Server Big Data Clusters in Azure Data Studio](https://docs.microsoft.com/en-us/sql/big-data-cluster/spark-submit-job?view=sqlallproducts-allversions)\r\n",
                 "* [Submit Spark jobs on SQL Server Big Data Clusters in IntelliJ](https://docs.microsoft.com/en-us/sql/big-data-cluster/spark-submit-job-intellij-tool-plugin?view=sqlallproducts-allversions)\r\n",
-                "* [Submit Spark jobs on SQL Server big data cluster in Visual Studio Code](https://docs.microsoft.com/en-us/sql/big-data-cluster/spark-hive-tools-vscode?view=sqlallproducts-allversions)\r\n",
-                ""
+                "* [Submit Spark jobs on SQL Server big data cluster in Visual Studio Code](https://docs.microsoft.com/en-us/sql/big-data-cluster/spark-hive-tools-vscode?view=sqlallproducts-allversions)\r\n"
             ],
             "metadata": {
                 "azdata_cell_guid": "7d1b55c0-1961-45f7-8449-a24a913106e4"
             }
         }
     ]
-}
+}

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-# Create a Kubernetes cluster using Kubeadm on Ubuntu 16.04 LTS or 18.04 LTS`
	`1`	`+# Create a Kubernetes cluster using Kubeadm on Ubuntu 20.04 LTS`
`2`	`2`
`3`	`3`
`4`	`4`	`## __[ubuntu](ubuntu/)__`
Original file line number	Diff line number	Diff line change
`@@ -49,8 +49,8 @@ export DEBIAN_FRONTEND=noninteractive`
`49`	`49`
`50`	`50`	`# Kube version.`
`51`	`51`	`#`
`52`		`-KUBE_DPKG_VERSION=1.16.3-00`
`53`		`-KUBE_VERSION=1.16.3`
	`52`	`+KUBE_DPKG_VERSION=1.20.7-00`
	`53`	`+KUBE_VERSION=1.20.7`
`54`	`54`
`55`	`55`	`# Wait for 5 minutes for the cluster to be ready.`
`56`	`56`	`#`
`@@ -61,7 +61,7 @@ RETRY_INTERVAL=5`
`61`	`61`	`#`
`62`	`62`	`export DOCKER_REGISTRY="mcr.microsoft.com"`
`63`	`63`	`export DOCKER_REPOSITORY="mssql/bdc"`
`64`		`-export DOCKER_TAG="2019-CU8-ubuntu-16.04"`
	`64`	`+export DOCKER_TAG="2019-CU13-ubuntu-20.04"`
`65`	`65`
`66`	`66`	`# Variables used for azdata cluster creation.`
`67`	`67`	`#`
Original file line number	Diff line number	Diff line change
`@@ -34,8 +34,8 @@ export DEBIAN_FRONTEND=noninteractive`
`34`	`34`
`35`	`35`	`# Kube version.`
`36`	`36`	`#`
`37`		`-KUBE_DPKG_VERSION=1.18.3-00`
`38`		`-KUBE_VERSION=1.18.3`
	`37`	`+KUBE_DPKG_VERSION=1.20.7-00`
	`38`	`+KUBE_VERSION=1.20.7`
`39`	`39`
`40`	`40`	`# Wait for 5 minutes for the cluster to be ready.`
`41`	`41`	`#`
`@@ -46,7 +46,7 @@ RETRY_INTERVAL=5`
`46`	`46`	`#`
`47`	`47`	`export DOCKER_REGISTRY="mcr.microsoft.com"`
`48`	`48`	`export DOCKER_REPOSITORY="mssql/bdc"`
`49`		`-export DOCKER_TAG="2019-CU8-ubuntu-16.04"`
	`49`	`+export DOCKER_TAG="2019-CU13-ubuntu-20.04"`
`50`	`50`
`51`	`51`	`# Variables used for azdata cluster creation.`
`52`	`52`	`#`
Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@`
`2`	`2`
`3`	`3`	`# Initialize a kubernetes cluster on the current node.`
`4`	`4`	`#`
`5`		`-KUBE_VERSION=1.16.2`
	`5`	`+KUBE_VERSION=1.20.7`
`6`	`6`	`sudo kubeadm init --pod-network-cidr=10.244.0.0/16 --kubernetes-version=$KUBE_VERSION`
`7`	`7`	`mkdir -p $HOME/.kube`
`8`	`8`	`sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config`