CTP 3.2 updates

Author: uc-msft

samples/features/sql-big-data-cluster/data-virtualization/oracle/customer-oracle.sql

@@ -49,7 +49,7 @@ GO
 SELECT pr.pr_item_sk, pc.pr_review_content, pr.pr_user_sk AS customerid 
 FROM dbo.product_reviews as pr
 JOIN (SELECT TOP(100) * FROM dbo.product_reviews_hdfs_csv) AS pc ON pc.pr_review_sk = pr.pr_review_sk
-JOIN dbo.customer_ora AS c ON c.c_customer_sk = pr.pr_user_sk
+JOIN dbo.customer_ora AS c ON c.C_CUSTOMER_SK = pr.pr_user_sk
 JOIN dbo.item AS i ON i.i_item_sk = pr.pr_item_sk
 INNER JOIN (
     SELECT

samples/features/sql-big-data-cluster/data-virtualization/oracle/inventory-oracle.sql

@@ -20,22 +20,22 @@ IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'OracleSalesS
 --       As a result, the names are case-sensitive so specify the name in the external table definition
 --       that matches the exact case of the table and column names in the Oracle metadata.
 CREATE EXTERNAL TABLE [inventory_ora]
-    ([inv_date] DECIMAL(10,0) NOT NULL, [inv_item] DECIMAL(10,0) NOT NULL,
-    [inv_warehouse] DECIMAL(10,0) NOT NULL, [inv_quantity_on_hand] DECIMAL(10,0))
+    ([INV_DATE] DECIMAL(10,0) NOT NULL, [INV_ITEM] DECIMAL(10,0) NOT NULL,
+    [INV_WAREHOUSE] DECIMAL(10,0) NOT NULL, [INV_QUANTITY_ON_HAND] DECIMAL(10,0))
 WITH (DATA_SOURCE=[OracleSalesSrvr],
       LOCATION='<oracle_service_name,nvarchar(30),xe>.SALES.INVENTORY');
 GO
 
 -- Find quantity of certain items from inventory for a specific category
 --
-SELECT TOP(100) w.w_warehouse_name, i.inv_item, SUM(i.inv_quantity_on_hand) as total_quantity
+SELECT TOP(100) w.w_warehouse_name, i.INV_ITEM, SUM(i.INV_QUANTITY_ON_HAND) as total_quantity
   FROM [inventory_ora] as i
   JOIN item as it
-    ON it.i_item_sk = i.inv_item
+    ON it.i_item_sk = i.INV_ITEM
   JOIN warehouse as w
-    ON w.w_warehouse_sk = i.inv_warehouse
- WHERE it.i_category = 'Movies & TV' and i.inv_item BETWEEN 17401 and 17402 --> get items within specific range
- GROUP BY w.w_warehouse_name, i.inv_item;
+    ON w.w_warehouse_sk = i.INV_WAREHOUSE
+ WHERE it.i_category = 'Movies & TV' and i.INV_ITEM BETWEEN 17401 and 17402 --> get items within specific range
+ GROUP BY w.w_warehouse_name, i.INV_ITEM;
 GO
 
 -- Cleanup

samples/features/sql-big-data-cluster/data-virtualization/storage-pool/web-clickstreams-hdfs-parquet.sql

@@ -90,8 +90,8 @@ AS
 						 AND w.wcs_user_sk IS NOT NULL)
 	  GROUP BY w.wcs_user_sk
 	) AS q
-	INNER JOIN customer as c ON q.wcs_user_sk = c.c_customer_sk
-	INNER JOIN customer_demographics as cd ON c.c_current_cdemo_sk = cd.cd_demo_sk;
+	INNER JOIN customer_ora as c ON q.wcs_user_sk = c.C_CUSTOMER_SK
+	INNER JOIN customer_demographics as cd ON c.C_CURRENT_CDEMO_SK = cd.cd_demo_sk;
 GO
 
 

samples/features/sql-big-data-cluster/deployment/README.md

@@ -3,13 +3,13 @@
 
 SQL Server 2019 big data cluster is deployed as docker containers on a Kubernetes cluster. These samples provide scripts that can be used to provision a Kubernetes clusters using different environments.
 
-## Create a Kubernetes cluster using Kubeadm on Ubuntu 16.04 LTS or 18.04 LTS 
+## Deploy a Kubernetes cluster using kubeadm
 
-Use the scripts in the **kubeadm** folder to deploy Kubernetes over multiple Linux machines (physical or virtualized) using `kubeadm` utility. 
+Use the scripts in the **kubeadm** folder to deploy a Kubernetes cluster over one or more Linux machines (physical or virtualized) using `kubeadm` utility.
 
-## Deploy a SQL Server big data cluster on Azure Kubernetes Service (AKS) 
+## Deploy a SQL Server big data cluster on Azure Kubernetes Service (AKS)
 
-Using the sample Python script in **aks** folder, you will deploy a Kubernetes cluster in Azure using AKS and a SQL Server big data cluster using on top of it. 
+Using the sample Python script in **aks** folder, you will deploy a Kubernetes cluster in Azure using AKS and a SQL Server big data cluster using on top of it.
 
 ## Push SQL Server big data cluster images to your own private Docker repository
 

samples/features/sql-big-data-cluster/deployment/kubeadm/README.md

@@ -1,30 +1,10 @@
 # Create a Kubernetes cluster using Kubeadm on Ubuntu 16.04 LTS or 18.04 LTS
 
 
-In this example, we will deploy Kubernetes over multiple Linux machines (physical or virtualized) using kubeadm utility. These instructions have been tested primarily with Ubuntu 16.04 LTS & 18.04 LTS versions.
+## __[ubuntu](ubuntu/)__
 
-## Pre-requisites
+This folder contains scripts that provide a template for deploying a Kubernetes cluster using kubeadm on one or more Linux machines.
 
-1. Multiple Linux machines or virtual machines. Recommended configuration is 8 CPUs, 32 GB memory each and at least 100 GB storage for each machine. Minimum number of machines required is three machines
-1. Designate one machine as the Kubernetes master
-1. Rest of the machines will be used as the Kubernetes agents
+## __[ubuntu-single-node-vm](ubuntu-single-node-vm/)__
 
-**NOTE: Ensure there is sufficient local storage on your agents. Each volume will use up to 10GB by default. The script creates 25 volumes. Not all of the volumes will be used since it depends on the number of pods being deployed on each agent node. It is recommended to have at least 200 GB of storage on the agent nodes**
-
-### Useful resources
-
-[Deploy SQL Server 2019 big data cluster on Kubernetes](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions)
-
-[Creating a cluster using kubeadm](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/)
-
-[Troubleshooting kubeadm](https://kubernetes.io/docs/setup/independent/troubleshooting-kubeadm/)
-
-### Instructions
-
-1. Start a sudo shell context
-1. Execute [ubuntu/setup-k8s-prereqs.sh](ubuntu/setup-k8s-prereqs.sh/) script on each machine
-1. Execute [ubuntu/setup-k8s-master.sh](ubuntu/setup-k8s-master.sh/) script on the machine designated as Kubernetes master
-1. After successful initialization of the Kubernetes master, follow the kubeadm join commands output by the setup script on each agent machine
-1. Execute [ubuntu/setup-volumes-agent.sh](ubuntu/setup-volumes-agent.sh/) script on each agent machine to create volumes for local storage
-1. Execute ***kubectl apply -f ubuntu/local-storage-provisioner.yaml*** against the Kubernetes cluster to create the local storage provisioner.
-1. Now, you can deploy the SQL Server 2019 big data cluster following instructions [here](https://docs.microsoft.com/en-us/sql/big-data-cluster/deployment-guidance?view=sqlallproducts-allversions)
+This folder contains a sample script that can be used to deploy a single-node Kubernetes cluster on a Linux machine.

samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/README.md

@@ -0,0 +1,41 @@
+
+# Deploy a SQL Server big data cluster on single node Kubernetes cluster (kubeadm)
+
+Using this sample bash script, you will deploy a single node Kubernetes cluster using  kubeadm and a SQL Server big data cluster on top of it. The script must be run from the VM you are planning to use for your kubeadm deployment.
+
+## Pre-requisites
+
+1. A vanilla Ubuntu 16.04 or 18.04 VM. All dependencies will be setup by the script. Using Azure Linux VMs is not yet supported.
+1. VM should have at least 8CPUs, 64GB RAM and 100GB disk space.After installing the images you will be left with 50GB for data/logs across all components.
+
+## Instructions
+
+1. Download the script on the VM you are planning to use for the deployment
+
+```
+
+curl --output kickstarter-azdata.sh  http://rima-5.guest.corp.microsoft.com/kickstarter-azdata.sh
+```
+
+1. Make the script executable
+
+```
+
+chmod +x kickstarter-azdata.sh
+```
+
+1. Run the script (make sure you are running with sudo)
+
+```
+
+sudo ./kickstarter-azdata.sh
+```
+
+1. Refresh alias setup for azdata
+
+```
+
+source ~/.bashrc
+```
+
+When prompted, provide your input for the password that will be used for all external endpoints: controller, SQL Server master and gateway. The password should be sufficiently complex based on existing rules for SQL Server password. The controller username is defaulted to *admin*.

samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/cleanup-bdc.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+if [ "$EUID" -ne 0 ]
+  then echo "Please run as root"
+  exit
+fi
+DIR_PREFIX=$1
+
+kubeadm reset --force
+
+systemctl stop kubelet
+rm -rf /var/lib/cni/
+rm -rf /var/lib/etcd/
+rm -rf /run/flannel/
+rm -rf /var/lib/kubelet/*
+rm -rf /etc/cni/
+rm -rf /etc/kubernetes/*
+ip link set cni0 down
+#brctl delbr cni0
+ip link set flannel.1 down
+#brctl delbr flannel.1
+iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X
+
+rm -rf .azdata/
+
+SERVICE_STOP_FAILED=0
+
+systemctl | grep "/var/lib/kubelet/pods" | while read -r line; do
+
+    # Retrieve the mount path
+    #
+    MOUNT_PATH=`echo "$line"  | grep -v echo | egrep -oh -m 1 "(/var/lib/kubelet/pods).+"`
+
+    if [ -z "$MOUNT_PATH" ]; then
+        continue
+    fi
+
+    if [[ ! -d "$MOUNT_PATH" ]] && [[ ! -f "$MOUNT_PATH" ]]; then
+
+        SERVICE=$(echo $line | cut -f1 -d' ')
+
+        echo "Mount "$MOUNT_PATH" no longer exists."
+        echo "Stopping orphaned mount service: '$SERVICE'"
+
+        systemctl stop $SERVICE
+
+        if [ $? -ne 0 ]; then
+            SERVICE_STOP_FAILED=1
+        fi
+
+        echo ""
+    fi
+done
+
+if [ $SERVICE_STOP_FAILED -ne 0 ]; then
+    echo "Not all services were stopped successfully. Please check the above output for more inforamtion."
+else
+    echo "All orphaned services successfully stopped."
+fi
+
+for i in $(seq 1 30); do
+
+  vol="vol$i"
+
+  sudo umount /mnt/local-storage/$vol
+
+  sudo rm -rf /mnt/local-storage/$vol
+
+done
+
+
+kubeadm reset -y
+sudo apt-get -y purge kubeadm kubectl kubelet kubernetes-cni kube*
+sudo apt-get autoremove
+sudo rm -rf ~/.kube