Added new Python tutorial

Author: negust-microsoft

samples/features/machine-learning-services/python/getting-started/customer-clustering/customer_clustering.sql

@@ -0,0 +1,104 @@
+USE [tpcxbb_1gb]
+GO
+
+-- Stored procedure that performs customer clustering using Python and SQL Server ML Services
+DROP PROCEDURE IF EXISTS [dbo].[py_generate_customer_return_clusters]
+GO
+CREATE procedure [dbo].[py_generate_customer_return_clusters]
+AS
+
+BEGIN
+	DECLARE
+
+-- Input query to generate the purchase history & return metrics
+	 @input_query NVARCHAR(MAX) = N'
+SELECT
+  ss_customer_sk AS customer,
+  CAST( (ROUND(COALESCE(returns_count / NULLIF(1.0*orders_count, 0), 0), 7) ) AS FLOAT) AS orderRatio,
+  CAST( (ROUND(COALESCE(returns_items / NULLIF(1.0*orders_items, 0), 0), 7) ) AS FLOAT) AS itemsRatio,
+  CAST( (ROUND(COALESCE(returns_money / NULLIF(1.0*orders_money, 0), 0), 7) ) AS FLOAT) AS monetaryRatio,
+  CAST( (COALESCE(returns_count, 0)) AS FLOAT) AS frequency  
+FROM
+  (
+    SELECT
+      ss_customer_sk,
+      -- return order ratio
+      COUNT(distinct(ss_ticket_number)) AS orders_count,
+      -- return ss_item_sk ratio
+      COUNT(ss_item_sk) AS orders_items,
+      -- return monetary amount ratio
+      SUM( ss_net_paid ) AS orders_money
+    FROM store_sales s
+    GROUP BY ss_customer_sk
+  ) orders
+  LEFT OUTER JOIN
+  (
+    SELECT
+      sr_customer_sk,
+      -- return order ratio
+      count(distinct(sr_ticket_number)) as returns_count,
+      -- return ss_item_sk ratio
+      COUNT(sr_item_sk) as returns_items,
+      -- return monetary amount ratio
+      SUM( sr_return_amt ) AS returns_money
+    FROM store_returns
+    GROUP BY sr_customer_sk
+  ) returned ON ss_customer_sk=sr_customer_sk 
+ '
+
+EXEC sp_execute_external_script
+	  @language = N'Python'
+	, @script = N'
+
+import pandas as pd
+from sklearn.cluster import KMeans
+
+#get data from input query
+customer_data = my_input_data
+
+#We concluded in step2 in the tutorial that 4 would be a good number of clusters
+n_clusters = 4
+
+#Perform clustering
+est = KMeans(n_clusters=n_clusters, random_state=111).fit(customer_data[["orderRatio","itemsRatio","monetaryRatio","frequency"]])
+clusters = est.labels_
+customer_data["cluster"] = clusters
+
+OutputDataSet = customer_data
+'
+	, @input_data_1 = @input_query
+	, @input_data_1_name = N'my_input_data'
+			 with result sets (("Customer" int, "orderRatio" float,"itemsRatio" float,"monetaryRatio" float,"frequency" float,"cluster" float));
+END;
+GO
+
+
+
+--Creating a table for storing the clustering data
+DROP TABLE IF EXISTS [dbo].[py_customer_clusters];
+GO
+--Create a table to store the predictions in
+CREATE TABLE [dbo].[py_customer_clusters](
+ [Customer] [bigint] NULL,
+ [OrderRatio] [float] NULL,
+ [itemsRatio] [float] NULL,
+ [monetaryRatio] [float] NULL,
+ [frequency] [float] NULL,
+ [cluster] [int] NULL,
+ ) ON [PRIMARY]
+GO
+
+--Execute the clustering and insert results into table
+INSERT INTO py_customer_clusters
+EXEC [dbo].[py_generate_customer_return_clusters];
+
+-- Select contents of the table
+SELECT * FROM py_customer_clusters;
+
+--Get email addresses of customers in cluster 0 
+SELECT customer.[c_email_address], customer.c_customer_sk
+  FROM dbo.customer
+  JOIN
+  [dbo].[py_customer_clusters] as c
+  ON c.Customer = customer.c_customer_sk
+  WHERE c.cluster = 0;

samples/features/machine-learning-services/python/getting-started/customer-clustering/customer_clustering_ng.py

@@ -0,0 +1,116 @@
+# Load packages.
+import pandas as pd
+from revoscalepy import RxInSqlServer, RxSqlServerData, RxComputeContext, rx_import
+from sklearn.cluster import KMeans
+from sklearn.decomposition import PCA
+import matplotlib.pyplot as plt
+from mpl_toolkits.mplot3d import Axes3D
+from scipy.spatial.distance import cdist, pdist
+import numpy as np
+
+
+def perform_clustering():
+    ##########################################################################################################################################
+
+    ##	Connect to DB and select data
+
+    ##########################################################################################################################################
+
+    # Connection string to connect to SQL Server named instance
+    conn_str = 'Driver=SQL Server;Server=localhost;Database=tpcxbb_1gb;Trusted_Connection=True;'
+
+    input_query = '''SELECT
+    ss_customer_sk AS customer,
+    ROUND(COALESCE(returns_count / NULLIF(1.0*orders_count, 0), 0), 7) AS orderRatio,
+    ROUND(COALESCE(returns_items / NULLIF(1.0*orders_items, 0), 0), 7) AS itemsRatio,
+    ROUND(COALESCE(returns_money / NULLIF(1.0*orders_money, 0), 0), 7) AS monetaryRatio,
+    COALESCE(returns_count, 0) AS frequency 
+    FROM
+    (
+    SELECT
+      ss_customer_sk,
+      -- return order ratio
+      COUNT(distinct(ss_ticket_number)) AS orders_count,
+      -- return ss_item_sk ratio
+      COUNT(ss_item_sk) AS orders_items,
+      -- return monetary amount ratio
+      SUM( ss_net_paid ) AS orders_money
+    FROM store_sales s
+    GROUP BY ss_customer_sk
+  ) orders
+  LEFT OUTER JOIN
+  (
+    SELECT
+      sr_customer_sk,
+      -- return order ratio
+      count(distinct(sr_ticket_number)) as returns_count,
+      -- return ss_item_sk ratio
+      COUNT(sr_item_sk) as returns_items,
+      -- return monetary amount ratio
+      SUM( sr_return_amt ) AS returns_money
+    FROM store_returns
+    GROUP BY sr_customer_sk ) returned ON ss_customer_sk=sr_customer_sk'''
+
+
+ # Define the columns we wish to import
+    column_info = {
+        "customer": {"type": "integer"},
+        "orderRatio": {"type": "integer"},
+        "itemsRatio": {"type": "integer"},
+        "frequency": {"type": "integer"}
+    }
+
+    data_source = RxSqlServerData(sql_query=input_query, column_Info=column_info, connection_string=conn_str)
+    RxInSqlServer(connection_string=conn_str, num_tasks=1, auto_cleanup=False)
+    # import data source and convert to pandas dataframe
+    customer_data = pd.DataFrame(rx_import(data_source))
+    print("Data frame:", customer_data.head(n=20))
+
+    ##########################################################################################################################################
+
+    ##	Determine number of clusters using the Elbow method
+
+    ##########################################################################################################################################
+
+    cdata = customer_data
+    K = range(1, 20)
+    KM = [KMeans(n_clusters=k).fit(cdata) for k in K]
+    centroids = [k.cluster_centers_ for k in KM]
+
+    D_k = [cdist(cdata, cent, 'euclidean') for cent in centroids]
+    dist = [np.min(D, axis=1) for D in D_k]
+    avgWithinSS = [sum(d) / cdata.shape[0] for d in dist]
+    plt.plot(K, avgWithinSS, 'b*-')
+    plt.grid(True)
+    plt.xlabel('Number of clusters')
+    plt.ylabel('Average within-cluster sum of squares')
+    plt.title('Elbow for KMeans clustering')
+    plt.show()
+
+
+    ##########################################################################################################################################
+
+    ##	Perform clustering using Kmeans
+
+    ##########################################################################################################################################
+
+    #It looks like k=4 is a good number to use based on the elbow graph
+    n_clusters = 4
+
+    est = KMeans(n_clusters=n_clusters, random_state=111).fit(customer_data[["orderRatio", "itemsRatio", "monetaryRatio", "frequency"]])
+    clusters = est.labels_
+    customer_data['cluster'] = clusters
+
+    #Print some data about the clusters:
+
+    #For each cluster, count the members
+    for c in range(n_clusters):
+        cluster_members=customer_data[customer_data['cluster']== c][:]
+        print('Cluster{0}(n={1}):'.format(c,len(cluster_members)))
+        print('-------------------')
+
+    #Print mean values per cluster
+    print(customer_data.groupby(['cluster']).mean())
+
+
+perform_clustering()