Skip to content

Commit 6649942

Browse files
bweissmanbweissman
authored
Merge pull request #2 from microsoft/master
full
2 parents dbc2a33 + a5fb597 commit 6649942

10 files changed

Lines changed: 142 additions & 123 deletions

File tree

‎samples/features/sql-big-data-cluster/app-deploy/sentiment-analysis/pre-package-install.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
#!/bin/bash -e
2+
apt update
23
apt install microsoft-mlserver-mml-r-9.3.0

‎samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd‎

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,10 @@ if NOT EXIST tpcxbb_1gb.bak (
4141
set SQLCMDSERVER=%SQL_MASTER_INSTANCE%
4242
set SQLCMDUSER=sa
4343
set SQLCMDPASSWORD=%SQL_MASTER_SA_PASSWORD%
44-
for /F "usebackq" %%v in (`sqlcmd -I -b -h-1 -Q "print RTRIM((CAST(SERVERPROPERTY('ProductLevel') as nvarchar(128))));"`) do SET CTP_VERSION=%%v
45-
if /i "%CTP_VERSION%" EQU "CTP2.4" (set MASTER_POD_NAME=mssql-master-pool-0) else (set MASTER_POD_NAME=master-0)
44+
for /F "usebackq tokens=1,2" %%v in (`sqlcmd -I -b -h-1 -W -Q "SET NOCOUNT ON; SELECT @@SERVERNAME, SERVERPROPERTY('IsHadrEnabled');"`) do (
45+
SET MASTER_POD_NAME=%%v
46+
SET HADR_ENABLED=%%w
47+
)
4648

4749
REM Copy the backup file, restore the database, create necessary objects and data file
4850
echo Copying sales database backup file to SQL Master instance...
@@ -58,6 +60,9 @@ if /i "%AW_WWI_SAMPLES%" EQU "--install-extra-samples" (
5860
)
5961
echo Copying %%f database backup file to SQL Master instance...
6062
%DEBUG% kubectl cp %%f %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:var/opt/mssql/data/ -c mssql-server || goto exit
63+
64+
echo Removing database backup file...
65+
%DEBUG% kubectl exec %MASTER_POD_NAME% -n %CLUSTER_NAMESPACE% -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/%%f"
6166
)
6267

6368
set FILES=WideWorldImporters-Full.bak WideWorldImportersDW-Full.bak
@@ -68,39 +73,49 @@ if /i "%AW_WWI_SAMPLES%" EQU "--install-extra-samples" (
6873
)
6974
echo Copying %%f database backup file to SQL Master instance...
7075
%DEBUG% kubectl cp %%f %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:var/opt/mssql/data/ -c mssql-server || goto exit
76+
77+
echo Removing database backup file...
78+
%DEBUG% kubectl exec %MASTER_POD_NAME% -n %CLUSTER_NAMESPACE% -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/%%f"
7179
)
7280
)
7381

82+
REM If HADR is enabled then port-forward 1533 temporarily to connect to the primary directly
83+
REM Default timeout for port-forward is 5 minutes so start command in background & it will terminate automatically
84+
if /i "%HADR_ENABLED%" EQU "1" (
85+
%DEBUG% start kubectl port-forward pods/%MASTER_POD_NAME% 1533:1533 -n %CLUSTER_NAMESPACE%
86+
SET SQLCMDSERVER=127.0.0.1,1533
87+
)
88+
7489
echo Configuring sample database(s)...
7590
%DEBUG% sqlcmd -i "%STARTUP_PATH%bootstrap-sample-db.sql" -o "bootstrap.out" -I -b -v SA_PASSWORD="%KNOX_PASSWORD%" || goto exit
7691

7792
REM remove files copied into the pod:
78-
echo Removing database backup files...
93+
echo Removing database backup file...
7994
%DEBUG% kubectl exec %MASTER_POD_NAME% -n %CLUSTER_NAMESPACE% -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/tpcxbb_1gb.bak"
8095

8196
for %%F in (web_clickstreams inventory customer) do (
8297
if NOT EXIST %%F.csv (
8398
echo Exporting %%F data...
8499
if /i %%F EQU web_clickstreams (set DELIMITER=,) else (SET DELIMITER=^|)
85-
%DEBUG% bcp sales.dbo.%%F out "%%F.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t"!DELIMITER!" -o "%%F.out" -e "%%F.err" || goto exit
100+
%DEBUG% bcp sales.dbo.%%F out "%%F.csv" -S %SQLCMDSERVER% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t"!DELIMITER!" -o "%%F.out" -e "%%F.err" || goto exit
86101
)
87102
)
88103

89104
if NOT EXIST product_reviews.csv (
90105
echo Exporting product_reviews data...
91-
%DEBUG% bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "product_reviews.out" -e "product_reviews.err" || goto exit
106+
%DEBUG% bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S %SQLCMDSERVER% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "product_reviews.out" -e "product_reviews.err" || goto exit
92107
)
93108

94109
REM Copy the data file to HDFS
95110
echo Uploading web_clickstreams data to HDFS...
96-
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || goto exit
97-
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "web_clickstreams.csv" || goto exit
111+
%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || goto exit
112+
%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "web_clickstreams.csv" || goto exit
98113
:: del /q web_clickstreams.*
99114

100115
echo.
101116
echo Uploading product_reviews data to HDFS...
102-
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || goto exit
103-
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || goto exit
117+
%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || goto exit
118+
%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || goto exit
104119
:: del /q product_reviews.*
105120

106121
REM %DEBUG% del /q *.out *.err *.csv

‎samples/features/sql-big-data-cluster/bootstrap-sample-db.sh‎

Lines changed: 22 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -51,18 +51,10 @@ then
5151
$DEBUG curl -G "https://sqlchoice.blob.core.windows.net/sqlchoice/static/tpcxbb_1gb.bak" -o tpcxbb_1gb.bak
5252
fi
5353

54-
CTP_VERSION=$(sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b -h-1 -Q "print RTRIM((CAST(SERVERPROPERTY('ProductLevel') as nvarchar(128))));")
55-
56-
if [ "$CTP_VERSION" == "CTP2.4" ]
57-
then
58-
MASTER_POD_NAME=mssql-master-pool-0
59-
else
60-
MASTER_POD_NAME=master-0
61-
fi
54+
read -r MASTER_POD_NAME HADR_ENABLED <<<$(sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b -h-1 -Q "SET NOCOUNT ON; SELECT @@SERVERNAME, SERVERPROPERTY('IsHadrEnabled');")
6255

6356
echo Copying sales database backup file...
6457
$DEBUG kubectl cp tpcxbb_1gb.bak $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1)
65-
# $DEBUG rm tpcxbb_1gb.bak
6658

6759
if [ "$AW_WWI_SAMPLES" == "--install-extra-samples" ]
6860
then
@@ -75,6 +67,9 @@ then
7567
fi
7668
echo Copying $file database backup file to SQL Master instance...
7769
$DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1)
70+
71+
echo Removing database backup file...
72+
$DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/$file"
7873
done
7974

8075

@@ -87,16 +82,27 @@ then
8782
fi
8883
echo Copying $file database backup file to SQL Master instance...
8984
$DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1)
85+
86+
echo Removing database backup file...
87+
$DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/$file"
9088
done
9189
fi
9290

91+
# If HADR is enabled then port-forward 1533 temporarily to connect to the primary directly
92+
# Default timeout for port-forward is 5 minutes so start command in background & it will terminate automatically
93+
if [ "$HADR_ENABLED" == "1" ]
94+
then
95+
$DEBUG kubectl port-forward pods/$MASTER_POD_NAME 1533:1533 -n $CLUSTER_NAMESPACE &
96+
SQL_MASTER_INSTANCE=127.0.0.1,1533
97+
fi
98+
9399
echo Configuring sample database...
94100
# WSL ex: "/mnt/c/Program Files/Microsoft SQL Server/Client SDK/ODBC/130/Tools/Binn/SQLCMD.EXE"
95101
export SA_PASSWORD=$KNOX_PASSWORD
96102
$DEBUG sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b -i "$STARTUP_PATH/bootstrap-sample-db.sql" -o "bootstrap.out" || (echo $ERROR_MESSAGE && exit 2)
97103

98104
# remove files copied into the pod:
99-
echo Removing database backup files...
105+
echo Removing database backup file...
100106
$DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/tpcxbb_1gb.bak"
101107

102108
for table in web_clickstreams inventory customer
@@ -111,26 +117,26 @@ for table in web_clickstreams inventory customer
111117
# WSL ex: "/mnt/c/Program Files/Microsoft SQL Server/Client SDK/ODBC/130/Tools/Binn/bcp.exe"
112118
if [ ! -f $table.csv ]
113119
then
114-
$DEBUG bcp sales.dbo.$table out "$table.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t"$DELIMITER" -e "$table.err" || (echo $ERROR_MESSAGE && exit 3)
120+
$DEBUG bcp sales.dbo.$table out "$table.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t"$DELIMITER" -e "$table.err" > "$table.out" || (echo $ERROR_MESSAGE && exit 3)
115121
fi
116122
done
117123

118124
if [ ! -f product_reviews.csv ]
119125
then
120126
echo Exporting product_reviews data...
121-
$DEBUG bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t, -e "product_reviews.err" || (echo $ERROR_MESSAGE && exit 3)
127+
$DEBUG bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t, -e "product_reviews.err" > "$table.out" || (echo $ERROR_MESSAGE && exit 3)
122128
fi
123129

124130
# Copy the data file to HDFS
125131
echo Uploading web_clickstreams data to HDFS...
126-
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 4)
127-
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H 'Content-Type: application/octet-stream' -T "web_clickstreams.csv" || (echo $ERROR_MESSAGE && exit 5)
132+
$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 4)
133+
$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H 'Content-Type: application/octet-stream' -T "web_clickstreams.csv" || (echo $ERROR_MESSAGE && exit 5)
128134
#$DEBUG rm -f web_clickstreams.*
129135

130136
echo
131137
echo Uploading product_reviews data to HDFS...
132-
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 6)
133-
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || (echo $ERROR_MESSAGE && exit 7)
138+
$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 6)
139+
$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || (echo $ERROR_MESSAGE && exit 7)
134140
#$DEBUG rm -f product_reviews.*
135141

136142
echo

‎samples/features/sql-big-data-cluster/bootstrap-sample-db.sql‎

Lines changed: 43 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,16 @@ BEGIN
1818
END;
1919
GO
2020

21-
CREATE OR ALTER PROCEDURE #restore_database (@backup_file nvarchar(255))
21+
-- Enable option to allow INSERT against external table defined on HADOOP data source
22+
DECLARE @config_option nvarchar(100) = 'allow polybase export';
23+
IF NOT EXISTS(SELECT * FROM sys.configurations WHERE name = @config_option and value_in_use = 1)
24+
BEGIN
25+
EXECUTE sp_configure @config_option, 1;
26+
RECONFIGURE WITH OVERRIDE;
27+
END;
28+
GO
29+
30+
CREATE OR ALTER PROCEDURE #restore_database (@backup_file nvarchar(255), @db_name nvarchar(128))
2231
AS
2332
BEGIN
2433
DECLARE @restore_filelist_tmpl nvarchar(1000) = N'RESTORE FILELISTONLY FROM DISK = N''/var/opt/mssql/data/%F''';
@@ -53,7 +62,7 @@ BEGIN
5362
INSERT INTO @files
5463
EXECUTE(@restore_cmd);
5564

56-
SET @restore_cmd = REPLACE(REPLACE(@restore_database_tmpl, '%F', @backup_file), '%D', LEFT(@backup_file, CHARINDEX('.', @backup_file)-1));
65+
SET @restore_cmd = REPLACE(REPLACE(@restore_database_tmpl, '%F', @backup_file), '%D', @db_name);
5766
SET @restore_cur = CURSOR FAST_FORWARD FOR SELECT LogicalName, REVERSE(LEFT(REVERSE(PhysicalName), CHARINDEX('\', REVERSE(PhysicalName))-1)) FROM @files;
5867
OPEN @restore_cur;
5968
WHILE(1=1)
@@ -84,50 +93,58 @@ BEGIN
8493
WITH (LOCATION = 'sqlhdfs://controller-svc/default');
8594

8695
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData')
87-
IF SERVERPROPERTY('ProductLevel') = 'CTP3.1'
88-
CREATE EXTERNAL DATA SOURCE HadoopData
89-
WITH(
90-
TYPE=HADOOP,
91-
LOCATION='hdfs://nmnode-0-svc:9000/',
92-
RESOURCE_MANAGER_LOCATION='master-svc:8032'
93-
);
94-
ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.2'
95-
CREATE EXTERNAL DATA SOURCE HadoopData
96-
WITH(
97-
TYPE=HADOOP,
98-
LOCATION='hdfs://nmnode-0-svc:9000/',
99-
RESOURCE_MANAGER_LOCATION='sparkhead-svc:8032'
100-
);
96+
CREATE EXTERNAL DATA SOURCE HadoopData
97+
WITH(
98+
TYPE=HADOOP,
99+
LOCATION='hdfs://nmnode-0-svc:9000/',
100+
RESOURCE_MANAGER_LOCATION='sparkhead-svc:8032'
101+
);
101102
END;
102103
GO
103104

104105
--- Sample dbs:
105106
DECLARE @sample_dbs CURSOR, @proc nvarchar(255);
106107
SET @sample_dbs = CURSOR FAST_FORWARD FOR
107-
SELECT file_or_directory_name
108-
FROM sys.dm_os_enumerate_filesystem('/var/opt/mssql/data', '*.bak')
109-
WHERE DB_ID(REPLACE(REPLACE(file_or_directory_name, 'tpcxbb_1gb', 'sales'), '.bak', '')) IS NULL;
110-
DECLARE @file nvarchar(260);
108+
SELECT file_or_directory_name, d.db_name
109+
FROM sys.dm_os_enumerate_filesystem('/var/opt/mssql/data', '*.bak') as f
110+
CROSS APPLY (VALUES(REPLACE(REPLACE(file_or_directory_name, 'tpcxbb_1gb', 'sales'), '.bak', ''))) as d(db_name)
111+
WHERE DB_ID(d.db_name) IS NULL;
112+
DECLARE @file nvarchar(260), @db_name nvarchar(128);
111113
OPEN @sample_dbs;
112114
WHILE(1=1)
113115
BEGIN
114-
FETCH @sample_dbs INTO @file;
116+
FETCH @sample_dbs INTO @file, @db_name;
115117
IF @@FETCH_STATUS < 0 BREAK;
116118

117119
-- Restore the sample databases:
118-
EXECUTE #restore_database @file;
120+
EXECUTE #restore_database @file, @db_name;
119121

120122
-- Get database name used in restore:
121-
SET @proc = CONCAT(QUOTENAME(LEFT(@file, CHARINDEX('.', @file)-1)), N'.sys.sp_executesql');
123+
SET @db_name = QUOTENAME(@db_name);
124+
SET @proc = CONCAT(@db_name, N'.sys.sp_executesql');
122125

123126
EXECUTE @proc N'#create_data_sources';
124127

125128
-- Set compatibility level to 150:
126129
EXECUTE @proc N'ALTER DATABASE CURRENT SET COMPATIBILITY_LEVEL = 150';
127130

128-
-- Rename TPCx-BB database:
129-
IF DB_ID('tpcxbb_1gb') IS NOT NULL
130-
ALTER DATABASE tpcxbb_1gb MODIFY NAME = sales;
131+
-- Check for HADR & add database to containedag:
132+
IF SERVERPROPERTY('IsHadrEnabled') = 1
133+
BEGIN
134+
DECLARE @command nvarchar(1000);
135+
IF EXISTS(SELECT * FROM sys.databases WHERE name = PARSENAME(@db_name,1) and recovery_model_desc = 'SIMPLE')
136+
BEGIN
137+
-- Set recovery to full
138+
EXECUTE @proc N'ALTER DATABASE CURRENT SET RECOVERY FULL';
139+
140+
SET @command = CONCAT(N'BACKUP DATABASE ', @db_name, ' TO DISK = ''NUL'';' );
141+
EXEC(@command);
142+
END;
143+
144+
-- Add database to AG
145+
SET @command = CONCAT(N'ALTER AVAILABILITY GROUP containedag ADD DATABASE ', @db_name);
146+
EXEC(@command);
147+
END;
131148
END;
132149
GO
133150

‎samples/features/sql-big-data-cluster/data-pool/data-ingestion-sql.sql‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,8 @@ GO
44
-- Create external data source for Data Pool inside a SQL big data cluster
55
--
66
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlDataPool')
7-
IF SERVERPROPERTY('ProductLevel') = 'CTP3.0'
8-
CREATE EXTERNAL DATA SOURCE SqlDataPool
9-
WITH (LOCATION = 'sqldatapool://controller-svc:8080/datapools/default');
10-
ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.1'
11-
CREATE EXTERNAL DATA SOURCE SqlDataPool
12-
WITH (LOCATION = 'sqldatapool://controller-svc/default');
7+
CREATE EXTERNAL DATA SOURCE SqlDataPool
8+
WITH (LOCATION = 'sqldatapool://controller-svc/default');
139

1410
-- Create external table in a data pool in SQL Server 2019 big data cluster.
1511
-- The SqlDataPool data source is a special data source that is available in

‎samples/features/sql-big-data-cluster/data-virtualization/hadoop/inventory-export-hdfs-rcfile.sql‎

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,20 +17,12 @@ GO
1717
-- execution.
1818
--
1919
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData')
20-
IF SERVERPROPERTY('ProductLevel') = 'CTP3.1'
21-
CREATE EXTERNAL DATA SOURCE HadoopData
22-
WITH(
23-
TYPE=HADOOP,
24-
LOCATION='hdfs://nmnode-0-svc:9000/',
25-
RESOURCE_MANAGER_LOCATION='master-svc:8032'
26-
);
27-
ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.2'
28-
CREATE EXTERNAL DATA SOURCE HadoopData
29-
WITH(
30-
TYPE=HADOOP,
31-
LOCATION='hdfs://nmnode-0-svc:9000/',
32-
RESOURCE_MANAGER_LOCATION='sparkhead-svc:8032'
33-
);
20+
CREATE EXTERNAL DATA SOURCE HadoopData
21+
WITH(
22+
TYPE=HADOOP,
23+
LOCATION='hdfs://nmnode-0-svc:9000/',
24+
RESOURCE_MANAGER_LOCATION='sparkhead-svc:8032'
25+
);
3426

3527

3628
-- Create file format for RCFILE with appropriate properties.

‎samples/features/sql-big-data-cluster/data-virtualization/hadoop/product-reviews-hdfs-orc.sql‎

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,12 @@ GO
77
-- execution.
88
--
99
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData')
10-
IF SERVERPROPERTY('ProductLevel') = 'CTP3.1'
11-
CREATE EXTERNAL DATA SOURCE HadoopData
12-
WITH(
13-
TYPE=HADOOP,
14-
LOCATION='hdfs://nmnode-0-svc:9000/',
15-
RESOURCE_MANAGER_LOCATION='master-svc:8032'
16-
);
17-
ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.2'
18-
CREATE EXTERNAL DATA SOURCE HadoopData
19-
WITH(
20-
TYPE=HADOOP,
21-
LOCATION='hdfs://nmnode-0-svc:9000/',
22-
RESOURCE_MANAGER_LOCATION='sparkhead-svc:8032'
23-
);
10+
CREATE EXTERNAL DATA SOURCE HadoopData
11+
WITH(
12+
TYPE=HADOOP,
13+
LOCATION='hdfs://nmnode-0-svc:9000/',
14+
RESOURCE_MANAGER_LOCATION='sparkhead-svc:8032'
15+
);
2416

2517

2618
-- Create file format for orc file with appropriate properties.

‎samples/features/sql-big-data-cluster/data-virtualization/hadoop/web-clickstreams-hdfs-orc.sql‎

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,12 @@ GO
77
-- execution.
88
--
99
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData')
10-
IF SERVERPROPERTY('ProductLevel') = 'CTP3.1'
11-
CREATE EXTERNAL DATA SOURCE HadoopData
12-
WITH(
13-
TYPE=HADOOP,
14-
LOCATION='hdfs://nmnode-0-svc:9000/',
15-
RESOURCE_MANAGER_LOCATION='master-svc:8032'
16-
);
17-
ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.2'
18-
CREATE EXTERNAL DATA SOURCE HadoopData
19-
WITH(
20-
TYPE=HADOOP,
21-
LOCATION='hdfs://nmnode-0-svc:9000/',
22-
RESOURCE_MANAGER_LOCATION='sparkhead-svc:8032'
23-
);
10+
CREATE EXTERNAL DATA SOURCE HadoopData
11+
WITH(
12+
TYPE=HADOOP,
13+
LOCATION='hdfs://nmnode-0-svc:9000/',
14+
RESOURCE_MANAGER_LOCATION='sparkhead-svc:8032'
15+
);
2416

2517

2618
-- Create file format for orc file with appropriate properties.

0 commit comments

Comments
 (0)