Skip to content

Commit 5bee11e

Browse files
uc-msftuc-msft
committed
Updated for RC changes & HA
1 parent 1897cca commit 5bee11e

3 files changed

Lines changed: 89 additions & 43 deletions

File tree

‎samples/features/sql-big-data-cluster/bootstrap-sample-db.cmd‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,10 @@ if NOT EXIST tpcxbb_1gb.bak (
4141
set SQLCMDSERVER=%SQL_MASTER_INSTANCE%
4242
set SQLCMDUSER=sa
4343
set SQLCMDPASSWORD=%SQL_MASTER_SA_PASSWORD%
44-
for /F "usebackq" %%v in (`sqlcmd -I -b -h-1 -Q "print @@SERVERNAME;"`) do SET MASTER_POD_NAME=%%v
44+
for /F "usebackq tokens=1,2" %%v in (`sqlcmd -I -b -h-1 -W -Q "SET NOCOUNT ON; SELECT @@SERVERNAME, SERVERPROPERTY('IsHadrEnabled');"`) do (
45+
SET MASTER_POD_NAME=%%v
46+
SET HADR_ENABLED=%%w
47+
)
4548

4649
REM Copy the backup file, restore the database, create necessary objects and data file
4750
echo Copying sales database backup file to SQL Master instance...
@@ -57,6 +60,9 @@ if /i "%AW_WWI_SAMPLES%" EQU "--install-extra-samples" (
5760
)
5861
echo Copying %%f database backup file to SQL Master instance...
5962
%DEBUG% kubectl cp %%f %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:var/opt/mssql/data/ -c mssql-server || goto exit
63+
64+
echo Removing database backup file...
65+
%DEBUG% kubectl exec %MASTER_POD_NAME% -n %CLUSTER_NAMESPACE% -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/%%f"
6066
)
6167

6268
set FILES=WideWorldImporters-Full.bak WideWorldImportersDW-Full.bak
@@ -67,39 +73,49 @@ if /i "%AW_WWI_SAMPLES%" EQU "--install-extra-samples" (
6773
)
6874
echo Copying %%f database backup file to SQL Master instance...
6975
%DEBUG% kubectl cp %%f %CLUSTER_NAMESPACE%/%MASTER_POD_NAME%:var/opt/mssql/data/ -c mssql-server || goto exit
76+
77+
echo Removing database backup file...
78+
%DEBUG% kubectl exec %MASTER_POD_NAME% -n %CLUSTER_NAMESPACE% -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/%%f"
7079
)
7180
)
7281

82+
REM If HADR is enabled then port-forward 1533 temporarily to connect to the primary directly
83+
REM Default timeout for port-forward is 5 minutes so start command in background & it will terminate automatically
84+
if /i "%HADR_ENABLED%" EQU "1" (
85+
%DEBUG% start kubectl port-forward pods/%MASTER_POD_NAME% 1533:1533 -n %CLUSTER_NAMESPACE%
86+
SET SQLCMDSERVER=127.0.0.1,1533
87+
)
88+
7389
echo Configuring sample database(s)...
7490
%DEBUG% sqlcmd -i "%STARTUP_PATH%bootstrap-sample-db.sql" -o "bootstrap.out" -I -b -v SA_PASSWORD="%KNOX_PASSWORD%" || goto exit
7591

7692
REM remove files copied into the pod:
77-
echo Removing database backup files...
93+
echo Removing database backup file...
7894
%DEBUG% kubectl exec %MASTER_POD_NAME% -n %CLUSTER_NAMESPACE% -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/tpcxbb_1gb.bak"
7995

8096
for %%F in (web_clickstreams inventory customer) do (
8197
if NOT EXIST %%F.csv (
8298
echo Exporting %%F data...
8399
if /i %%F EQU web_clickstreams (set DELIMITER=,) else (SET DELIMITER=^|)
84-
%DEBUG% bcp sales.dbo.%%F out "%%F.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t"!DELIMITER!" -o "%%F.out" -e "%%F.err" || goto exit
100+
%DEBUG% bcp sales.dbo.%%F out "%%F.csv" -S %SQLCMDSERVER% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t"!DELIMITER!" -o "%%F.out" -e "%%F.err" || goto exit
85101
)
86102
)
87103

88104
if NOT EXIST product_reviews.csv (
89105
echo Exporting product_reviews data...
90-
%DEBUG% bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S %SQL_MASTER_INSTANCE% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "product_reviews.out" -e "product_reviews.err" || goto exit
106+
%DEBUG% bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S %SQLCMDSERVER% -Usa -P%SQL_MASTER_SA_PASSWORD% -c -t, -o "product_reviews.out" -e "product_reviews.err" || goto exit
91107
)
92108

93109
REM Copy the data file to HDFS
94110
echo Uploading web_clickstreams data to HDFS...
95-
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || goto exit
96-
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "web_clickstreams.csv" || goto exit
111+
%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || goto exit
112+
%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "web_clickstreams.csv" || goto exit
97113
:: del /q web_clickstreams.*
98114

99115
echo.
100116
echo Uploading product_reviews data to HDFS...
101-
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || goto exit
102-
%DEBUG% curl -i -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || goto exit
117+
%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || goto exit
118+
%DEBUG% curl -L -k -u root:%KNOX_PASSWORD% -X PUT "https://%KNOX_ENDPOINT%/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || goto exit
103119
:: del /q product_reviews.*
104120

105121
REM %DEBUG% del /q *.out *.err *.csv

‎samples/features/sql-big-data-cluster/bootstrap-sample-db.sh‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -51,11 +51,10 @@ then
5151
$DEBUG curl -G "https://sqlchoice.blob.core.windows.net/sqlchoice/static/tpcxbb_1gb.bak" -o tpcxbb_1gb.bak
5252
fi
5353

54-
MASTER_POD_NAME=$(sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b -h-1 -Q "print @@SERVERNAME;")
54+
read -r MASTER_POD_NAME HADR_ENABLED <<<$(sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b -h-1 -Q "SET NOCOUNT ON; SELECT @@SERVERNAME, SERVERPROPERTY('IsHadrEnabled');")
5555

5656
echo Copying sales database backup file...
5757
$DEBUG kubectl cp tpcxbb_1gb.bak $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1)
58-
# $DEBUG rm tpcxbb_1gb.bak
5958

6059
if [ "$AW_WWI_SAMPLES" == "--install-extra-samples" ]
6160
then
@@ -68,6 +67,9 @@ then
6867
fi
6968
echo Copying $file database backup file to SQL Master instance...
7069
$DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1)
70+
71+
echo Removing database backup file...
72+
$DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/$file"
7173
done
7274

7375

@@ -80,16 +82,27 @@ then
8082
fi
8183
echo Copying $file database backup file to SQL Master instance...
8284
$DEBUG kubectl cp $file $CLUSTER_NAMESPACE/$MASTER_POD_NAME:var/opt/mssql/data -c mssql-server || (echo $ERROR_MESSAGE && exit 1)
85+
86+
echo Removing database backup file...
87+
$DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/$file"
8388
done
8489
fi
8590

91+
# If HADR is enabled then port-forward 1533 temporarily to connect to the primary directly
92+
# Default timeout for port-forward is 5 minutes so start command in background & it will terminate automatically
93+
if [ "$HADR_ENABLED" == "1" ]
94+
then
95+
$DEBUG kubectl port-forward pods/$MASTER_POD_NAME 1533:1533 -n $CLUSTER_NAMESPACE &
96+
SQL_MASTER_INSTANCE=127.0.0.1,1533
97+
fi
98+
8699
echo Configuring sample database...
87100
# WSL ex: "/mnt/c/Program Files/Microsoft SQL Server/Client SDK/ODBC/130/Tools/Binn/SQLCMD.EXE"
88101
export SA_PASSWORD=$KNOX_PASSWORD
89102
$DEBUG sqlcmd -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -I -b -i "$STARTUP_PATH/bootstrap-sample-db.sql" -o "bootstrap.out" || (echo $ERROR_MESSAGE && exit 2)
90103

91104
# remove files copied into the pod:
92-
echo Removing database backup files...
105+
echo Removing database backup file...
93106
$DEBUG kubectl exec $MASTER_POD_NAME -n $CLUSTER_NAMESPACE -c mssql-server -i -t -- bash -c "rm -rvf /var/opt/mssql/data/tpcxbb_1gb.bak"
94107

95108
for table in web_clickstreams inventory customer
@@ -104,26 +117,26 @@ for table in web_clickstreams inventory customer
104117
# WSL ex: "/mnt/c/Program Files/Microsoft SQL Server/Client SDK/ODBC/130/Tools/Binn/bcp.exe"
105118
if [ ! -f $table.csv ]
106119
then
107-
$DEBUG bcp sales.dbo.$table out "$table.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t"$DELIMITER" -e "$table.err" || (echo $ERROR_MESSAGE && exit 3)
120+
$DEBUG bcp sales.dbo.$table out "$table.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t"$DELIMITER" -e "$table.err" > "$table.out" || (echo $ERROR_MESSAGE && exit 3)
108121
fi
109122
done
110123

111124
if [ ! -f product_reviews.csv ]
112125
then
113126
echo Exporting product_reviews data...
114-
$DEBUG bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t, -e "product_reviews.err" || (echo $ERROR_MESSAGE && exit 3)
127+
$DEBUG bcp "select pr_review_sk, replace(replace(pr_review_content, ',', ';'), char(34), '') as pr_review_content from sales.dbo.product_reviews" queryout "product_reviews.csv" -S $SQL_MASTER_INSTANCE -Usa -P$SQL_MASTER_SA_PASSWORD -c -t, -e "product_reviews.err" > "$table.out" || (echo $ERROR_MESSAGE && exit 3)
115128
fi
116129

117130
# Copy the data file to HDFS
118131
echo Uploading web_clickstreams data to HDFS...
119-
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 4)
120-
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H 'Content-Type: application/octet-stream' -T "web_clickstreams.csv" || (echo $ERROR_MESSAGE && exit 5)
132+
$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 4)
133+
$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/clickstream_data/web_clickstreams.csv?op=create&overwrite=true" -H 'Content-Type: application/octet-stream' -T "web_clickstreams.csv" || (echo $ERROR_MESSAGE && exit 5)
121134
#$DEBUG rm -f web_clickstreams.*
122135

123136
echo
124137
echo Uploading product_reviews data to HDFS...
125-
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 6)
126-
$DEBUG curl -i -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || (echo $ERROR_MESSAGE && exit 7)
138+
$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data?op=MKDIRS" || (echo $ERROR_MESSAGE && exit 6)
139+
$DEBUG curl -L -k -u root:$KNOX_PASSWORD -X PUT "https://$KNOX_ENDPOINT/gateway/default/webhdfs/v1/product_review_data/product_reviews.csv?op=create&overwrite=true" -H "Content-Type: application/octet-stream" -T "product_reviews.csv" || (echo $ERROR_MESSAGE && exit 7)
127140
#$DEBUG rm -f product_reviews.*
128141

129142
echo

‎samples/features/sql-big-data-cluster/bootstrap-sample-db.sql‎

Lines changed: 43 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,16 @@ BEGIN
1818
END;
1919
GO
2020

21-
CREATE OR ALTER PROCEDURE #restore_database (@backup_file nvarchar(255))
21+
-- Enable option to allow INSERT against external table defined on HADOOP data source
22+
DECLARE @config_option nvarchar(100) = 'allow polybase export';
23+
IF NOT EXISTS(SELECT * FROM sys.configurations WHERE name = @config_option and value_in_use = 1)
24+
BEGIN
25+
EXECUTE sp_configure @config_option, 1;
26+
RECONFIGURE WITH OVERRIDE;
27+
END;
28+
GO
29+
30+
CREATE OR ALTER PROCEDURE #restore_database (@backup_file nvarchar(255), @db_name nvarchar(128))
2231
AS
2332
BEGIN
2433
DECLARE @restore_filelist_tmpl nvarchar(1000) = N'RESTORE FILELISTONLY FROM DISK = N''/var/opt/mssql/data/%F''';
@@ -53,7 +62,7 @@ BEGIN
5362
INSERT INTO @files
5463
EXECUTE(@restore_cmd);
5564

56-
SET @restore_cmd = REPLACE(REPLACE(@restore_database_tmpl, '%F', @backup_file), '%D', LEFT(@backup_file, CHARINDEX('.', @backup_file)-1));
65+
SET @restore_cmd = REPLACE(REPLACE(@restore_database_tmpl, '%F', @backup_file), '%D', @db_name);
5766
SET @restore_cur = CURSOR FAST_FORWARD FOR SELECT LogicalName, REVERSE(LEFT(REVERSE(PhysicalName), CHARINDEX('\', REVERSE(PhysicalName))-1)) FROM @files;
5867
OPEN @restore_cur;
5968
WHILE(1=1)
@@ -84,50 +93,58 @@ BEGIN
8493
WITH (LOCATION = 'sqlhdfs://controller-svc/default');
8594

8695
IF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'HadoopData')
87-
IF SERVERPROPERTY('ProductLevel') = 'CTP3.1'
88-
CREATE EXTERNAL DATA SOURCE HadoopData
89-
WITH(
90-
TYPE=HADOOP,
91-
LOCATION='hdfs://nmnode-0-svc:9000/',
92-
RESOURCE_MANAGER_LOCATION='master-svc:8032'
93-
);
94-
ELSE IF SERVERPROPERTY('ProductLevel') = 'CTP3.2'
95-
CREATE EXTERNAL DATA SOURCE HadoopData
96-
WITH(
97-
TYPE=HADOOP,
98-
LOCATION='hdfs://nmnode-0-svc:9000/',
99-
RESOURCE_MANAGER_LOCATION='sparkhead-svc:8032'
100-
);
96+
CREATE EXTERNAL DATA SOURCE HadoopData
97+
WITH(
98+
TYPE=HADOOP,
99+
LOCATION='hdfs://nmnode-0-svc:9000/',
100+
RESOURCE_MANAGER_LOCATION='sparkhead-svc:8032'
101+
);
101102
END;
102103
GO
103104

104105
--- Sample dbs:
105106
DECLARE @sample_dbs CURSOR, @proc nvarchar(255);
106107
SET @sample_dbs = CURSOR FAST_FORWARD FOR
107-
SELECT file_or_directory_name
108-
FROM sys.dm_os_enumerate_filesystem('/var/opt/mssql/data', '*.bak')
109-
WHERE DB_ID(REPLACE(REPLACE(file_or_directory_name, 'tpcxbb_1gb', 'sales'), '.bak', '')) IS NULL;
110-
DECLARE @file nvarchar(260);
108+
SELECT file_or_directory_name, d.db_name
109+
FROM sys.dm_os_enumerate_filesystem('/var/opt/mssql/data', '*.bak') as f
110+
CROSS APPLY (VALUES(REPLACE(REPLACE(file_or_directory_name, 'tpcxbb_1gb', 'sales'), '.bak', ''))) as d(db_name)
111+
WHERE DB_ID(d.db_name) IS NULL;
112+
DECLARE @file nvarchar(260), @db_name nvarchar(128);
111113
OPEN @sample_dbs;
112114
WHILE(1=1)
113115
BEGIN
114-
FETCH @sample_dbs INTO @file;
116+
FETCH @sample_dbs INTO @file, @db_name;
115117
IF @@FETCH_STATUS < 0 BREAK;
116118

117119
-- Restore the sample databases:
118-
EXECUTE #restore_database @file;
120+
EXECUTE #restore_database @file, @db_name;
119121

120122
-- Get database name used in restore:
121-
SET @proc = CONCAT(QUOTENAME(LEFT(@file, CHARINDEX('.', @file)-1)), N'.sys.sp_executesql');
123+
SET @db_name = QUOTENAME(@db_name);
124+
SET @proc = CONCAT(@db_name, N'.sys.sp_executesql');
122125

123126
EXECUTE @proc N'#create_data_sources';
124127

125128
-- Set compatibility level to 150:
126129
EXECUTE @proc N'ALTER DATABASE CURRENT SET COMPATIBILITY_LEVEL = 150';
127130

128-
-- Rename TPCx-BB database:
129-
IF DB_ID('tpcxbb_1gb') IS NOT NULL
130-
ALTER DATABASE tpcxbb_1gb MODIFY NAME = sales;
131+
-- Check for HADR & add database to containedag:
132+
IF SERVERPROPERTY('IsHadrEnabled') = 1
133+
BEGIN
134+
DECLARE @command nvarchar(1000);
135+
IF EXISTS(SELECT * FROM sys.databases WHERE name = PARSENAME(@db_name,1) and recovery_model_desc = 'SIMPLE')
136+
BEGIN
137+
-- Set recovery to full
138+
EXECUTE @proc N'ALTER DATABASE CURRENT SET RECOVERY FULL';
139+
140+
SET @command = CONCAT(N'BACKUP DATABASE ', @db_name, ' TO DISK = ''/dev/null'';' );
141+
EXEC(@command);
142+
END;
143+
144+
-- Add database to AG
145+
SET @command = CONCAT(N'ALTER AVAILABILITY GROUP containedag ADD DATABASE ', @db_name);
146+
EXEC(@command);
147+
END;
131148
END;
132149
GO
133150

0 commit comments

Comments
 (0)