Skip to content

Commit 9264429

Browse files
barbkessbarbkess
committed
Merge pull request #27 from sidneyh/make-directory
new files for telemetry
2 parents 403f76e + 8904f61 commit 9264429

4 files changed

Lines changed: 486 additions & 0 deletions

File tree

‎samples/manage/azure-sql-db-elastic-pools/Pool Telemetry PS User Guide.docx‎

30.2 KB
Binary file not shown.

‎samples/manage/azure-sql-db-elastic-pools/PoolTelemetry.ps1‎

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
# ----------------------------------------------------------------------------------
2+
#
3+
# Copyright Microsoft Corporation
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
# ---------------------------------------------------------------------------------
14+
#
15+
# Sample script for loading SQL Database telemetry for pools and elastic databases on a single server
16+
# to a user-supplied Azure SQL database.
17+
#
18+
#
19+
# See additional comments in PoolTelemetryRunner.ps1, which includes script and instructions for running
20+
# this script as a PowerShell job, enabling data gathering for large numbers of servers in the background.
21+
#
22+
#
23+
function Load-PoolTelemetryForServer {
24+
param (
25+
[Parameter(Mandatory=$true)][string]$SubscriptionId, # Azure subscription owning the server from which telemetry will be gathered - see https://portal.azure.com
26+
[Parameter(Mandatory=$true)][string]$ResourceGroupName, # name of resource group containing the server from which telemetry will be gathered - see https://portal.azure.com
27+
[Parameter(Mandatory=$true)][string]$ServerName, # name of server from which telemetry will be gathered - e.g. "myappserver"
28+
[Parameter(Mandatory=$true)][string]$Location, # location of server, e.g. "Australia Southeast"
29+
[Parameter(Mandatory=$true)][PSCredential]$ServerCred,
30+
[Parameter(Mandatory=$true)][string]$OutputServerName, # server name of telemetry database, like "telemetryserver"
31+
[Parameter(Mandatory=$true)][string]$OutputDatabaseName, # telemetry database name, like "telemetrydb"
32+
[Parameter(Mandatory=$true)][PSCredential]$OutputServerCred,
33+
[Parameter(Mandatory=$true)][int]$IntervalMinutes, # interval for collection of telemetry
34+
[Parameter(Mandatory=$true)][int]$DurationMinutes, # total duration for collection of telemetry
35+
[Parameter(Mandatory=$true)][bool]$IncludeDatabases # indicates if telemetry should be gathered for databases as well as pools
36+
)
37+
38+
# Create output database metrics collection tables if it does not already exist.
39+
40+
$OutputServerCred.Password.MakeReadOnly()
41+
$sqlCred = new-object ("System.Data.SqlClient.SqlCredential") -ArgumentList $OutputServerCred.UserName, $OutputServerCred.Password
42+
43+
$outputConnection = New-Object ("System.Data.SqlClient.SqlConnection") "Data Source=$OutputServerName.database.windows.net;Integrated Security=false;Initial Catalog=$OutputDatabaseName"
44+
$outputConnection.Credential = $sqlCred
45+
$outputConnection.Open()
46+
47+
$poolResourceStatsTable = "pool_resource_stats" # <<< update if a different table name is required
48+
$dbResourceStatsTable = "db_resource_stats" # <<< update if a different table name is required
49+
50+
$sql =`
51+
"-- Create table for holding collected pool resource stats
52+
IF NOT EXISTS (SELECT * FROM sys.objects
53+
WHERE object_id = OBJECT_ID(N'$($poolResourceStatsTable)') AND type in (N'U'))
54+
55+
BEGIN
56+
Create Table $($poolResourceStatsTable) (subscription_guid uniqueidentifier, resource_group_name varchar(128), server_name varchar(128), location varchar(128), elastic_pool_name varchar(128), end_time datetime,
57+
elastic_pool_DTU_limit int, avg_cpu_percent decimal(5,2), avg_data_io_percent decimal(5,2), avg_log_io_percent decimal(5,2), max_worker_percent decimal(5,2), max_session_percent decimal(5,2)
58+
, avg_DTU_percent decimal(5,2), avg_storage_percent decimal(5,2), elastic_pool_storage_limit_mb bigint);
59+
Create Clustered Index ci_endtime ON $($poolResourceStatsTable) (end_time);
60+
END
61+
62+
-- Create table for holding collected database resource stats
63+
IF NOT EXISTS (SELECT * FROM sys.objects
64+
WHERE object_id = OBJECT_ID(N'$($dbResourceStatsTable)') AND type in (N'U'))
65+
66+
BEGIN
67+
Create Table $($dbResourceStatsTable) (subscription_guid uniqueidentifier, resource_group_name varchar(128),server_name varchar(128), location varchar(128), elastic_pool_name varchar(128), database_name varchar(128), end_time datetime,
68+
database_DTU_limit int, avg_cpu_percent decimal(5,2), avg_data_io_percent decimal(5,2), avg_log_io_percent decimal(5,2), max_worker_percent decimal(5,2), max_session_percent decimal(5,2), avg_DTU_percent decimal(5,2), db_size float);
69+
Create Clustered Index ci_endtime ON $($dbResourceStatsTable) (end_time);
70+
END
71+
72+
-- Create a function to get aggregated metrics for a given time interval
73+
IF NOT EXISTS (SELECT * FROM sys.objects
74+
WHERE name = N'get_aggregated_pool_metrics' AND type in ('IF'))
75+
EXEC sp_executesql @Statement = N'
76+
Create function get_aggregated_pool_metrics(
77+
@start datetime
78+
,@end datetime)
79+
RETURNS TABLE
80+
AS
81+
RETURN
82+
(
83+
SELECT
84+
location, server_name, elastic_pool_name
85+
,avg([avg_dtu_percent]) as avg_eDTU_percent
86+
,avg([avg_cpu_percent]) as avg_cpu_percent
87+
,avg([avg_data_io_percent]) as avg_data_io_percent
88+
,avg([avg_log_io_percent]) as avg_log_io_percent
89+
,avg([avg_storage_percent]) as avg_storage_percent
90+
,max([avg_dtu_percent]) as max_of_avg_eDTU_percent
91+
,max([avg_cpu_percent]) as max_of_avg_cpu_percent
92+
,max([avg_data_io_percent]) as max_of_data_io_percent
93+
,max([avg_log_io_percent]) as max_of_avg_log_io_percent
94+
,max([avg_storage_percent]) as max_of_avg_storage_percent
95+
,max([max_worker_percent]) as max_workers_percent
96+
,max([max_session_percent]) as max_session_percent
97+
FROM [dbo].[pool_resource_stats]
98+
WHERE end_time between @start and @end
99+
group by location, server_name, elastic_pool_name
100+
)'
101+
"
102+
103+
104+
$outputServerFullname = $OutputServerName + '.database.windows.net' # assumes server is in Azure SQL Database
105+
106+
Invoke-Sqlcmd -ServerInstance $outputServerFullName -Database $OutputDatabaseName -Username $OutputServerCred.UserName -Password $OutputServerCred.GetNetworkCredential().Password -Query $sql -ConnectionTimeout 120 -QueryTimeout 120
107+
108+
$sourceServerFullName = $ServerName + '.database.windows.net'
109+
110+
$interval = $IntervalMinutes
111+
$now = [DateTime]::UtcNow
112+
[DateTime]$startTime = $now.AddMinutes(-$interval) # sets the start time for the first telemetry collection
113+
[DateTime]$endTime = $now # sets the end time
114+
[DateTime]$finishTime = $now.AddMinutes($DurationMinutes) # sets the overall finish time for a telemetry collection session
115+
116+
Write-Host "Starting to collect telemetry for" $ServerName
117+
118+
$poolLagMinutes = 30 # This accommodates the lag in pool-level metrics being available in the DMV, does not affect per database telemetry
119+
120+
while($startTime -lt $finishTime)
121+
{
122+
$poolStartTime = $startTime.AddMinutes(-$poolLagMinutes) # sets the start of query window
123+
$poolEndTime = $endTime.AddMinutes(-$poolLagMinutes) # sets end of query window
124+
125+
Write-Host "Starting to collect elastic pool telemetry for period" $poolStartTime "to" $poolEndTime "(UTC)"
126+
127+
# Collect metrics for all elastic pools in this server.
128+
$sql = `
129+
"SELECT subscription_guid = CAST ('$($SubscriptionId)' AS uniqueidentifier), resource_group_name = '$($ResourceGroupName)', server_name = '$($ServerName)', location = '$($Location)', elastic_pool_name
130+
, end_time, elastic_pool_dtu_limit, avg_cpu_percent, avg_data_io_percent, avg_log_write_percent as avg_log_io_percent, max_worker_percent, max_session_percent
131+
,(SELECT Max(v) FROM (VALUES (avg_cpu_percent), (avg_data_io_percent), (avg_log_write_percent)) AS value(v)) AS avg_DTU_percent , avg_storage_percent, elastic_pool_storage_limit_mb FROM sys.elastic_pool_resource_stats
132+
WHERE end_time > '$($poolStartTime)' and end_time <= '$($poolEndTime)';"
133+
134+
$poolResult = Invoke-Sqlcmd -ServerInstance $sourceServerFullName -Database "master" -Username $ServerCred.UserName -Password $ServerCred.GetNetworkCredential().Password -Query $sql -ConnectionTimeout 120 -QueryTimeout 3600
135+
136+
if ($poolResult -ne $null)
137+
{
138+
#bulk copy the pool telemetry metrics to output database
139+
$bulkCopy = new-object ("Data.SqlClient.SqlBulkCopy") $outputConnection
140+
$bulkCopy.BulkCopyTimeout = 600
141+
$bulkCopy.DestinationTableName = "$poolResourceStatsTable";
142+
$bulkCopy.WriteToServer($poolResult);
143+
144+
Write-Host "Elastic pool telemetry loaded for server" $ServerName
145+
}
146+
else
147+
{
148+
Write-Host "No elastic pool telemetry found for server" $ServerName "for period" $poolStartTime "to" $poolEndTime "(UTC)"
149+
}
150+
151+
# gather elastic database telemetry if requested
152+
if ($IncludeDatabases)
153+
{
154+
# Get the list of current elastic databases
155+
$sql =`
156+
"select Name, elastic_pool_name from sys.databases as db
157+
inner join sys.database_service_objectives as dbso on db.database_id = dbso.database_id
158+
where (dbso.service_objective = 'ElasticPool') and db.Name != 'master'"
159+
160+
$dbList = Invoke-Sqlcmd -ServerInstance $sourceServerFullName -Database "master" -Username $ServerCred.UserName -Password $ServerCred.GetNetworkCredential().Password -Query $sql -ConnectionTimeout 120 -QueryTimeout 3600
161+
162+
if ($dbList -ne $null)
163+
{
164+
Write-Host $dbList.Count "elastic databases found on server" $ServerName
165+
166+
# Collect telemetry for each elastic database
167+
foreach ($db in $dbList)
168+
{
169+
$sql= `
170+
"Declare @db_size float;
171+
SELECT @db_size = SUM(reserved_page_count) * 8.0/1024/1024 FROM sys.dm_db_partition_stats
172+
SELECT subscription_guid = CAST ('$($SubscriptionId)' AS uniqueidentifier), resource_group_name = '$($ResourceGroupName)',server_name = '$($ServerName)', location = '$($Location)', elastic_pool_name = '$($db.elastic_pool_name)'
173+
, '$($db.Name)' as database_name, end_time, dtu_limit as database_dtu_limit, avg_cpu_percent, avg_data_io_percent, avg_log_write_percent as avg_log_io_percent, max_worker_percent, max_session_percent
174+
,(SELECT Max(v) FROM (VALUES (avg_cpu_percent), (avg_data_io_percent), (avg_log_write_percent)) AS value(v)) AS avg_DTU_percent ,@db_size as db_size FROM sys.dm_db_resource_stats
175+
WHERE end_time > '$($startTime)' and end_time <= '$($endTime)';"
176+
177+
$dbResult = Invoke-Sqlcmd -ServerInstance $SourceServerFullName -Database $db.Name -Username $ServerCred.UserName -Password $ServerCred.GetNetworkCredential().Password -Query $sql -ConnectionTimeout 120 -QueryTimeout 3600
178+
179+
if ($dbResult -ne $null)
180+
{
181+
#bulk copy the data to the telemetry database
182+
$bulkCopy = new-object ("Data.SqlClient.SqlBulkCopy") $outputConnection
183+
$bulkCopy.BulkCopyTimeout = 600
184+
$bulkCopy.DestinationTableName = "$dbResourceStatsTable";
185+
$bulkCopy.WriteToServer($dbResult);
186+
187+
Write-Host "Telemetry loaded for elastic database" $db.Name
188+
}
189+
else
190+
{
191+
Write-Host "No telemetry found for elastic database " $db.Name "for period" $startTime "to" $endTime
192+
}
193+
}
194+
}
195+
else
196+
{
197+
$now = [DateTime]::UtcNow
198+
199+
Write-Host "No elastic databases found for" $ServerName "when checking at" $now
200+
}
201+
}
202+
203+
Write-Host "Finished collection for server" $ServerName "for period" $startTime "to" $endTime
204+
205+
# set up time period for next collection
206+
$startTime = $startTime.AddMinutes($interval)
207+
$endTime = $endTime.AddMinutes($interval)
208+
209+
# end if the next period doesn't start before the finish time
210+
If ($startTime -ge $finishTime) {break}
211+
212+
Write-Host "Sleeping until" $endTime "(UTC)"
213+
214+
do
215+
{
216+
Start-Sleep 1
217+
} until (([DateTime]::UtcNow) -ge $endTime)
218+
}
219+
}

0 commit comments

Comments
 (0)