Implement feature/instrumentation statsbeat (#1076)

Author: lzchen

contrib/opencensus-ext-azure/CHANGELOG.md

@@ -4,12 +4,14 @@
 
 - Enable AAD authorization via TokenCredential
 ([#1021](https://github.com/census-instrumentation/opencensus-python/pull/1021))
-- Implement attach rate metrics via Statbeat
+- Implement attach rate metrics via Statsbeat
 ([#1053](https://github.com/census-instrumentation/opencensus-python/pull/1053))
-- Implement network metrics via Statbeat - Success count
+- Implement network metrics via Statsbeat - Success count
 ([#1059](https://github.com/census-instrumentation/opencensus-python/pull/1059))
-- Implement network metrics via Statbeat - Others
+- Implement network metrics via Statsbeat - Others
 ([#1062](https://github.com/census-instrumentation/opencensus-python/pull/1062))
+- Implement feature and instrumentation metrics via Statsbeat
+([#1076](https://github.com/census-instrumentation/opencensus-python/pull/1076))
 
 ## 1.0.8
 Released 2021-05-13

contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py

@@ -111,7 +111,6 @@ def __init__(self, *args, **kwargs):
         credential=None,  # Credential class used by AAD auth
         enable_local_storage=True,
         enable_standard_metrics=True,  # Used by metrics exporter, True to send standard metrics  # noqa: E501
-        enable_stats_metrics=True,  # True to send stats metrics
         endpoint='https://dc.services.visualstudio.com/v2/track',
         export_interval=15.0,
         grace_period=5.0,

contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py

@@ -14,6 +14,7 @@
 
 import json
 import logging
+import os
 import threading
 import time
 
@@ -29,8 +30,7 @@
 
 class TransportMixin(object):
     def _check_stats_collection(self):
-        return self.options.enable_stats_metrics and \
-            (not hasattr(self, '_is_stats') or not self._is_stats)
+        return not os.environ.get("APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL") and (not hasattr(self, '_is_stats') or not self._is_stats)  # noqa: E501
 
     def _transmit_from_storage(self):
         if self.storage:

contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import logging
+import os
 import random
 import threading
 import time
@@ -63,10 +64,8 @@ def __init__(self, **options):
         self._worker = Worker(self._queue, self)
         self._worker.start()
         # start statsbeat on exporter instantiation
-        if self.options.enable_stats_metrics:
-            statsbeat_metrics.collect_statsbeat_metrics(
-                self.options.instrumentation_key
-            )
+        if not os.environ.get("APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL"):
+            statsbeat_metrics.collect_statsbeat_metrics(self.options)
 
     def _export(self, batch, event=None):  # pragma: NO COVER
         try:

contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py

@@ -14,6 +14,7 @@
 
 import atexit
 import logging
+import os
 
 from opencensus.common import utils as common_utils
 from opencensus.ext.azure.common import Options, utils
@@ -162,10 +163,8 @@ def new_metrics_exporter(**options):
                                     producers,
                                     exporter,
                                     interval=exporter.options.export_interval)
-    if exporter.options.enable_stats_metrics:
+    if not os.environ.get("APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL"):
         from opencensus.ext.azure.metrics_exporter import statsbeat_metrics
         # Stats will track the user's ikey
-        statsbeat_metrics.collect_statsbeat_metrics(
-            exporter.options.instrumentation_key
-        )
+        statsbeat_metrics.collect_statsbeat_metrics(exporter.options)
     return exporter

contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/statsbeat_metrics/__init__.py

@@ -27,7 +27,7 @@
 _STATSBEAT_LOCK = threading.Lock()
 
 
-def collect_statsbeat_metrics(ikey):
+def collect_statsbeat_metrics(options):
     with _STATSBEAT_LOCK:
         # Only start statsbeat if did not exist before
         global _STATSBEAT_METRICS  # pylint: disable=global-statement
@@ -39,9 +39,7 @@ def collect_statsbeat_metrics(ikey):
                 export_interval=_STATS_SHORT_EXPORT_INTERVAL,  # 15m by default
             )
             # The user's ikey is the one being tracked
-            producer = _AzureStatsbeatMetricsProducer(
-                instrumentation_key=ikey
-            )
+            producer = _AzureStatsbeatMetricsProducer(options)
             _STATSBEAT_METRICS = producer
             # Export some initial stats on program start
             exporter.export_metrics(_STATSBEAT_METRICS.get_initial_metrics())
@@ -54,10 +52,11 @@ def collect_statsbeat_metrics(ikey):
 class _AzureStatsbeatMetricsProducer(MetricProducer):
     """Implementation of the producer of statsbeat metrics.
 
-    Includes Azure attach rate metrics, implemented using gauges.
+    Includes Azure attach rate, network and feature metrics,
+    implemented using gauges.
     """
-    def __init__(self, instrumentation_key):
-        self._statsbeat = _StatsbeatMetrics(instrumentation_key)
+    def __init__(self, options):
+        self._statsbeat = _StatsbeatMetrics(options)
 
     def get_metrics(self):
         return self._statsbeat.get_metrics()

contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/statsbeat_metrics/statsbeat.py

@@ -30,6 +30,7 @@
 )
 from opencensus.metrics.label_key import LabelKey
 from opencensus.metrics.label_value import LabelValue
+from opencensus.trace.integrations import get_integrations
 
 _AIMS_URI = "http://169.254.169.254/metadata/instance/compute"
 _AIMS_API_VERSION = "api-version=2017-12-01"
@@ -40,18 +41,31 @@
 _DEFAULT_STATS_LONG_EXPORT_INTERVAL = 86400  # 24 hours
 
 _ATTACH_METRIC_NAME = "Attach"
+_FEATURE_METRIC_NAME = "Feature"
 _REQ_SUC_COUNT_NAME = "Request Success Count"
 _REQ_FAIL_COUNT_NAME = "Request Failure Count"
 _REQ_DURATION_NAME = "Request Duration"
 _REQ_RETRY_NAME = "Request Retry Count"
 _REQ_THROTTLE_NAME = "Request Throttle Count"
 _REQ_EXCEPTION_NAME = "Request Exception Count"
 
-_RP_NAMES = ["appsvc", "function", "vm", "unknown"]
+_ENDPOINT_TYPES = ["breeze"]
+_RP_NAMES = ["appsvc", "functions", "vm", "unknown"]
 
 _logger = logging.getLogger(__name__)
 
 
+class _FEATURE_TYPES:
+    FEATURE = 0
+    INSTRUMENTATION = 1
+
+
+class _StatsbeatFeature:
+    NONE = 0
+    DISK_RETRY = 1
+    AAD_HANDLING = 2
+
+
 def _get_stats_connection_string():
     cs_env = os.environ.get("APPLICATION_INSIGHTS_STATS_CONNECTION_STRING")
     if cs_env:
@@ -103,6 +117,15 @@ def _get_attach_properties():
 
 def _get_network_properties():
     properties = _get_common_properties()
+    properties.append(LabelKey("endpoint", "ingestion endpoint type"))
+    properties.append(LabelKey("host", "destination of ingestion endpoint"))
+    return properties
+
+
+def _get_feature_properties():
+    properties = _get_common_properties()
+    properties.insert(4, LabelKey("feature", 'represents enabled features'))
+    properties.insert(4, LabelKey("type", 'type, either feature or instrumentation'))  # noqa: E501
     return properties
 
 
@@ -162,8 +185,12 @@ def _get_exception_count_value():
 
 class _StatsbeatMetrics:
 
-    def __init__(self, instrumentation_key):
-        self._instrumentation_key = instrumentation_key
+    def __init__(self, options):
+        self._options = options
+        self._instrumentation_key = options.instrumentation_key
+        self._feature = _StatsbeatFeature.NONE
+        if options.enable_local_storage:
+            self._feature |= _StatsbeatFeature.DISK_RETRY
         self._stats_lock = threading.Lock()
         self._vm_data = {}
         self._vm_retry = True
@@ -218,6 +245,21 @@ def __init__(self, instrumentation_key):
             'count',
             _get_network_properties(),
         )
+        # feature/instrumentation metrics
+        # metrics related to what features and instrumentations are enabled
+        self._feature_metric = LongGauge(
+            _FEATURE_METRIC_NAME,
+            'Statsbeat metric related to features enabled',  # noqa: E501
+            'count',
+            _get_feature_properties(),
+        )
+        # Instrumentation metric uses same name/properties as feature
+        self._instrumentation_metric = LongGauge(
+            _FEATURE_METRIC_NAME,
+            'Statsbeat metric related to instrumentations enabled',  # noqa: E501
+            'count',
+            _get_feature_properties(),
+        )
 
     # Metrics that are sent on application start
     def get_initial_metrics(self):
@@ -226,6 +268,14 @@ def get_initial_metrics(self):
             attach_metric = self._get_attach_metric()
             if attach_metric:
                 stats_metrics.append(attach_metric)
+        if self._feature_metric:
+            feature_metric = self._get_feature_metric()
+            if feature_metric:
+                stats_metrics.append(feature_metric)
+        if self._instrumentation_metric:
+            instr_metric = self._get_instrumentation_metric()
+            if instr_metric:
+                stats_metrics.append(instr_metric)
         return stats_metrics
 
     # Metrics sent every statsbeat interval
@@ -248,6 +298,8 @@ def get_metrics(self):
 
     def _get_network_metrics(self):
         properties = self._get_common_properties()
+        properties.append(LabelValue(_ENDPOINT_TYPES[0]))  # endpoint
+        properties.append(LabelValue(self._options.endpoint))  # host
         metrics = []
         for fn, metric in self._network_metrics.items():
             # NOTE: A time series is a set of unique label values
@@ -260,6 +312,20 @@ def _get_network_metrics(self):
                 metrics.append(stats_metric)
         return metrics
 
+    def _get_feature_metric(self):
+        properties = self._get_common_properties()
+        properties.insert(4, LabelValue(self._feature))  # feature long
+        properties.insert(4, LabelValue(_FEATURE_TYPES.FEATURE))  # type
+        self._feature_metric.get_or_create_time_series(properties)
+        return self._feature_metric.get_metric(datetime.datetime.utcnow())
+
+    def _get_instrumentation_metric(self):
+        properties = self._get_common_properties()
+        properties.insert(4, LabelValue(get_integrations()))  # instr long
+        properties.insert(4, LabelValue(_FEATURE_TYPES.INSTRUMENTATION))  # type  # noqa: E501
+        self._instrumentation_metric.get_or_create_time_series(properties)
+        return self._instrumentation_metric.get_metric(datetime.datetime.utcnow())  # noqa: E501
+
     def _get_attach_metric(self):
         properties = []
         rp = ''
@@ -303,9 +369,7 @@ def _get_common_properties(self):
         properties.append(LabelValue(platform.python_version()))
         properties.append(LabelValue(self._os_type or platform.system()))  # os
         properties.append(LabelValue("python"))  # language
-        # version
-        properties.append(
-            LabelValue(ext_version))
+        properties.append(LabelValue(ext_version))  # version
         return properties
 
     def _get_azure_compute_metadata(self):

contrib/opencensus-ext-azure/opencensus/ext/azure/trace_exporter/__init__.py

@@ -15,6 +15,7 @@
 import atexit
 import json
 import logging
+import os
 
 from opencensus.common.schedule import QueueExitEvent
 from opencensus.ext.azure.common import Options, utils
@@ -74,10 +75,8 @@ def __init__(self, **options):
         super(AzureExporter, self).__init__(**options)
         atexit.register(self._stop, self.options.grace_period)
         # start statsbeat on exporter instantiation
-        if self.options.enable_stats_metrics:
-            statsbeat_metrics.collect_statsbeat_metrics(
-                self.options.instrumentation_key
-            )
+        if not os.environ.get("APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL"):
+            statsbeat_metrics.collect_statsbeat_metrics(self.options)
 
     def span_data_to_envelope(self, sd):
         envelope = Envelope(