Add utils to automatically detect monitored resources. (#292)

Author: mayurkale22

opencensus/common/http_handler.py

@@ -0,0 +1,43 @@
+# Copyright 2018 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+try:
+    # For Python 3.0 and later
+    from urllib.request import urlopen, Request
+    from urllib.error import HTTPError, URLError
+except ImportError:
+    # Fall back to Python 2's urllib2
+    from urllib2 import urlopen, Request
+    from urllib2 import HTTPError, URLError
+
+
+import socket
+
+_REQUEST_TIMEOUT = 2  # in secs
+
+
+def get_request(request_url, request_headers=dict()):
+    """Execute http get request on given request_url with optional headers
+    """
+    request = Request(request_url)
+    for key, val in request_headers.items():
+        request.add_header(key, val)
+
+    try:
+        response = urlopen(request, timeout=_REQUEST_TIMEOUT)
+        response_content = response.read()
+    except (HTTPError, URLError, socket.timeout):
+        response_content = None
+
+    return response_content

opencensus/common/monitored_resource_util/__init__.py

@@ -0,0 +1,90 @@
+# Copyright 2018 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from opencensus.common.http_handler import get_request
+import json
+
+# AWS provides Instance Metadata via below url
+_AWS_INSTANCE_IDENTITY_DOCUMENT_URI = \
+    "http://169.254.169.254/latest/dynamic/instance-identity/document"
+
+_AWS_ATTRIBUTES = {
+    # Region is the AWS region for the VM. The format of this field is
+    # "aws:{region}", where supported values for {region} are listed at
+    # http://docs.aws.amazon.com/general/latest/gr/rande.html.
+    'region': 'region',
+
+    # accountId is the AWS account number for the VM.
+    'accountId': 'aws_account',
+
+    # instanceId is the instance id of the instance.
+    'instanceId': 'instance_id'
+}
+
+# inited is used to make sure AWS initialize executes only once.
+inited = False
+
+# Detects if the application is running on EC2 by making a connection to AWS
+# instance identity document URI.If connection is successful, application
+# should be on an EC2 instance.
+is_running_on_aws = False
+
+aws_metadata_map = {}
+
+
+class AwsIdentityDocumentUtils(object):
+    """Util methods for getting and parsing AWS instance identity document."""
+
+    inited = False
+    is_running = False
+
+    @classmethod
+    def _initialize_aws_identity_document(cls):
+        """This method, tries to establish an HTTP connection to AWS instance
+        identity document url. If the application is running on an EC2
+        instance, we should be able to get back a valid JSON document. Make a
+        http get request call and store data in local map.
+        This method should only be called once.
+        """
+
+        if cls.inited:
+            return
+
+        content = get_request(_AWS_INSTANCE_IDENTITY_DOCUMENT_URI)
+        if content is not None:
+            content = json.loads(content)
+            for env_var, attribute_key in _AWS_ATTRIBUTES.items():
+                attribute_value = content.get(env_var)
+                if attribute_value is not None:
+                    aws_metadata_map[attribute_key] = attribute_value
+
+            cls.is_running = True
+
+        cls.inited = True
+
+    @classmethod
+    def is_running_on_aws(cls):
+        cls._initialize_aws_identity_document()
+        return cls.is_running
+
+    def get_aws_metadata(self):
+        """AWS Instance Identity Document is a JSON file.
+        See docs.aws.amazon.com/AWSEC2/latest/UserGuide/
+        instance-identity-documents.html.
+        :return:
+        """
+        if self.is_running_on_aws():
+            return aws_metadata_map
+
+        return dict()

opencensus/common/monitored_resource_util/aws_identity_doc_utils.py

@@ -0,0 +1,152 @@
+# Copyright 2018 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from opencensus.common.http_handler import get_request
+import os
+
+_GCP_METADATA_URI = 'http://metadata/computeMetadata/v1/'
+_GCP_METADATA_URI_HEADER = {'Metadata-Flavor': 'Google'}
+
+# GCE common attributes
+# See: https://cloud.google.com/appengine/docs/flexible/python/runtime#
+#      environment_variables
+_GCE_ATTRIBUTES = {
+    # ProjectID is the identifier of the GCP project associated with this
+    # resource, such as "my-project".
+    'project_id': 'project/project-id',
+
+    # instance_id is the numeric VM instance identifier assigned by
+    # Compute Engine.
+    'instance_id': 'instance/id',
+
+    # zone is the Compute Engine zone in which the VM is running.
+    'zone': 'instance/zone'
+}
+
+_GKE_ATTRIBUTES = {
+    # ProjectID is the identifier of the GCP project associated with this
+    # resource, such as "my-project".
+    'project_id': 'project/project-id',
+
+    # instance_id is the numeric VM instance identifier assigned by
+    # Compute Engine.
+    'instance_id': 'instance/id',
+
+    # zone is the Compute Engine zone in which the VM is running.
+    'zone': 'instance/zone',
+
+    # cluster_name is the name for the cluster the container is running in.
+    'cluster_name': 'instance/attributes/cluster-name'
+}
+
+# Following attributes are derived from environment variables. They are
+# configured via yaml file. For details refer to:
+# https://cloud.google.com/kubernetes-engine/docs/tutorials/
+# custom-metrics-autoscaling#exporting_metrics_from_the_application
+_GKE_ENV_ATTRIBUTES = {
+    # ContainerName is the name of the container.
+    'container_name': 'CONTAINER_NAME',
+
+    # namespace_id is the identifier for the cluster namespace the container
+    # is running in
+    'namespace_id': 'NAMESPACE',
+
+    # pod_id is the identifier for the pod the container is running in.
+    'pod_id': 'HOSTNAME'
+}
+
+# Kubenertes environment variables
+_KUBERNETES_SERVICE_HOST = 'KUBERNETES_SERVICE_HOST'
+
+gcp_metadata_map = {}
+
+
+class GcpMetadataConfig(object):
+    """GcpMetadata represents metadata retrieved from GCP (GKE and GCE)
+    environment. Some attributes are retrieved from the system environment.
+    see : <a href="https://cloud.google.com/compute/docs/
+    storing-retrieving-metadata"> https://cloud.google.com/compute/docs/storing
+    -retrieving-metadata</a>
+    """
+    inited = False
+    is_running = False
+
+    @classmethod
+    def _initialize_metadata_service(cls):
+        """Initialize metadata service once and load gcp metadata into map
+        This method should only be called once.
+        """
+        if cls.inited:
+            return
+
+        instance_id = cls._get_attribute('instance_id')
+
+        if instance_id is not None:
+            cls.is_running = True
+
+            gcp_metadata_map['instance_id'] = instance_id
+
+            attributes = _GCE_ATTRIBUTES
+            if _KUBERNETES_SERVICE_HOST in os.environ:
+                attributes = _GKE_ATTRIBUTES
+
+            # fetch attributes from metadata request
+            for attribute_key, attribute_uri in attributes.items():
+                if attribute_key not in gcp_metadata_map:
+                    attribute_value = cls._get_attribute(attribute_key)
+                    if attribute_value is not None:
+                        gcp_metadata_map[attribute_key] = attribute_value
+
+        cls.inited = True
+
+    @classmethod
+    def is_running_on_gcp(cls):
+        cls._initialize_metadata_service()
+        return cls.is_running
+
+    def get_gce_metadata(self):
+        """for GCP GCE instance"""
+        if self.is_running_on_gcp():
+            return gcp_metadata_map
+
+        return dict()
+
+    def get_gke_metadata(self):
+        """for GCP GKE container."""
+        gke_metadata = {}
+
+        if self.is_running_on_gcp():
+            gke_metadata = gcp_metadata_map
+
+        # fetch attributes from Environment Variables
+        for attribute_key, attribute_env in _GKE_ENV_ATTRIBUTES.items():
+            attribute_value = os.environ.get(attribute_env)
+            if attribute_value is not None:
+                gke_metadata[attribute_key] = attribute_value
+
+        return gke_metadata
+
+    @staticmethod
+    def _get_attribute(attribute_key):
+        """
+        Fetch the requested instance metadata entry.
+        :param attribute_uri: attribute_uri: attribute name relative to the
+        computeMetadata/v1 prefix
+        :return:  The value read from the metadata service or None
+        """
+        attribute_value = get_request(_GCP_METADATA_URI +
+                                      _GKE_ATTRIBUTES[attribute_key],
+                                      _GCP_METADATA_URI_HEADER)
+
+        return attribute_value

opencensus/common/monitored_resource_util/gcp_metadata_config.py

@@ -0,0 +1,86 @@
+# Copyright 2018 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from opencensus.common.monitored_resource_util.gcp_metadata_config \
+    import GcpMetadataConfig
+from opencensus.common.monitored_resource_util.aws_identity_doc_utils \
+    import AwsIdentityDocumentUtils
+
+# supported environments (resource types)
+_GCE_INSTANCE = "gce_instance"
+_GKE_CONTAINER = "gke_container"
+_AWS_EC2_INSTANCE = "aws_ec2_instance"
+
+
+class MonitoredResource(object):
+    """MonitoredResource returns the resource type and resource labels.
+    """
+
+    @property
+    def resource_type(self):
+        """Returns the resource type this MonitoredResource.
+        :return:
+        """
+        raise NotImplementedError  # pragma: NO COVER
+
+    def get_resource_labels(self):
+        """Returns the resource labels for this MonitoredResource.
+        :return:
+        """
+        raise NotImplementedError  # pragma: NO COVER
+
+
+class GcpGceMonitoredResource(MonitoredResource):
+    """GceMonitoredResource represents gce_instance type monitored resource.
+    For definition refer to
+    https://cloud.google.com/monitoring/api/resources#tag_gce_instance
+    """
+
+    @property
+    def resource_type(self):
+        return _GCE_INSTANCE
+
+    def get_resource_labels(self):
+        gcp_config = GcpMetadataConfig()
+        return gcp_config.get_gce_metadata()
+
+
+class GcpGkeMonitoredResource(MonitoredResource):
+    """GkeMonitoredResource represents gke_container type monitored resource.
+    For definition refer to
+    https://cloud.google.com/monitoring/api/resources#tag_gke_container
+    """
+
+    @property
+    def resource_type(self):
+        return _GKE_CONTAINER
+
+    def get_resource_labels(self):
+        gcp_config = GcpMetadataConfig()
+        return gcp_config.get_gke_metadata()
+
+
+class AwsMonitoredResource(MonitoredResource):
+    """AwsMonitoredResource represents aws_ec2_instance type monitored resource.
+    For definition refer to
+    https://cloud.google.com/monitoring/api/resources#tag_aws_ec2_instance
+    """
+    @property
+    def resource_type(self):
+        return _AWS_EC2_INSTANCE
+
+    def get_resource_labels(self):
+        aws_util = AwsIdentityDocumentUtils()
+        return aws_util.get_aws_metadata()