Rename internal_types to collectd_types

Fix examples, docs, comments after #8 has merged.

Change-Id: Ib9ba9fc1c89b5779fdecf8c43630271effb0a411

Author: zzzeek

examples/helloworld/client.conf

@@ -1,28 +1,20 @@
-# A config file that lets you test collectd locally.
+# A config file that lets you test SQLAlchemy-collectd locally.
 # Run just like this:
-# collectd -f -C collectdconsole.conf
+# collectd -f -C client.conf
 # received messages will be dumped as JSON to stdout.
 
 LoadPlugin logfile
 <Plugin logfile>
-        # note "debug" will usually not work
-       LogLevel "info"
-       File STDOUT
-       Timestamp true
-       PrintSeverity true
+    # note "debug" will usually not work
+    LogLevel "info"
+    File STDOUT
+    Timestamp true
+    PrintSeverity true
 </Plugin>
 
 LoadPlugin write_log
 <Plugin write_log>
-   Format JSON
-</Plugin>
-
-LoadPlugin network
-<Plugin network>
-  <Server "localhost" "25826">
-    SecurityLevel "None"
-
-  </Server>
+    Format JSON
 </Plugin>
 
 LoadPlugin python
@@ -36,18 +28,10 @@ LoadPlugin python
         listen "localhost" 25827
 
         # set to "debug" to show messaging
-        loglevel "debug"
+        loglevel "info"
 
     </Module>
 
-    Import "sqlalchemy_collectd.connmon.plugin"
-    <Module "sqlalchemy_collectd.connmon.plugin">
-      monitor "localhost" 25828
-
-        # set to "debug" to show messaging
-        #loglevel "debug"
-    </Module>
-
 </Plugin>
 
 

examples/helloworld/client_plus_connmon.conf

@@ -3,7 +3,7 @@
 #
 #
 # Run the server as:
-# collectd -f -C collectdconsole.conf
+# collectd -f -C client_plus_connmon.conf
 #
 # received messages will be dumped as JSON to stdout.
 #
@@ -13,17 +13,17 @@
 # send logging to STDOUT
 LoadPlugin logfile
 <Plugin logfile>
-      # note "debug" will usually not work
-     LogLevel "info"
-     File STDOUT
-     Timestamp true
-     PrintSeverity true
+    # note "debug" will usually not work
+    LogLevel "info"
+    File STDOUT
+    Timestamp true
+    PrintSeverity true
 </Plugin>
 
 # write messages in JSON format to the log
 LoadPlugin write_log
 <Plugin write_log>
-   Format JSON
+    Format JSON
 </Plugin>
 
 LoadPlugin python

sqlalchemy_collectd/client/sender.py

@@ -1,6 +1,6 @@
 import threading
 
-from .. import internal_types
+from .. import collectd_types
 from .. import protocol
 
 
@@ -26,7 +26,7 @@ def __init__(
         collectd_host,
         collectd_port,
         log,
-        plugin=internal_types.COLLECTD_PLUGIN_NAME,
+        plugin=collectd_types.COLLECTD_PLUGIN_NAME,
     ):
         self.hostname = hostname
         self.stats_name = stats_name
@@ -69,10 +69,10 @@ def get_sender(
             cls.create_mutex.release()
 
 
-@sends(internal_types.pool_internal)
+@sends(collectd_types.pool_internal)
 def _send_pool(values, collection_target):
     return values.build(
-        type=internal_types.pool_internal.name,
+        type=collectd_types.pool_internal.name,
         values=[
             collection_target.num_pools,
             collection_target.num_checkedout,
@@ -84,10 +84,10 @@ def _send_pool(values, collection_target):
     )
 
 
-@sends(internal_types.totals_internal)
+@sends(collectd_types.totals_internal)
 def _send_connection_totals(values, collection_target):
     return values.build(
-        type=internal_types.totals_internal.name,
+        type=collectd_types.totals_internal.name,
         values=[
             collection_target.total_checkouts,
             collection_target.total_invalidated,

sqlalchemy_collectd/internal_types.py sqlalchemy_collectd/collectd_types.pysqlalchemy_collectd/internal_types.py renamed to sqlalchemy_collectd/collectd_types.py

@@ -1,7 +1,24 @@
-from . import protocol
+"""Type objects representing the collectd types used.
+
+There are two classes of types here; "internal" types are used between
+the SQLAlchemy-collectd client plugin embedded in applications and the
+SQLAlchemy-collectd server plugin.  These type objects are not part of
+collectd's types.db, and in the interests of configuration simplicity
+and portability, these types are not exposed outside of the communication
+between the two SQLAlchemy-collectd plugins.
+
+The other type defined here are "external" types, which are public collectd
+types defined in /usr/share/collectd/types.db or similar.  We are using the
+"count" and "derive" types which are generic types that define a single "GAUGE"
+and a single "DERIVE" value, respectively.  When the SQLAlchemy-collectd server
+plugin aggregates and reports on the results it receives from the client, it
+converts the "internal" types into "external" types and streams them into
+collectd as the result.   These are the stats that are then consumable by all
+collectd "writer" systems such as logging, network and grafana.
 
-# internal types.  These are used by the SQLAlchemy client plugin
-# to talk to the SQLAlchemy collectd plugin.
+
+"""
+from . import protocol
 
 
 COLLECTD_PLUGIN_NAME = "sqlalchemy"
@@ -45,6 +62,6 @@
 )
 
 
+# external types "count" and "derive".
 count_external = protocol.Type("count", ("value", protocol.VALUE_GAUGE))
-
 derive_external = protocol.Type("derive", ("value", protocol.VALUE_DERIVE))

sqlalchemy_collectd/connmon/main.py

@@ -5,7 +5,7 @@
 
 from . import display
 from . import stat
-from .. import internal_types
+from .. import collectd_types
 from .. import protocol
 
 log = logging.getLogger(__name__)
@@ -30,29 +30,17 @@ def main(argv=None):
 
     network_receiver = protocol.NetworkReceiver(
         protocol.ServerConnection(options.host, options.port, log),
-        [internal_types.count_external, internal_types.derive_external],
+        [collectd_types.count_external, collectd_types.derive_external],
     )
 
     stat_ = stat.Stat(network_receiver, log)
     stat_.start()
 
     service_str = "[Direct host: %s:%s]" % (options.host, options.port)
 
-    # _dummy_wait()
     display_ = display.Display(stat_, service_str)
     display_.start()
 
 
-def _dummy_wait():
-    import time
-    import logging
-
-    # logging.basicConfig()
-    log.setLevel(logging.DEBUG)
-
-    while True:
-        time.sleep(5)
-
-
 if __name__ == "__main__":
     main()

sqlalchemy_collectd/connmon/plugin.py

@@ -4,7 +4,7 @@
 
 import collectd
 
-from .. import internal_types
+from .. import collectd_types
 from .. import protocol
 from ..server.logging import CollectdHandler
 
@@ -27,7 +27,7 @@ def start_plugin(config):
     )
     CollectdHandler.setup(__name__, config_dict.get("loglevel", ("info",))[0])
 
-    types = [internal_types.derive_external, internal_types.count_external]
+    types = [collectd_types.derive_external, collectd_types.count_external]
 
     message_sender = protocol.NetworkSender(
         protocol.ClientConnection(monitor_host, int(monitor_port), log), types
@@ -42,17 +42,16 @@ def start_plugin(config):
 
 
 def write(cd_values_obj):
-    """Receive values from the collectd server in which we are embedded, and
-    send them to an aggregator that will broadcast them to clients.
+    """Receive values from the collectd server and forward out on UDP
+    for connmon listeners to receive.
 
     The values are received as "external" types, meaning they use the
-    "derive" and "count" types in collectd types.db; an aggregator collects
-    these and builds them into "internal" types which combine multiple
-    types of values into single records.
+    "derive" and "count" types in collectd types.db.  The connmon client
+    now receives and interprets these objects directly.
 
     """
 
-    if cd_values_obj.plugin == internal_types.COLLECTD_PLUGIN_NAME:
+    if cd_values_obj.plugin == collectd_types.COLLECTD_PLUGIN_NAME:
         values_obj = protocol.Values.from_collectd_values(cd_values_obj, log)
         message_sender.send(values_obj)
 

sqlalchemy_collectd/server/receiver.py

@@ -3,7 +3,7 @@
 import itertools
 import logging
 
-from .. import internal_types
+from .. import collectd_types
 from .. import protocol
 from .. import stream
 
@@ -12,20 +12,20 @@
 
 class Receiver(object):
     def __init__(
-        self, host, port, log, plugin=internal_types.COLLECTD_PLUGIN_NAME
+        self, host, port, log, plugin=collectd_types.COLLECTD_PLUGIN_NAME
     ):
         self.log = log
         self.plugin = plugin
-        self.internal_types = [
-            internal_types.pool_internal,
-            internal_types.totals_internal,
-            internal_types.process_internal,
+        self.collectd_types = [
+            collectd_types.pool_internal,
+            collectd_types.totals_internal,
+            collectd_types.process_internal,
         ]
         self.network_receiver = protocol.NetworkReceiver(
-            protocol.ServerConnection(host, port, log), self.internal_types
+            protocol.ServerConnection(host, port, log), self.collectd_types
         )
-        self.translator = stream.StreamTranslator(*self.internal_types)
-        self.bucket_names = [t.name for t in self.internal_types]
+        self.translator = stream.StreamTranslator(*self.collectd_types)
+        self.bucket_names = [t.name for t in self.collectd_types]
         self.buckets = {
             name: stream.TimeBucket() for name in self.bucket_names
         }
@@ -35,7 +35,7 @@ def receive(self):
         self._set_stats(values_obj)
 
     def summarize(self, collectd, timestamp):
-        for type_ in self.internal_types:
+        for type_ in self.collectd_types:
             for values_obj in self.get_stats_by_progname(
                 type_.name, timestamp
             ):
@@ -70,14 +70,14 @@ def _set_stats(self, values):
             # process_token (which is roughly the pid plus a unique key
             # generated by the client plugin).   we also use a larger interval
             # for this value so that the process count changes more slowly
-            process_bucket = self.buckets[internal_types.process_internal.name]
+            process_bucket = self.buckets[collectd_types.process_internal.name]
             process_records = process_bucket.get_data(
                 timestamp, interval=interval * 5
             )
             process_records[
                 (hostname, progname, process_token)
             ] = values.build(
-                type=internal_types.process_internal.name, values=[1]
+                type=collectd_types.process_internal.name, values=[1]
             )
 
     def get_stats_by_progname(self, bucket_name, timestamp, agg_func=sum):

sqlalchemy_collectd/stream.py

@@ -2,34 +2,35 @@
 
 
 class StreamTranslator(object):
-    """Translates Value objects between internal types and "external"
+    """Translates Value objects from the "internal" to the "external"
     types.
 
-    The "external" types are the simple types that are in collectd
-    types.db, where we are looking at the "derive" type, which is a mapping
-    to a single protocol.VALUE_DERIVE value, and the "count" type, which is
-    a mapping to a zero-bounded protocol.VALUE_GAUGE type.  There is also
-    a "gauge" type however we are working with zero-bounded ranges.
-
-    We use these pre-defined types because SQLAlchemy-collectd does not
-    have an entry in the collectd types.db file and collectd doesn't give us
-    a straightforward way to extend on these types.
+    The mapping from the "internal" types in collectd_types.py to
+    the "external" types is to break each internal type into individual
+    values of type GAUGE or DERIVE.  The collectd "count" and "derive" types
+    defined in types.db serve as the destination for these values.
 
+    While the "internal" types are more efficient and suitable for the large
+    volume of messages sent by SQLAlchemy clients, as they are sent with a low
+    "interval" setting as well as a full set of messages per  process, the
+    "external" types are publicly available for other collectd services and as
+    the per-process messages have been aggregated into per-"program" messages
+    and are usually at a lower interval, there is less message volume.
 
     """
 
-    def __init__(self, *internal_types):
-        self.internal_types = internal_types
+    def __init__(self, *collectd_types):
+        self.collectd_types = collectd_types
 
-        self._type_by_name = {t.name: t for t in internal_types}
+        self._type_by_name = {t.name: t for t in collectd_types}
         self.external_types = {}
         self.external_type_to_internal = {}
         self._protocol_type_string_names = {
             protocol.VALUE_GAUGE: "count",
             protocol.VALUE_DERIVE: "derive",
         }
 
-        for internal_type in internal_types:
+        for internal_type in collectd_types:
             for name, value_type in zip(
                 internal_type.names, internal_type.types
             ):