Update monitorstack to use updated openstacksdk and es

This change updates the functions so that they work with the most up to date
OpenStack SDK, and adds an elasticsearch output format.

Change-Id: If46d3fb8e2b2e4aa5e21bf7da470945b05d216cf
Signed-off-by: Kevin Carter <kevin@cloudnull.com>

etc/openstack.ini

@@ -14,6 +14,13 @@ auth_url = https://127.0.0.1:5000/v3
 username = admin
 password = Secrete
 
+# NOTE(Cloudnull):
+#   If the system already has a clouds.yaml configuration file in place, monitorstack
+#   can use this config by default. Create the "cloud" section and set the cloud option.
+#   When using this section, no other OpenStack options are needed.
+# [cloud]
+# cloud = default
+
 [keystone]
 # NOTE(cloudnull):
 #  When using keystone V3 you will need the .*domain_name configuration options.
@@ -45,3 +52,13 @@ project_name = ironic
 user_domain_name = users
 project_domain_name = projects
 password = SuperSecrete
+
+[elasticsearch]
+# List of hosts. Note, items in this list are strings with the hostname or IP only.
+hosts = ['localhost']
+# Optional settings when using authenticated environments.
+# http_auth = ('user', 'secret')
+
+# Optional settings when using SSL.
+# scheme="https"
+# port=443

monitorstack/cli.py

@@ -83,7 +83,8 @@ VALID_OUTPUT_FORMATS = [
     'json',
     'line',
     'telegraf',
-    'rax-maas'
+    'rax-maas',
+    'elasticsearch'
 ]
 
 
@@ -96,6 +97,9 @@ VALID_OUTPUT_FORMATS = [
         ', '.join(VALID_OUTPUT_FORMATS)
     ),
 )
+@click.option('--config-file',
+              help='MonitorStack configuration file',
+              default='openstack.ini')
 @click.option('-v', '--verbose', is_flag=True, help='Enables verbose mode.')
 @pass_context
 def cli(*args, **kwargs):
@@ -122,7 +126,7 @@ def process_result(results, output_format, **kwargs):
 
     exit_code = 0
     for result in results:
-        output_formatter(result)
+        output_formatter(result, kwargs['config_file'])
         if result['exit_code'] != 0:
             exit_code = result['exit_code']
     else:

monitorstack/common/formatters.py

@@ -18,15 +18,17 @@ import time
 
 import click
 
+from monitorstack import utils
 
-def write_json(result):
+
+def write_json(result, config_file):
     """Output in raw JSON format."""
     output = json.dumps(result, indent=2)
     click.echo(output)
     return True
 
 
-def write_line(result):
+def write_line(result, config_file):
     """Output in line format."""
     for key, value in result['variables'].items():
         click.echo("{} {}".format(key, value))
@@ -102,7 +104,7 @@ def _telegraf_line_format(sets, quote=False):
     return ','.join(store).rstrip(',')
 
 
-def write_telegraf(result):
+def write_telegraf(result, config_file):
     """Output in telegraf format."""
     resultant = [result['measurement_name']]
     if 'meta' in result:
@@ -120,7 +122,7 @@ def write_telegraf(result):
     return True
 
 
-def write_rax_maas(result):
+def write_rax_maas(result, config_file):
     """Output in Rackspace Monitoring as a Service format."""
     status = ['status']
     if result['exit_code'] == 0:
@@ -142,3 +144,37 @@ def write_rax_maas(result):
         click.echo(' '.join(metric))
 
     return True
+
+
+def write_elasticsearch(result, config_file):
+    """Output in elasticsearch format."""
+    import datetime
+    from elasticsearch import Elasticsearch
+
+    config = utils.read_config(config_file=config_file)
+    if 'elasticsearch' in config:
+        elastcisearch_config = config['elasticsearch']
+        es = Elasticsearch(**elastcisearch_config)
+    else:
+        es = Elasticsearch()
+
+    doc = {
+        'author': 'openstack',
+        'text': result['message'],
+        'timestamp': datetime.datetime.now(),
+        'measurement_name': result['measurement_name'],
+        'meta': result['meta'],
+        'variables': result['variables'],
+        'status': result['exit_code']
+    }
+
+    res = es.index(
+        index="monitorstack-{}".format(
+            datetime.date.today().strftime("%Y-%m-%d")
+        ),
+        id=_current_time(),
+        doc_type='openstack-metrics',
+        body=doc
+    )
+
+    click.echo(res['result'])

monitorstack/plugins/os_block_pools_totals.py

@@ -25,7 +25,7 @@ COMMAND_NAME = 'os_block_pools_totals'
 
 @click.command(COMMAND_NAME, short_help=DOC)
 @click.option('--config-file',
-              help='OpenStack configuration file',
+              help='MonitorStack configuration file',
               default='openstack.ini')
 @pass_context
 def cli(ctx, config_file):
@@ -43,9 +43,19 @@ def cli(ctx, config_file):
         },
         'variables': {}
     }
-    config = utils.read_config(config_file=config_file)['cinder']
-    interface = config.pop('interface', 'internal')
-    _ost = ost.OpenStack(os_auth_args=config)
+    os_config = utils.read_config(
+        config_file=config_file,
+        no_config_fatal=False
+    )
+    service_config = os_config.get('cinder')
+    cloud_config = os_config.get('cloud')
+    if service_config:
+        interface = service_config.pop('interface', 'internal')
+        _ost = ost.OpenStack(os_auth_args=service_config)
+    else:
+        interface = 'internal'
+        _ost = ost.OpenStack(os_auth_args=cloud_config)
+
     try:
         variables = output['variables']
         total_capacity_gb = 0

monitorstack/plugins/os_block_pools_usage.py

@@ -25,7 +25,7 @@ COMMAND_NAME = 'os_block_pools_usage'
 
 @click.command(COMMAND_NAME, short_help=DOC)
 @click.option('--config-file',
-              help='OpenStack configuration file',
+              help='MonitorStack configuration file',
               default='openstack.ini')
 @pass_context
 def cli(ctx, config_file):
@@ -43,9 +43,19 @@ def cli(ctx, config_file):
         },
         'variables': {}
     }
-    config = utils.read_config(config_file=config_file)['cinder']
-    interface = config.pop('interface', 'internal')
-    _ost = ost.OpenStack(os_auth_args=config)
+    os_config = utils.read_config(
+        config_file=config_file,
+        no_config_fatal=False
+    )
+    service_config = os_config.get('cinder')
+    cloud_config = os_config.get('cloud')
+    if service_config:
+        interface = service_config.pop('interface', 'internal')
+        _ost = ost.OpenStack(os_auth_args=service_config)
+    else:
+        interface = 'internal'
+        _ost = ost.OpenStack(os_auth_args=cloud_config)
+
     try:
         variables = output['variables']
         for item in _ost.get_volume_pool_stats(interface=interface):

monitorstack/plugins/os_vm_quota_cores.py

@@ -25,7 +25,7 @@ COMMAND_NAME = 'os_vm_quota_cores'
 
 @click.command(COMMAND_NAME, short_help=DOC)
 @click.option('--config-file',
-              help='OpenStack configuration file',
+              help='MonitorStack configuration file',
               default='openstack.ini')
 @pass_context
 def cli(ctx, config_file):
@@ -43,9 +43,19 @@ def cli(ctx, config_file):
         },
         'variables': {}
     }
-    nova_config = utils.read_config(config_file=config_file)['nova']
-    interface = nova_config.pop('interface', 'internal')
-    _ost = ost.OpenStack(os_auth_args=nova_config)
+    os_config = utils.read_config(
+        config_file=config_file,
+        no_config_fatal=False
+    )
+    service_config = os_config.get('nova')
+    cloud_config = os_config.get('cloud')
+    if service_config:
+        interface = service_config.pop('interface', 'internal')
+        _ost = ost.OpenStack(os_auth_args=service_config)
+    else:
+        interface = 'internal'
+        _ost = ost.OpenStack(os_auth_args=cloud_config)
+
     try:
         variables = output['variables']
         for project in _ost.get_projects():

monitorstack/plugins/os_vm_quota_instance.py

@@ -25,7 +25,7 @@ COMMAND_NAME = 'os_vm_quota_instance'
 
 @click.command(COMMAND_NAME, short_help=DOC)
 @click.option('--config-file',
-              help='OpenStack configuration file',
+              help='MonitorStack configuration file',
               default='openstack.ini')
 @pass_context
 def cli(ctx, config_file):
@@ -43,9 +43,19 @@ def cli(ctx, config_file):
         },
         'variables': {}
     }
-    nova_config = utils.read_config(config_file=config_file)['nova']
-    interface = nova_config.pop('interface', 'internal')
-    _ost = ost.OpenStack(os_auth_args=nova_config)
+    os_config = utils.read_config(
+        config_file=config_file,
+        no_config_fatal=False
+    )
+    service_config = os_config.get('nova')
+    cloud_config = os_config.get('cloud')
+    if service_config:
+        interface = service_config.pop('interface', 'internal')
+        _ost = ost.OpenStack(os_auth_args=service_config)
+    else:
+        interface = 'internal'
+        _ost = ost.OpenStack(os_auth_args=cloud_config)
+
     try:
         variables = output['variables']
         for project in _ost.get_projects():

monitorstack/plugins/os_vm_quota_ram.py

@@ -25,7 +25,7 @@ COMMAND_NAME = 'os_vm_quota_ram'
 
 @click.command(COMMAND_NAME, short_help=DOC)
 @click.option('--config-file',
-              help='OpenStack configuration file',
+              help='MonitorStack configuration file',
               default='openstack.ini')
 @pass_context
 def cli(ctx, config_file):
@@ -43,9 +43,19 @@ def cli(ctx, config_file):
         },
         'variables': {}
     }
-    nova_config = utils.read_config(config_file=config_file)['nova']
-    interface = nova_config.pop('interface', 'internal')
-    _ost = ost.OpenStack(os_auth_args=nova_config)
+    os_config = utils.read_config(
+        config_file=config_file,
+        no_config_fatal=False
+    )
+    service_config = os_config.get('nova')
+    cloud_config = os_config.get('cloud')
+    if service_config:
+        interface = service_config.pop('interface', 'internal')
+        _ost = ost.OpenStack(os_auth_args=service_config)
+    else:
+        interface = 'internal'
+        _ost = ost.OpenStack(os_auth_args=cloud_config)
+
     try:
         variables = output['variables']
         for project in _ost.get_projects():

monitorstack/plugins/os_vm_used_cores.py

@@ -27,7 +27,7 @@ COMMAND_NAME = 'os_vm_used_cores'
 
 @click.command(COMMAND_NAME, short_help=DOC)
 @click.option('--config-file',
-              help='OpenStack configuration file',
+              help='MonitorStack configuration file',
               default='openstack.ini')
 @pass_context
 def cli(ctx, config_file):
@@ -45,10 +45,18 @@ def cli(ctx, config_file):
         },
         'variables': {}
     }
+    os_config = utils.read_config(
+        config_file=config_file,
+        no_config_fatal=False
+    )
+    service_config = os_config.get('nova')
+    cloud_config = os_config.get('cloud')
+    if service_config:
+        _ost = ost.OpenStack(os_auth_args=service_config)
+    else:
+        _ost = ost.OpenStack(os_auth_args=cloud_config)
 
     used_collection = collections.Counter()
-    nova_config = utils.read_config(config_file=config_file)['nova']
-    _ost = ost.OpenStack(os_auth_args=nova_config)
     try:
         flavors = _ost.get_flavors()
         variables = output['variables']

monitorstack/plugins/os_vm_used_disk.py

@@ -27,7 +27,7 @@ COMMAND_NAME = 'os_vm_used_disk'
 
 @click.command(COMMAND_NAME, short_help=DOC)
 @click.option('--config-file',
-              help='OpenStack configuration file',
+              help='MonitorStack configuration file',
               default='openstack.ini')
 @pass_context
 def cli(ctx, config_file):
@@ -45,10 +45,18 @@ def cli(ctx, config_file):
         },
         'variables': {}
     }
+    os_config = utils.read_config(
+        config_file=config_file,
+        no_config_fatal=False
+    )
+    service_config = os_config.get('nova')
+    cloud_config = os_config.get('cloud')
+    if service_config:
+        _ost = ost.OpenStack(os_auth_args=service_config)
+    else:
+        _ost = ost.OpenStack(os_auth_args=cloud_config)
 
     used_collection = collections.Counter()
-    nova_config = utils.read_config(config_file=config_file)['nova']
-    _ost = ost.OpenStack(os_auth_args=nova_config)
     try:
         flavors = _ost.get_flavors()
         variables = output['variables']

monitorstack/plugins/os_vm_used_instance.py

@@ -27,7 +27,7 @@ COMMAND_NAME = 'os_vm_used_instance'
 
 @click.command(COMMAND_NAME, short_help=DOC)
 @click.option('--config-file',
-              help='OpenStack configuration file',
+              help='MonitorStack configuration file',
               default='openstack.ini')
 @pass_context
 def cli(ctx, config_file):
@@ -45,10 +45,18 @@ def cli(ctx, config_file):
         },
         'variables': {}
     }
+    os_config = utils.read_config(
+        config_file=config_file,
+        no_config_fatal=False
+    )
+    service_config = os_config.get('nova')
+    cloud_config = os_config.get('cloud')
+    if service_config:
+        _ost = ost.OpenStack(os_auth_args=service_config)
+    else:
+        _ost = ost.OpenStack(os_auth_args=cloud_config)
 
     used_collection = collections.Counter()
-    nova_config = utils.read_config(config_file=config_file)['nova']
-    _ost = ost.OpenStack(os_auth_args=nova_config)
     try:
         variables = output['variables']
         for used in _ost.get_consumer_usage():

monitorstack/plugins/os_vm_used_ram.py

@@ -27,11 +27,12 @@ COMMAND_NAME = 'os_vm_used_ram'
 
 @click.command(COMMAND_NAME, short_help=DOC)
 @click.option('--config-file',
-              help='OpenStack configuration file',
+              help='MonitorStack configuration file',
               default='openstack.ini')
 @pass_context
 def cli(ctx, config_file):
     """Get nova used ram."""
+    print(ctx.__dict__)
     setattr(cli, '__doc__', DOC)
 
     # Lower level import because we only want to load this module
@@ -45,10 +46,18 @@ def cli(ctx, config_file):
         },
         'variables': {}
     }
+    os_config = utils.read_config(
+        config_file=config_file,
+        no_config_fatal=False
+    )
+    service_config = os_config.get('nova')
+    cloud_config = os_config.get('cloud')
+    if service_config:
+        _ost = ost.OpenStack(os_auth_args=service_config)
+    else:
+        _ost = ost.OpenStack(os_auth_args=cloud_config)
 
     used_collection = collections.Counter()
-    nova_config = utils.read_config(config_file=config_file)['nova']
-    _ost = ost.OpenStack(os_auth_args=nova_config)
     try:
         flavors = _ost.get_flavors()
         variables = output['variables']

monitorstack/utils/__init__.py

@@ -139,15 +139,18 @@ class LocalCache(object):
         self.open_cache.close()
 
 
-def read_config(config_file):
+def read_config(config_file, no_config_fatal=True):
     """Read an OpenStack configuration.
 
     :param config_file: path to configuration file.
+    :param no_config_fatal: Boolean
     :type config_file: str
     """
     cfg = os.path.abspath(os.path.expanduser(config_file))
-    if not os.path.isfile(cfg):
+    if not os.path.isfile(cfg) and no_config_fatal:
         raise IOError('Config file "{}" was not found'.format(cfg))
+    elif not os.path.isfile(cfg) and not no_config_fatal:
+        return dict()
 
     parser = ConfigParser.ConfigParser()
     parser.optionxform = str

monitorstack/utils/os_utils.py

@@ -24,6 +24,7 @@ try:
 except ImportError:  # pragma: no cover
         raise SystemExit('No urlparse module was found.')
 try:
+    import openstack
     from openstack import connection as os_conn  # pragma: no cover
 except ImportError as e:  # pragma: no cover
     raise SystemExit('OpenStack plugins require access to the OpenStackSDK.'
@@ -43,6 +44,9 @@ class OpenStack(object):
         :type os_auth_args: dict
         """
         self.os_auth_args = os_auth_args
+        self.verify = False
+
+        if self.os_auth_args:
             insecure = bool(strtobool(self.os_auth_args.get('insecure', 'False')))
             self.verify = insecure is False
 
@@ -52,7 +56,12 @@ class OpenStack(object):
 
         :returns: object
         """
+        if self.os_auth_args and 'cloud' in self.os_auth_args:
+            return openstack.connect(**self.os_auth_args)
+        elif self.os_auth_args:
             return os_conn.Connection(verify=self.verify, **self.os_auth_args)
+        else:
+            return openstack.connect(cloud='default')
 
     def _session_req(self, path, service_type, interface='internal'):
         """Return compute resource limits for a project.
@@ -67,7 +76,7 @@ class OpenStack(object):
             interface=interface,
             service_type=service_type
         )
-        sess_url = urlparse.urljoin(endpoint_url, path)
+        sess_url = endpoint_url + path
         return self.conn.session.get(sess_url).json()
 
     def get_consumer_usage(self, servers=None, marker=None, limit=512):

requirements.txt

@@ -1,5 +1,6 @@
 click
 diskcache
+elasticsearch>=6.0.0,<7.0.0
 openstacksdk>=0.9.14
 pymemcache>=1.2.9,!=1.3.0  # Apache 2.0 License
 psutil>=5.2.0