Add a mechanism to use the oslo libraries

Implement a mechanism for masakarimonitors to use oslo.log,
oslo.config and oslo.service.

Change-Id: I3e933d339b0998468464c6a804fff623f37afd55

etc/masakarimonitors/README-masakarimonitors.conf.txt

@@ -0,0 +1,4 @@
+To generate the sample masakarimonitors.conf file, run the following command from the top
+level of the masakari directory:
+
+    tox -egenconfig

etc/masakarimonitors/masakarimonitors-config-generator.conf

@@ -0,0 +1,6 @@
+[DEFAULT]
+output_file = etc/masakarimonitors/masakarimonitors.conf.sample
+wrap_width = 80
+namespace = masakarimonitors.conf
+namespace = oslo.log
+namespace = oslo.middleware

masakarimonitors/common/config.py

@@ -0,0 +1,22 @@
+# Copyright(c) 2016 Nippon Telegraph and Telephone Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+from oslo_middleware import cors
+
+
+def set_middleware_defaults():
+    """Update default configuration options for oslo.middleware."""
+    # CORS Defaults
+    cfg.set_defaults(cors.CORS_OPTS)

masakarimonitors/conf/__init__.py

@@ -0,0 +1,22 @@
+# Copyright(c) 2016 Nippon Telegraph and Telephone Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from oslo_config import cfg
+
+from masakarimonitors.conf import base
+from masakarimonitors.conf import service
+
+CONF = cfg.CONF
+
+base.register_opts(CONF)
+service.register_opts(CONF)

masakarimonitors/conf/base.py

@@ -0,0 +1,54 @@
+# Copyright(c) 2016 Nippon Telegraph and Telephone Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+
+base_options = [
+    cfg.StrOpt(
+        'tempdir',
+        help='Explicitly specify the temporary working directory.'),
+    cfg.BoolOpt(
+        'monkey_patch',
+        default=False,
+        help="""
+Determine if monkey patching should be applied.
+
+Related options:
+
+  * ``monkey_patch_modules``: This must have values set for this option to have
+  any effect
+"""),
+    cfg.ListOpt(
+        'monkey_patch_modules',
+        default=['masakarimonitors.cmd'],
+        help="""
+List of modules/decorators to monkey patch.
+
+This option allows you to patch a decorator for all functions in specified
+modules.
+
+Related options:
+
+  * ``monkey_patch``: This must be set to ``True`` for this option to
+    have any effect
+"""),
+]
+
+
+def register_opts(conf):
+    conf.register_opts(base_options)
+
+
+def list_opts():
+    return {'DEFAULT': base_options}

masakarimonitors/conf/opts.py

@@ -0,0 +1,86 @@
+# Copyright(c) 2016 Nippon Telegraph and Telephone Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This is the single point of entry to generate the sample configuration
+file for Masakari Monitors. It collects all the necessary info from the
+other modules in this package. It is assumed that:
+
+* every other module in this package has a 'list_opts' function which
+  return a dict where
+  * the keys are strings which are the group names
+  * the value of each key is a list of config options for that group
+* the masakari.conf package doesn't have further packages with config options
+* this module is only used in the context of sample file generation
+"""
+
+import collections
+import importlib
+import os
+import pkgutil
+
+LIST_OPTS_FUNC_NAME = "list_opts"
+
+
+def _tupleize(dct):
+    """Take the dict of options and convert to the 2-tuple format."""
+    return [(key, val) for key, val in dct.items()]
+
+
+def list_opts():
+    opts = collections.defaultdict(list)
+    module_names = _list_module_names()
+    imported_modules = _import_modules(module_names)
+    _append_config_options(imported_modules, opts)
+    return _tupleize(opts)
+
+
+def _list_module_names():
+    module_names = []
+    package_path = os.path.dirname(os.path.abspath(__file__))
+    for _, modname, ispkg in pkgutil.iter_modules(path=[package_path]):
+        if modname == "opts" or ispkg:
+            continue
+        else:
+            module_names.append(modname)
+    return module_names
+
+
+def _import_modules(module_names):
+    imported_modules = []
+    for modname in module_names:
+        mod = importlib.import_module("masakarimonitors.conf." + modname)
+        if not hasattr(mod, LIST_OPTS_FUNC_NAME):
+            msg = "The module 'masakarimonitors.conf.%s' should have a '%s' "\
+                  "function which returns the config options." % \
+                  (modname, LIST_OPTS_FUNC_NAME)
+            raise Exception(msg)
+        else:
+            imported_modules.append(mod)
+    return imported_modules
+
+
+def _process_old_opts(configs):
+    """Convert old-style 2-tuple configs to dicts."""
+    if isinstance(configs, tuple):
+        configs = [configs]
+    return {label: options for label, options in configs}
+
+
+def _append_config_options(imported_modules, config_options):
+    for mod in imported_modules:
+        configs = mod.list_opts()
+        if not isinstance(configs, dict):
+            configs = _process_old_opts(configs)
+        for key, val in configs.items():
+            config_options[key].extend(val)

masakarimonitors/conf/service.py

@@ -0,0 +1,41 @@
+# Copyright(c) 2016 Nippon Telegraph and Telephone Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import socket
+
+from oslo_config import cfg
+
+service_opts = [
+    cfg.StrOpt('host',
+               default=socket.gethostname(),
+               help='''
+Hostname, FQDN or IP address of this host. Must be valid within AMQP key.
+
+Possible values:
+
+* String with hostname, FQDN or IP address. Default is hostname of this host.
+'''),
+    cfg.StrOpt('instancemonitor_manager',
+               default='masakarimonitors.instancemonitor.instance'
+                       '.InstancemonitorManager',
+               help='Full class name for the Manager for instancemonitor.'),
+    ]
+
+
+def register_opts(conf):
+    conf.register_opts(service_opts)
+
+
+def list_opts():
+    return {'DEFAULT': service_opts}

masakarimonitors/config.py

@@ -0,0 +1,32 @@
+# Copyright(c) 2016 Nippon Telegraph and Telephone Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from oslo_log import log
+
+import masakarimonitors.conf
+from masakarimonitors import version
+
+
+CONF = masakarimonitors.conf.CONF
+
+
+def parse_args(argv, default_config_files=None):
+    log.register_options(CONF)
+    # We use the oslo.log default log levels which includes suds=INFO
+    # and add only the extra levels that Masakari needs
+    log.set_defaults(default_log_levels=log.get_default_log_levels())
+
+    CONF(argv[1:],
+         project='masakarimonitors',
+         version=version.version_string(),
+         default_config_files=default_config_files)

masakarimonitors/i18n.py

@@ -0,0 +1,39 @@
+# Copyright 2016 NTT DATA
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import oslo_i18n
+
+DOMAIN = 'masakarimonitors'
+
+_translators = oslo_i18n.TranslatorFactory(domain=DOMAIN)
+
+# The primary translation function using the well-known name "_"
+_ = _translators.primary
+
+# Translators for log levels.
+#
+# The abbreviated names are meant to reflect the usual use of a short
+# name like '_'. The "L" is for "log" and the other letter comes from
+# the level.
+_LI = _translators.log_info
+_LW = _translators.log_warning
+_LE = _translators.log_error
+_LC = _translators.log_critical
+
+
+def translate(value, user_locale):
+    return oslo_i18n.translate(value, user_locale)
+
+
+def get_available_languages():
+    return oslo_i18n.get_available_languages(DOMAIN)

masakarimonitors/service.py

@@ -0,0 +1,140 @@
+# Copyright(c) 2016 Nippon Telegraph and Telephone Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generic Node base class for all workers that run on hosts."""
+
+import os
+import sys
+
+from oslo_log import log as logging
+from oslo_service import service
+from oslo_utils import importutils
+
+import masakarimonitors.conf
+from masakarimonitors.i18n import _
+from masakarimonitors.i18n import _LE
+from masakarimonitors.i18n import _LI
+from masakarimonitors import utils
+
+
+LOG = logging.getLogger(__name__)
+
+CONF = masakarimonitors.conf.CONF
+
+
+class Service(service.Service):
+    """Service object for binaries running on hosts.
+
+       A service takes a manager.
+    """
+
+    def __init__(self, host, binary, manager):
+        super(Service, self).__init__()
+        self.host = host
+        self.binary = binary
+        self.manager_class_name = manager
+        manager_class = importutils.import_class(self.manager_class_name)
+        self.manager = manager_class(host=self.host)
+
+    def __repr__(self):
+        return "<%(cls_name)s: host=%(host)s, binary=%(binary)s, " \
+               "manager_class_name=%(manager)s>" %\
+               {
+                   'cls_name': self.__class__.__name__,
+                   'host': self.host,
+                   'binary': self.binary,
+                   'manager': self.manager_class_name
+               }
+
+    def start(self):
+        LOG.info(_LI('Starting %s'), self.binary)
+        self.basic_config_check()
+        self.manager.init_host()
+        self.manager.main()
+
+    def __getattr__(self, key):
+        manager = self.__dict__.get('manager', None)
+        return getattr(manager, key)
+
+    @classmethod
+    def create(cls, host=None, binary=None, manager=None):
+        """Instantiates class and passes back application object.
+
+        :param host: defaults to CONF.host
+        :param binary: defaults to basename of executable
+        :param manager: defaults to CONF.<Latter part of binary>_manager
+
+        """
+        if not host:
+            host = CONF.host
+        if not binary:
+            binary = os.path.basename(sys.argv[0])
+
+        if not manager:
+            manager_cls = ('%s_manager' %
+                           binary.rpartition('masakarimonitors-')[2])
+            manager = CONF.get(manager_cls, None)
+
+        service_obj = cls(host, binary, manager)
+
+        return service_obj
+
+    def kill(self):
+        """Destroy the service object in the datastore.
+
+        NOTE: Although this method is not used anywhere else than tests, it is
+        convenient to have it here, so the tests might easily and in clean way
+        stop and remove the service_ref.
+
+        """
+        self.stop()
+
+    def stop(self):
+        LOG.info(_LI('Stopping %s'), self.binary)
+        super(Service, self).stop()
+
+    def basic_config_check(self):
+        """Perform basic config checks before starting processing."""
+        # Make sure the tempdir exists and is writable
+        try:
+            with utils.tempdir():
+                pass
+        except Exception as e:
+            LOG.error(_LE('Temporary directory is invalid: %s'), e)
+            sys.exit(1)
+
+    def reset(self):
+        self.manager.reset()
+
+
+def process_launcher():
+    return service.ProcessLauncher(CONF)
+
+
+# NOTE: the global launcher is to maintain the existing
+#       functionality of calling service.serve +
+#       service.wait
+_launcher = None
+
+
+def serve(server, workers=None):
+    global _launcher
+    if _launcher:
+        raise RuntimeError(_('serve() can only be called once'))
+
+    _launcher = service.launch(CONF, server, workers=workers)
+
+
+def wait():
+    _launcher.wait()

masakarimonitors/utils.py

@@ -0,0 +1,93 @@
+# Copyright(c) 2016 Nippon Telegraph and Telephone Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities and helper functions."""
+
+import contextlib
+import inspect
+import pyclbr
+import shutil
+import sys
+import tempfile
+
+from oslo_log import log as logging
+from oslo_utils import importutils
+import six
+
+import masakarimonitors.conf
+from masakarimonitors.i18n import _LE
+
+
+CONF = masakarimonitors.conf.CONF
+
+LOG = logging.getLogger(__name__)
+
+
+def monkey_patch():
+    """monkey_patch function.
+
+    If the CONF.monkey_patch set as True,
+    this function patches a decorator
+    for all functions in specified modules.
+    You can set decorators for each modules
+    using CONF.monkey_patch_modules.
+    The format is "Module path:Decorator function".
+    name - name of the function
+    function - object of the function
+    """
+    # If CONF.monkey_patch is not True, this function do nothing.
+    if not CONF.monkey_patch:
+        return
+    if six.PY2:
+        is_method = inspect.ismethod
+    else:
+        def is_method(obj):
+            # Unbound methods became regular functions on Python 3
+            return inspect.ismethod(obj) or inspect.isfunction(obj)
+    # Get list of modules and decorators
+    for module_and_decorator in CONF.monkey_patch_modules:
+        module, decorator_name = module_and_decorator.split(':')
+        # import decorator function
+        decorator = importutils.import_class(decorator_name)
+        __import__(module)
+        # Retrieve module information using pyclbr
+        module_data = pyclbr.readmodule_ex(module)
+        for key, value in module_data.items():
+            # set the decorator for the class methods
+            if isinstance(value, pyclbr.Class):
+                clz = importutils.import_class("%s.%s" % (module, key))
+                for method, func in inspect.getmembers(clz, is_method):
+                    setattr(clz, method,
+                            decorator("%s.%s.%s" % (module, key,
+                                                    method), func))
+            # set the decorator for the function
+            if isinstance(value, pyclbr.Function):
+                func = importutils.import_class("%s.%s" % (module, key))
+                setattr(sys.modules[module], key,
+                        decorator("%s.%s" % (module, key), func))
+
+
+@contextlib.contextmanager
+def tempdir(**kwargs):
+    argdict = kwargs.copy()
+    if 'dir' not in argdict:
+        argdict['dir'] = CONF.tempdir
+    tmpdir = tempfile.mkdtemp(**argdict)
+    try:
+        yield tmpdir
+    finally:
+        try:
+            shutil.rmtree(tmpdir)
+        except OSError as e:
+            LOG.error(_LE('Could not remove tmpdir: %s'), e)

masakarimonitors/version.py

@@ -0,0 +1,88 @@
+# Copyright(c) 2016 Nippon Telegraph and Telephone Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pbr import version as pbr_version
+
+from masakarimonitors.i18n import _LE
+
+MONITORS_VENDOR = "OpenStack Foundation"
+MONITORS_PRODUCT = "OpenStack Masakari Monitors"
+MONITORS_PACKAGE = None  # OS distro package version suffix
+
+loaded = False
+version_info = pbr_version.VersionInfo('masakarimonitors')
+version_string = version_info.version_string
+
+
+def _load_config():
+    # Don't load in global context, since we can't assume
+    # these modules are accessible when distutils uses
+    # this module
+    from six.moves import configparser
+
+    from oslo_config import cfg
+
+    from oslo_log import log as logging
+
+    global loaded, MONITORS_VENDOR, MONITORS_PRODUCT, MONITORS_PACKAGE
+    if loaded:
+        return
+
+    loaded = True
+
+    cfgfile = cfg.CONF.find_file("release")
+    if cfgfile is None:
+        return
+
+    try:
+        cfg = configparser.RawConfigParser()
+        cfg.read(cfgfile)
+
+        if cfg.has_option("Masakarimonitors", "vendor"):
+            MONITORS_VENDOR = cfg.get("Masakarimonitors", "vendor")
+
+        if cfg.has_option("Masakarimonitors", "product"):
+            MONITORS_PRODUCT = cfg.get("Masakarimonitors", "product")
+
+        if cfg.has_option("Masakarimonitors", "package"):
+            MONITORS_PACKAGE = cfg.get("Masakarimonitors", "package")
+    except Exception as ex:
+        LOG = logging.getLogger(__name__)
+        LOG.error(_LE("Failed to load %(cfgfile)s: %(ex)s"),
+                  {'cfgfile': cfgfile, 'ex': ex})
+
+
+def vendor_string():
+    _load_config()
+
+    return MONITORS_VENDOR
+
+
+def product_string():
+    _load_config()
+
+    return MONITORS_PRODUCT
+
+
+def package_string():
+    _load_config()
+
+    return MONITORS_PACKAGE
+
+
+def version_string_with_package():
+    if package_string() is None:
+        return version_info.version_string()
+    else:
+        return "%s-%s" % (version_info.version_string(), package_string())

requirements.txt

@@ -2,4 +2,10 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
+oslo.config>=3.10.0 # Apache-2.0
+oslo.i18n>=2.1.0 # Apache-2.0
+oslo.log>=1.14.0 # Apache-2.0
+oslo.middleware>=3.0.0 # Apache-2.0
+oslo.service>=1.10.0 # Apache-2.0
+oslo.utils>=3.11.0 # Apache-2.0
 pbr>=1.6 # Apache-2.0

setup.cfg

@@ -23,6 +23,16 @@ classifier =
 packages =
     masakarimonitors
 
+[entry_points]
+oslo.config.opts =
+    masakarimonitors.conf = masakarimonitors.conf.opts:list_opts
+
+oslo.config.opts.defaults =
+    masakarimonitors.instancemonitor = masakarimonitors.common.config:set_middleware_defaults
+
+console_scripts =
+    masakari-instancemonitor = masakarimonitors.cmd.instancemonitor:main
+
 [build_sphinx]
 source-dir = doc/source
 build-dir = doc/build
@@ -48,4 +58,4 @@ output_file = masakarimonitors/locale/masakarimonitors.pot
 [build_releasenotes]
 all_files = 1
 build-dir = releasenotes/build
-source-dir = releasenotes/source
+source-dir = releasenotes/source

tox.ini

@@ -12,6 +12,9 @@ setenv =
 deps = -r{toxinidir}/test-requirements.txt
 commands = python setup.py test --slowest --testr-args='{posargs}'
 
+[testenv:genconfig]
+commands = oslo-config-generator --config-file=etc/masakarimonitors/masakarimonitors-config-generator.conf
+
 [testenv:pep8]
 commands = flake8 {posargs}