Use oslo-config-generator

It is common for OpenStack projects to use oslo-config-generator to generate
a sample config file which contains all available configuration options.

Work items:

* removed etc/monasca/log-api.conf (it is autogenerated)
* keep all config in one place to mimic the configuration file
* added configuration files to documentation

Story: 2000970
Task: 4121

Change-Id: I8777ed6cd38739e87b19be248b9c120e84626ad1

.gitignore

@@ -14,7 +14,7 @@ ChangeLog
 MANIFEST
 AUTHORS
 monasca-log-api.log
-
+etc/monasca/log-api.conf.sample
 
 *.swp
 *.iml

config-generator/README.md

@@ -0,0 +1,7 @@
+# config-generator
+
+To generate sample configuration execute
+
+```sh
+tox -e genconfig
+```

config-generator/monasca-log-api.conf

@@ -0,0 +1,7 @@
+[DEFAULT]
+output_file = etc/monasca/log-api.conf.sample
+wrap_width = 79
+format = ini
+summarize = True
+namespace = monasca_log_api
+namespace = oslo.log

devstack/plugin.sh

@@ -184,17 +184,28 @@ function configure_monasca_log_api {
         # ensure fresh installation of configuration files
         rm -rf $MONASCA_LOG_API_CONF $MONASCA_LOG_API_PASTE $MONASCA_LOG_API_LOGGING_CONF
 
-        if [[ "$MONASCA_LOG_API_CONF_DIR" != "$MONASCA_LOG_API_DIR/etc/monasca" ]]; then
-            install -m 600 $MONASCA_LOG_API_DIR/etc/monasca/log-api.conf $MONASCA_LOG_API_CONF
-            install -m 600 $MONASCA_LOG_API_DIR/etc/monasca/log-api-paste.ini $MONASCA_LOG_API_PASTE
-            install -m 600 $MONASCA_LOG_API_DIR/etc/monasca/log-api-logging.conf $MONASCA_LOG_API_LOGGING_CONF
-        fi
+        $MONASCA_LOG_API_BIN_DIR/oslo-config-generator \
+            --config-file $MONASCA_LOG_API_DIR/config-generator/monasca-log-api.conf \
+            --output-file /tmp/log-api.conf
+
+        install -m 600 /tmp/log-api.conf $MONASCA_LOG_API_CONF && rm -rf /tmp/log-api.conf
+        install -m 600 $MONASCA_LOG_API_DIR/etc/monasca/log-api-paste.ini $MONASCA_LOG_API_PASTE
+        install -m 600 $MONASCA_LOG_API_DIR/etc/monasca/log-api-logging.conf $MONASCA_LOG_API_LOGGING_CONF
 
         # configure log-api.conf
         iniset "$MONASCA_LOG_API_CONF" DEFAULT log_config_append $MONASCA_LOG_API_LOGGING_CONF
         iniset "$MONASCA_LOG_API_CONF" service region $REGION_NAME
+
         iniset "$MONASCA_LOG_API_CONF" log_publisher kafka_url $KAFKA_SERVICE_HOST:$KAFKA_SERVICE_PORT
+        iniset "$MONASCA_LOG_API_CONF" log_publisher topics log
+
         iniset "$MONASCA_LOG_API_CONF" kafka_healthcheck kafka_url $KAFKA_SERVICE_HOST:$KAFKA_SERVICE_PORT
+        iniset "$MONASCA_LOG_API_CONF" kafka_healthcheck kafka_topics log
+
+        iniset "$MONASCA_LOG_API_CONF" roles_middleware path "/v2.0/log,/v3.0/logs"
+        iniset "$MONASCA_LOG_API_CONF" roles_middleware default_roles monasca-user
+        iniset "$MONASCA_LOG_API_CONF" roles_middleware agent_roles monasca-agent
+        iniset "$MONASCA_LOG_API_CONF" roles_middleware delegate_roles admin
 
         # configure keystone middleware
         configure_auth_token_middleware "$MONASCA_LOG_API_CONF" "admin" $MONASCA_LOG_API_CACHE_DIR

doc/source/.gitignore

@@ -0,0 +1 @@
+_static/*.conf.sample

doc/source/conf.py

@@ -37,9 +37,8 @@ extensions = [
     # 'sphinx.ext.autodoc' causes gate failures, enable as soon as sorted out
     # TODO(trebskit)
     'sphinx.ext.viewcode',
-    # TODO(trebskit) enable as soon as we get configgen in place
-    # 'oslo_config.sphinxconfiggen'
-    # 'oslo_config.sphinxext',
+    'oslo_config.sphinxconfiggen',
+    'oslo_config.sphinxext',
     'openstackdocstheme',
 ]
 
@@ -53,6 +52,11 @@ bug_tag = u'doc'
 copyright = u'2014-present, OpenStack Foundation'
 author = u'OpenStack Foundation'
 
+# sample config
+config_generator_config_file = [
+    ('config-generator/monasca-log-api.conf', '_static/log-api')
+]
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
@@ -125,7 +129,7 @@ html_theme = 'openstackdocs'
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-# html_static_path = []
+html_static_path = ['_static']
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied

doc/source/configuration/index.rst

@@ -12,7 +12,8 @@ monasca-log-api uses the following configuration files
 for its various services.
 
 .. toctree::
-   :glob:
    :maxdepth: 1
 
-   *
+   configuring
+   monasca_log_api
+   sample

doc/source/configuration/monasca_log_api.rst

@@ -0,0 +1,8 @@
+.. _monasca-log-api.conf:
+
+--------------------
+monasca-log-api.conf
+--------------------
+
+.. show-options::
+   :config-file: config-generator/monasca-log-api.conf

doc/source/configuration/sample.rst

@@ -0,0 +1,37 @@
+.. _sample-configuration:
+
+====================================
+monasca-log-api sample configuration
+====================================
+
+The following are sample configuration files for monasca-log-api and
+utilities. These are generated from code
+(apart from logging conf and paste conf) and reflect the current state
+of code in the monasca-log-api repository.
+
+
+Sample configuration for monasca-log-api
+----------------------------------------
+
+This sample configuration can also be viewed in `log-api.conf.sample
+<../_static/log-api.conf.sample>`_.
+
+.. literalinclude:: ../_static/log-api.conf.sample
+
+
+Sample configuration for logging
+--------------------------------
+
+This sample configuration can also be viewed in `log-api-logging.conf
+<https://github.com/openstack/monasca-log-api/blob/master/etc/monasca/log-api-logging.conf>`_.
+
+.. literalinclude:: ../../../etc/monasca/log-api-logging.conf
+
+
+Sample configuration for paste
+------------------------------
+
+This sample configuration can also be viewed in `log-api-paste.ini
+<https://github.com/openstack/monasca-log-api/blob/master/etc/monasca/log-api-paste.ini>`_.
+
+.. literalinclude:: ../../../etc/monasca/log-api-paste.ini

etc/monasca/log-api.conf

@@ -1,56 +0,0 @@
-#
-# Copyright 2016 FUJITSU LIMITED
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# note(trebskit) all URLs points to default monasca-log-api devstack setup
-
-[DEFAULT]
-log_config_append=/etc/monasca/log-api-logging.conf
-
-[monitoring]
-statsd_host = 127.0.0.1
-statsd_port = 8125
-statsd_buffer = 50
-
-[service]
-region = region-one
-max_log_size = 1048576
-
-[log_publisher]
-topics = log
-kafka_url = 192.168.10.6:9092
-max_message_size = 1048576
-
-[keystone_authtoken]
-auth_uri = http://127.0.0.1:5000
-auth_url = http://127.0.0.1:5000
-identity_uri = http://127.0.0.1:35357
-admin_user = admin
-admin_password = secretadmin
-admin_tenant_name = admin
-insecure = True
-token_cache_time = 300
-memcached_servers = 127.0.0.1:11211
-
-[kafka_healthcheck]
-kafka_url = 192.168.10.6:9092
-kafka_topics = log
-
-[roles_middleware]
-path = /v2.0/log,/v3.0/logs
-default_roles = user,domainuser,domainadmin,monasca-user
-agent_roles = monasca-agent
-delegate_roles = admin

monasca_log_api/app/api.py

@@ -20,21 +20,15 @@ of monasca-log-api
 import six
 
 import falcon
-from oslo_config import cfg
 from oslo_log import log
 
 from monasca_log_api.api.core import request
+from monasca_log_api import config
 from monasca_log_api.reference.common import error_handlers
 from monasca_log_api.reference import healthchecks
 from monasca_log_api.reference.v2 import logs as v2_logs
 from monasca_log_api.reference.v3 import logs as v3_logs
 from monasca_log_api.reference import versions
-from monasca_log_api import version
-
-LOG = log.getLogger(__name__)
-CONF = cfg.CONF
-
-_CONF_LOADED = False
 
 
 def error_trap(app_name):
@@ -48,7 +42,8 @@ def error_trap(app_name):
             try:
                 return func(*args, **kwargs)
             except Exception:
-                LOG.exception('Failed to load application \'%s\'', app_name)
+                logger = log.getLogger(__name__)
+                logger.exception('Failed to load application \'%s\'', app_name)
                 raise
 
         return _inner_wrapper
@@ -61,34 +56,9 @@ def singleton_config(func):
 
     @six.wraps(singleton_config)
     def _wrapper(global_config, **local_conf):
-        _load_config()
+        config.parse_args()
         return func(global_config, **local_conf)
 
-    def _load_config():
-        global _CONF_LOADED
-        if _CONF_LOADED:
-            LOG.debug('Configuration has been already loaded')
-            return
-
-        log.set_defaults()
-        log.register_options(CONF)
-
-        CONF(args=[],
-             # NOTE(trebskit) this disables any oslo.cfg CLI
-             # opts as gunicorn has some trouble with them
-             # i.e. gunicorn's argparse clashes with the one
-             # defined inside oslo.cfg
-             prog='log-api',
-             project='monasca',
-             version=version.version_str,
-             description='REST-ful API to collect log files')
-
-        log.setup(CONF,
-                  product_name='monasca-log-api',
-                  version=version.version_str)
-
-        _CONF_LOADED = True
-
     return _wrapper
 
 

monasca_log_api/conf/__init__.py

@@ -0,0 +1,84 @@
+# Copyright 2017 FUJITSU LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import pkgutil
+
+from oslo_config import cfg
+from oslo_log import log
+from oslo_utils import importutils
+
+CONF = cfg.CONF
+LOG = log.getLogger(__name__)
+
+
+def load_conf_modules():
+    """Loads all modules that contain configuration
+
+    Method iterates over modules of :py:module:`monasca_log_api.conf`
+    and imports only those that contain following methods:
+
+    - list_opts (required by oslo_config.genconfig)
+    - register_opts (required by :py:currentmodule:)
+
+    """
+    imported_modules = []
+    for modname in _list_module_names():
+        mod = importutils.import_module('monasca_log_api.conf.' + modname)
+        required_funcs = ['register_opts', 'list_opts']
+        for func in required_funcs:
+            if not hasattr(mod, func):
+                msg = ("The module 'monasca_log_api.conf.%s' should have a"
+                       " '%s' function which returns"
+                       " the config options."
+                       % (modname, func))
+                LOG.warning(msg)
+            else:
+                imported_modules.append(mod)
+
+    LOG.debug('Found %d modules that contain configuration',
+              len(imported_modules))
+
+    return imported_modules
+
+
+def _list_module_names():
+    module_names = []
+    package_path = os.path.dirname(os.path.abspath(__file__))
+    for _, modname, ispkg in pkgutil.iter_modules(path=[package_path]):
+        if not (modname == "opts" and ispkg):
+            module_names.append(modname)
+    return module_names
+
+
+def register_opts():
+    """Registers all conf modules opts
+
+    This method allows different modules to register
+    opts according to their needs.
+
+    """
+    for mod in load_conf_modules():
+        mod.register_opts(CONF)
+
+
+def list_opts():
+    """Lists all conf modules opts.
+
+    Goes through all conf modules and yields their opts
+
+    """
+    for mod in load_conf_modules():
+        mod_opts = mod.list_opts()
+        yield mod_opts[0], mod_opts[1]

monasca_log_api/conf/healthcheck.py

@@ -0,0 +1,36 @@
+# Copyright 2017 FUJITSU LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+
+kafka_check_opts = [
+    cfg.StrOpt('kafka_url',
+               required=True,
+               help='Url to kafka server'),
+    cfg.ListOpt('kafka_topics',
+                required=True,
+                default=['logs'],
+                help='Verify existence of configured topics')
+]
+kafka_check_group = cfg.OptGroup(name='kafka_healthcheck',
+                                 title='kafka_healthcheck')
+
+
+def register_opts(conf):
+    conf.register_group(kafka_check_group)
+    conf.register_opts(kafka_check_opts, kafka_check_group)
+
+
+def list_opts():
+    return kafka_check_group, kafka_check_opts

monasca_log_api/conf/log_publisher.py

@@ -0,0 +1,41 @@
+# Copyright 2017 FUJITSU LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+
+_MAX_MESSAGE_SIZE = 1048576
+
+log_publisher_opts = [
+    cfg.StrOpt('kafka_url',
+               required=True,
+               help='Url to kafka server'),
+    cfg.MultiStrOpt('topics',
+                    default=['logs'],
+                    help='Consumer topics'),
+    cfg.IntOpt('max_message_size',
+               default=_MAX_MESSAGE_SIZE,
+               required=True,
+               help=('Message max size that can be sent '
+                     'to kafka, default to %d bytes' % _MAX_MESSAGE_SIZE))
+]
+log_publisher_group = cfg.OptGroup(name='log_publisher', title='log_publisher')
+
+
+def register_opts(conf):
+    conf.register_group(log_publisher_group)
+    conf.register_opts(log_publisher_opts, log_publisher_group)
+
+
+def list_opts():
+    return log_publisher_group, log_publisher_opts

monasca_log_api/conf/monitoring.py

@@ -0,0 +1,46 @@
+# Copyright 2017 FUJITSU LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+
+_DEFAULT_HOST = '127.0.0.1'
+_DEFAULT_PORT = 8125
+_DEFAULT_BUFFER_SIZE = 50
+
+monitoring_opts = [
+    cfg.IPOpt('statsd_host',
+              default=_DEFAULT_HOST,
+              help=('IP address of statsd server, default to %s'
+                    % _DEFAULT_HOST)),
+    cfg.PortOpt('statsd_port',
+                default=_DEFAULT_PORT,
+                help='Port of statsd server, default to %d' % _DEFAULT_PORT),
+    cfg.IntOpt('statsd_buffer',
+               default=_DEFAULT_BUFFER_SIZE,
+               required=True,
+               help=('Maximum number of metric to buffer before sending, '
+                     'default to %d' % _DEFAULT_BUFFER_SIZE)),
+    cfg.DictOpt('dimensions', default={},
+                required=False, help='Additional dimensions that can be set')
+]
+monitoring_group = cfg.OptGroup(name='monitoring', title='monitoring')
+
+
+def register_opts(conf):
+    conf.register_group(monitoring_group)
+    conf.register_opts(monitoring_opts, monitoring_group)
+
+
+def list_opts():
+    return monitoring_group, monitoring_opts

monasca_log_api/conf/role_middleware.py

@@ -0,0 +1,43 @@
+# Copyright 2017 FUJITSU LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+
+role_m_opts = [
+    cfg.ListOpt(name='path',
+                default='/',
+                help='List of paths where middleware applies to'),
+    cfg.ListOpt(name='default_roles',
+                default=None,
+                help='List of roles allowed to enter api'),
+    cfg.ListOpt(name='agent_roles',
+                default=None,
+                help=('List of roles, that if set, mean that request '
+                      'comes from agent, thus is authorized in the same '
+                      'time')),
+    cfg.ListOpt(name='delegate_roles',
+                default=['admin'],
+                help=('Roles that are allowed to POST logs on '
+                      'behalf of another tenant (project)'))
+]
+role_m_group = cfg.OptGroup(name='roles_middleware', title='roles_middleware')
+
+
+def register_opts(conf):
+    conf.register_group(role_m_group)
+    conf.register_opts(role_m_opts, role_m_group)
+
+
+def list_opts():
+    return role_m_group, role_m_opts

monasca_log_api/conf/service.py

@@ -0,0 +1,37 @@
+# Copyright 2017 FUJITSU LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+
+_DEFAULT_MAX_LOG_SIZE = 1024 * 1024
+
+service_opts = [
+    cfg.StrOpt('region',
+               default=None,
+               help='Region'),
+    cfg.IntOpt('max_log_size',
+               default=_DEFAULT_MAX_LOG_SIZE,
+               help=('Refers to payload/envelope size. If either is exceeded'
+                     'API will throw an error'))
+]
+service_group = cfg.OptGroup(name='service', title='service')
+
+
+def register_opts(conf):
+    conf.register_group(service_group)
+    conf.register_opts(service_opts, service_group)
+
+
+def list_opts():
+    return service_group, service_opts

monasca_log_api/config.py

@@ -0,0 +1,51 @@
+# Copyright 2017 FUJITSU LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_log import log
+
+from monasca_log_api import conf
+from monasca_log_api import version
+
+CONF = conf.CONF
+LOG = log.getLogger(__name__)
+
+_CONF_LOADED = False
+
+
+def parse_args():
+    global _CONF_LOADED
+    if _CONF_LOADED:
+        LOG.debug('Configuration has been already loaded')
+        return
+
+    log.set_defaults()
+    log.register_options(CONF)
+
+    CONF(args=[],
+         # NOTE(trebskit) this disables any oslo.cfg CLI
+         # opts as gunicorn has some trouble with them
+         # i.e. gunicorn's argparse clashes with the one
+         # defined inside oslo.cfg
+         prog='log-api',
+         project='monasca',
+         version=version.version_str,
+         description='RESTful API to collect log files')
+
+    log.setup(CONF,
+              product_name='monasca-log-api',
+              version=version.version_str)
+
+    conf.register_opts()
+
+    _CONF_LOADED = True

monasca_log_api/healthcheck/kafka_check.py

@@ -1,4 +1,4 @@
-# Copyright 2015 FUJITSU LIMITED
+# Copyright 2015-2017 FUJITSU LIMITED
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -14,28 +14,13 @@
 
 import collections
 
-from oslo_config import cfg
+from monasca_common.kafka_lib import client
 from oslo_log import log
 
-from monasca_common.kafka_lib import client
+from monasca_log_api import conf
 
 LOG = log.getLogger(__name__)
-CONF = cfg.CONF
-
-kafka_check_opts = [
-    cfg.StrOpt('kafka_url',
-               required=True,
-               help='Url to kafka server'),
-    cfg.ListOpt('kafka_topics',
-                required=True,
-                default=['logs'],
-                help='Verify existence of configured topics')
-]
-kafka_check_group = cfg.OptGroup(name='kafka_healthcheck',
-                                 title='kafka_healthcheck')
-
-cfg.CONF.register_group(kafka_check_group)
-cfg.CONF.register_opts(kafka_check_opts, kafka_check_group)
+CONF = conf.CONF
 
 
 CheckResult = collections.namedtuple('CheckResult', ['healthy', 'message'])

monasca_log_api/middleware/role_middleware.py

@@ -1,4 +1,4 @@
-# Copyright 2015 FUJITSU LIMITED
+# Copyright 2015-2017 FUJITSU LIMITED
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -12,36 +12,15 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from oslo_config import cfg
 from oslo_log import log
 from oslo_middleware import base as om
 from webob import response
 
-CONF = cfg.CONF
+from monasca_log_api import conf
+
+CONF = conf.CONF
 LOG = log.getLogger(__name__)
 
-role_m_opts = [
-    cfg.ListOpt(name='path',
-                default='/',
-                help='List of paths where middleware applies to'),
-    cfg.ListOpt(name='default_roles',
-                default=None,
-                help='List of roles allowed to enter api'),
-    cfg.ListOpt(name='agent_roles',
-                default=None,
-                help=('List of roles, that if set, mean that request '
-                      'comes from agent, thus is authorized in the same '
-                      'time')),
-    cfg.ListOpt(name='delegate_roles',
-                default=['admin'],
-                help=('Roles that are allowed to POST logs on '
-                      'behalf of another tenant (project)'))
-]
-role_m_group = cfg.OptGroup(name='roles_middleware', title='roles_middleware')
-
-CONF.register_group(role_m_group)
-CONF.register_opts(role_m_opts, role_m_group)
-
 _X_IDENTITY_STATUS = 'X-Identity-Status'
 _X_ROLES = 'X-Roles'
 _X_MONASCA_LOG_AGENT = 'X-MONASCA-LOG-AGENT'

monasca_log_api/monitoring/client.py

@@ -1,4 +1,4 @@
-# Copyright 2016 FUJITSU LIMITED
+# Copyright 2016-2017 FUJITSU LIMITED
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -14,41 +14,18 @@
 
 import monascastatsd
 
-from oslo_config import cfg
 from oslo_log import log
 
-LOG = log.getLogger(__name__)
-CONF = cfg.CONF
+from monasca_log_api import conf
 
-_DEFAULT_HOST = '127.0.0.1'
-_DEFAULT_PORT = 8125
-_DEFAULT_BUFFER_SIZE = 50
+LOG = log.getLogger(__name__)
+CONF = conf.CONF
+
+_CLIENT_NAME = 'monasca'
 _DEFAULT_DIMENSIONS = {
     'service': 'monitoring',
     'component': 'monasca-log-api'
 }
-_CLIENT_NAME = 'monasca'
-
-monitoring_opts = [
-    cfg.IPOpt('statsd_host',
-              default=_DEFAULT_HOST,
-              help=('IP address of statsd server, default to %s'
-                    % _DEFAULT_HOST)),
-    cfg.PortOpt('statsd_port',
-                default=_DEFAULT_PORT,
-                help='Port of statsd server, default to %d' % _DEFAULT_PORT),
-    cfg.IntOpt('statsd_buffer',
-               default=_DEFAULT_BUFFER_SIZE,
-               required=True,
-               help=('Maximum number of metric to buffer before sending, '
-                     'default to %d' % _DEFAULT_BUFFER_SIZE)),
-    cfg.DictOpt('dimensions')
-]
-
-monitoring_group = cfg.OptGroup(name='monitoring', title='monitoring')
-
-cfg.CONF.register_group(monitoring_group)
-cfg.CONF.register_opts(monitoring_opts, monitoring_group)
 
 
 def get_client(dimensions=None):

monasca_log_api/reference/common/log_publisher.py

@@ -18,17 +18,16 @@ import time
 import falcon
 from monasca_common.kafka import producer
 from monasca_common.rest import utils as rest_utils
-from oslo_config import cfg
 from oslo_log import log
 
+from monasca_log_api import conf
 from monasca_log_api.monitoring import client
 from monasca_log_api.monitoring import metrics
 from monasca_log_api.reference.common import model
 
 LOG = log.getLogger(__name__)
-CONF = cfg.CONF
+CONF = conf.CONF
 
-_MAX_MESSAGE_SIZE = 1048576
 _RETRY_AFTER = 60
 _TIMESTAMP_KEY_SIZE = len(
     bytearray(str(int(time.time() * 1000)).encode('utf-8')))
@@ -37,25 +36,6 @@ _TRUNCATED_PROPERTY_SIZE = len(
 _KAFKA_META_DATA_SIZE = 32
 _TRUNCATION_SAFE_OFFSET = 1
 
-log_publisher_opts = [
-    cfg.StrOpt('kafka_url',
-               required=True,
-               help='Url to kafka server'),
-    cfg.MultiStrOpt('topics',
-                    default=['logs'],
-                    help='Consumer topics'),
-    cfg.IntOpt('max_message_size',
-               default=_MAX_MESSAGE_SIZE,
-               required=True,
-               help=('Message max size that can be sent '
-                     'to kafka, default to %d bytes' % _MAX_MESSAGE_SIZE))
-]
-
-log_publisher_group = cfg.OptGroup(name='log_publisher', title='log_publisher')
-
-cfg.CONF.register_group(log_publisher_group)
-cfg.CONF.register_opts(log_publisher_opts, log_publisher_group)
-
 
 class InvalidMessageException(Exception):
     pass

monasca_log_api/reference/common/validation.py

@@ -15,14 +15,14 @@
 import re
 
 import falcon
-from oslo_config import cfg
 from oslo_log import log
 import six
 
 from monasca_log_api.api import exceptions
+from monasca_log_api import conf
 
 LOG = log.getLogger(__name__)
-CONF = cfg.CONF
+CONF = conf.CONF
 
 APPLICATION_TYPE_CONSTRAINTS = {
     'MAX_LENGTH': 255,

monasca_log_api/reference/v2/common/service.py

@@ -20,27 +20,12 @@ from oslo_config import cfg
 from oslo_log import log
 
 from monasca_log_api.api import exceptions
+from monasca_log_api import conf
 from monasca_log_api.reference.common import model
 from monasca_log_api.reference.common import validation
 
 LOG = log.getLogger(__name__)
-CONF = cfg.CONF
-
-_DEFAULT_MAX_LOG_SIZE = 1024 * 1024
-
-service_opts = [
-    cfg.StrOpt('region',
-               default=None,
-               help='Region'),
-    cfg.IntOpt('max_log_size',
-               default=_DEFAULT_MAX_LOG_SIZE,
-               help=('Refers to payload/envelope size. If either is exceeded'
-                     'API will throw an error'))
-]
-service_group = cfg.OptGroup(name='service', title='service')
-
-CONF.register_group(service_group)
-CONF.register_opts(service_opts, service_group)
+CONF = conf.CONF
 
 EPOCH_START = datetime.datetime(1970, 1, 1)
 

monasca_log_api/reference/v3/common/bulk_processor.py

@@ -12,15 +12,15 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from oslo_config import cfg
 from oslo_log import log
 
+from monasca_log_api import conf
 from monasca_log_api.reference.common import log_publisher
 from monasca_log_api.reference.common import model
 from monasca_log_api.reference.common import validation
 
 LOG = log.getLogger(__name__)
-CONF = cfg.CONF
+CONF = conf.CONF
 
 
 class BulkProcessor(log_publisher.LogPublisher):

monasca_log_api/tests/base.py

@@ -27,10 +27,12 @@ from oslotest import base as os_test
 import six
 
 from monasca_log_api.api.core import request
+from monasca_log_api import conf
 
 
 def mock_config(test):
-    return test.useFixture(oo_cfg.Config())
+    conf.register_opts()
+    return test.useFixture(oo_cfg.Config(conf=conf.CONF))
 
 
 def mock_context(test):

monasca_log_api/tests/test_monitoring.py

@@ -19,6 +19,11 @@ from monasca_log_api.tests import base
 
 
 class TestMonitoring(base.BaseTestCase):
+
+    def setUp(self):
+        super(TestMonitoring, self).setUp()
+        base.mock_config(self)
+
     @mock.patch('monasca_log_api.monitoring.client.monascastatsd')
     def test_should_use_default_dimensions_if_none_specified(self,
                                                              monascastatsd):

releasenotes/notes/osloconfiggen-d8a0f0a8d1acb961.yaml

@@ -0,0 +1,18 @@
+---
+prelude: >
+    Matching OpenStack projects in configuration handling.
+other:
+  - Removed configuration file from project tree. It is now generated via
+    oslo-config-generator utility.
+
+    Moved all the code where configuration options are registered into single
+    location to mimic the configuration file (i.e. provide single place as it
+    was with configuration file).
+
+    Removed final place where configuration is duplicated. In other words
+    person providing and/or modyfying options could have been forced to include
+    them also in the configuration file held in the tree. Not the only place
+    where it is required is Python codebase.
+
+    Devstack plugin is also using new utility to provide configuration for
+    monasca-log-api launched inside the VM or Gate environment.

setup.cfg

@@ -43,6 +43,9 @@ wsgi_scripts =
 tempest.test_plugins =
     monasca_log_api_tests = monasca_log_api_tempest.plugin:MonascaLogApiTempestPlugin
 
+oslo.config.opts =
+    monasca_log_api = monasca_log_api.conf:list_opts
+
 [build_sphinx]
 all_files = 1
 build-dir = doc/build

tox.ini

@@ -81,6 +81,10 @@ commands =
   {[testenv:bashate]commands}
   {[testenv:checkniceness]commands}
 
+[testenv:genconfig]
+description = Generates sample documentation file for monasca-log-api
+commands = oslo-config-generator --config-file=config-generator/monasca-log-api.conf
+
 [testenv:docs]
 description = Builds api-ref, api-guide, releasenotes and devdocs
 commands =
@@ -143,3 +147,5 @@ show-source = True
 enable-extensions = H203,H106
 
 [hacking]
+import_exceptions =
+  six.moves