Merge "Generate the kolla-build.conf file using oslo_config"

.gitignore

@@ -9,6 +9,10 @@
 .tox/
 .buildconf
 
+# Files generated by setup.py
+kolla.egg-info
+build
+
 # Files generated by tools/genenv
 openrc
 openstack.env

etc/oslo-config-generator/kolla-build.conf

@@ -0,0 +1,3 @@
+[DEFAULT]
+output_file = etc/kolla/kolla-build.conf
+namespace = kolla

kolla/cmd/build.py

@@ -34,22 +34,19 @@ import docker
 import git
 import jinja2
 from oslo_config import cfg
-from oslo_config import types
 from requests.exceptions import ConnectionError
 import six
 from six.moves import range
 
+from kolla.common import config as common_config
+from kolla import version
+
 logging.basicConfig()
 LOG = logging.getLogger(__name__)
 LOG.setLevel(logging.INFO)
 
 signal.signal(signal.SIGINT, signal.SIG_DFL)
 
-RDO_MIRROR = "http://trunk.rdoproject.org/centos7"
-DELOREAN = "{}/current/delorean.repo".format(RDO_MIRROR)
-DELOREAN_DEPS = "{}/delorean-deps.repo".format(RDO_MIRROR)
-INSTALL_TYPE_CHOICES = ['binary', 'source', 'rdo', 'rhos']
-
 
 class KollaDirNotFoundException(Exception):
     pass
@@ -279,14 +276,6 @@ class WorkerThread(Thread):
             self.push_queue.put(image)
 
 
-def get_kolla_version():
-    local_conf_path = os.path.join(find_base_dir(), "setup.cfg")
-    config = six.moves.configparser.RawConfigParser()
-    config.read(local_conf_path)
-    version = config.get("metadata", "version")
-    return version
-
-
 def find_os_type():
     return platform.linux_distribution()
 
@@ -307,21 +296,6 @@ def find_base_dir():
     )
 
 
-def find_config_file(filename):
-    global_conf_path = os.path.join('/etc/kolla', filename)
-    local_conf_path = os.path.join(find_base_dir(), 'etc', 'kolla', filename)
-
-    if os.access(global_conf_path, os.R_OK):
-        return global_conf_path
-    elif os.access(local_conf_path, os.R_OK):
-        return local_conf_path
-    else:
-        raise KollaDirNotFoundException(
-            'Cant find kolla config. Searched at: %s and %s' %
-            (global_conf_path, local_conf_path)
-        )
-
-
 class KollaWorker(object):
 
     def __init__(self, conf):
@@ -417,6 +391,7 @@ class KollaWorker(object):
         LOG.debug('Set atime and mtime to 0 for all content in working dir')
 
     def create_dockerfiles(self):
+        kolla_version = version.version_info.cached_version_string()
         for path in self.docker_build_paths:
             template_name = "Dockerfile.j2"
             env = jinja2.Environment(loader=jinja2.FileSystemLoader(path))
@@ -429,7 +404,7 @@ class KollaWorker(object):
                       'namespace': self.namespace,
                       'tag': self.tag,
                       'maintainer': self.maintainer,
-                      'kolla_version': get_kolla_version(),
+                      'kolla_version': kolla_version,
                       'rpm_setup': self.rpm_setup}
             if self.include_header:
                 with open(self.include_header, 'r') as f:
@@ -616,128 +591,9 @@ class KollaWorker(object):
         return queue
 
 
-def get_conf():
-    conf = cfg.ConfigOpts()
-
-    _kolla_profile_opts = [
-        cfg.ListOpt('infra',
-                    default=['ceph', 'data', 'mariadb', 'haproxy',
-                             'keepalived', 'kolla-ansible', 'memcached',
-                             'mongodb', 'openvswitch', 'rabbitmq', 'rsyslog']),
-        cfg.ListOpt('main',
-                    default=['cinder', 'ceilometer', 'glance', 'heat',
-                             'horizon', 'keystone', 'neutron', 'nova',
-                             'swift']),
-        cfg.ListOpt('aux',
-                    default=['aodh', 'designate', 'gnocchi', 'ironic',
-                             'magnum', 'mistral', 'trove,' 'zaqar']),
-        cfg.ListOpt('default',
-                    default=['data', 'kolla-ansible', 'glance', 'haproxy',
-                             'heat', 'horizon', 'keepalived', 'keystone',
-                             'memcached', 'mariadb', 'neutron', 'nova',
-                             'openvswitch', 'rabbitmq', 'rsyslog']),
-        cfg.ListOpt('gate',
-                    default=['ceph', 'cinder', 'data', 'dind', 'glance',
-                             'haproxy', 'heat', 'horizon', 'keepalived',
-                             'keystone', 'kolla-ansible', 'mariadb',
-                             'memcached', 'neutron', 'nova', 'openvswitch',
-                             'rabbitmq', 'rsyslog'])
-    ]
-
-    _kolla_cli_opts = [
-        cfg.StrOpt('base', short='b', default='centos',
-                   deprecated_group='kolla-build',
-                   help='The base distro to use when building'),
-        cfg.StrOpt('base_tag', default='latest',
-                   deprecated_group='kolla-build',
-                   help='The base distro image tag'),
-        cfg.BoolOpt('debug', short='d', default=False,
-                    deprecated_group='kolla-build',
-                    help='Turn on debugging log level'),
-        cfg.StrOpt('include-header', short='i',
-                   deprecated_group='kolla-build',
-                   help=('Path to custom file to be added at '
-                         'beginning of base Dockerfile')),
-        cfg.StrOpt('include-footer', short='I',
-                   deprecated_group='kolla-build',
-                   help=('Path to custom file to be added at '
-                         'end of Dockerfiles for final images')),
-        cfg.BoolOpt('keep', default=False,
-                    deprecated_group='kolla-build',
-                    help='Keep failed intermediate containers'),
-        cfg.StrOpt('namespace', short='n', default='kollaglue',
-                   deprecated_group='kolla-build',
-                   help='The Docker namespace name'),
-        cfg.BoolOpt('cache', default=True,
-                    help='Use the Docker cache when building',
-                    ),
-        cfg.BoolOpt('no-cache', default=False,
-                    help='Do not use the Docker cache when building',
-                    deprecated_for_removal=True),
-        cfg.MultiOpt('profile', types.String(), short='p',
-                     deprecated_group='kolla-build',
-                     help=('Build a pre-defined set of images, see [profiles]'
-                           ' section in {}. The default profiles are:'
-                           ' {}'.format(
-                               find_config_file('kolla-build.conf'),
-                               ', '.join(
-                                   [opt.name for opt in _kolla_profile_opts])
-                           ))),
-        cfg.BoolOpt('push', default=False,
-                    deprecated_group='kolla-build',
-                    help='Push images after building'),
-        cfg.IntOpt('push-threads', default=1, min=1,
-                   deprecated_group='kolla-build',
-                   help=('The number of threads to user while pushing'
-                         ' Images. Note: Docker can not handle threading'
-                         ' push properly.')),
-        cfg.IntOpt('retries', short='r', default=3, min=0,
-                   deprecated_group='kolla-build',
-                   help='The number of times to retry while building'),
-        cfg.MultiOpt('regex', types.String(), positional=True,
-                     help=('Build only images matching regex and its'
-                           ' dependencies')),
-        cfg.StrOpt('registry', deprecated_group='kolla-build',
-                   help=('The docker registry host. The default registry host'
-                         ' is Docker Hub')),
-        cfg.StrOpt('type', short='t', default='binary',
-                   choices=INSTALL_TYPE_CHOICES,
-                   dest='install_type', deprecated_group='kolla-build',
-                   help=('The method of the Openstack install. The valid'
-                         ' types are: {}'.format(
-                             ', '.join(INSTALL_TYPE_CHOICES)))),
-        cfg.IntOpt('threads', short='T', default=8, min=1,
-                   deprecated_group='kolla-build',
-                   help=('The number of threads to use while building.'
-                         ' (Note: setting to one will allow real time'
-                         ' logging.)')),
-        cfg.StrOpt('tag', default=get_kolla_version(),
-                   deprecated_group='kolla-build',
-                   help='The Docker tag'),
-        cfg.BoolOpt('template-only', default=False,
-                    deprecated_group='kolla-build',
-                    help=("Don't build images. Generate Dockerfile only")),
-    ]
-
-    _kolla_base_opts = [
-        cfg.StrOpt('maintainer', deprecated_group='kolla-build',
-                   default='Kolla Project (https://launchpad.net/kolla)',
-                   help='The MAINTAINER field'),
-        cfg.ListOpt('rpm_setup_config', default=[DELOREAN, DELOREAN_DEPS],
-                    deprecated_group='kolla-build',
-                    help=('Comma separated list of .rpm or .repo file(s)'
-                          'or URL(s) to install before building containers'))
-    ]
-    conf.register_cli_opts(_kolla_cli_opts)
-    conf.register_opts(_kolla_profile_opts, group='profiles')
-    conf.register_opts(_kolla_base_opts)
-    conf(sys.argv[1:],
-         default_config_files=[find_config_file('kolla-build.conf')])
-    return conf
-
-
 def main():
-    conf = get_conf()
+    conf = cfg.ConfigOpts()
+    common_config.parse(conf, sys.argv[1:], prog='kolla-build')
 
     if conf.debug:
         LOG.setLevel(logging.DEBUG)

kolla/common/config.py

@@ -0,0 +1,263 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import itertools
+
+from oslo_config import cfg
+from oslo_config import types
+
+from kolla import version
+
+
+RDO_MIRROR = "http://trunk.rdoproject.org/centos7"
+DELOREAN = "{}/current/delorean.repo".format(RDO_MIRROR)
+DELOREAN_DEPS = "{}/delorean-deps.repo".format(RDO_MIRROR)
+INSTALL_TYPE_CHOICES = ['binary', 'source', 'rdo', 'rhos']
+
+_PROFILE_OPTS = [
+    cfg.ListOpt('infra',
+                default=['ceph', 'data', 'mariadb', 'haproxy',
+                         'keepalived', 'kolla-ansible', 'memcached',
+                         'mongodb', 'openvswitch', 'rabbitmq', 'rsyslog'],
+                help='Infra images'),
+    cfg.ListOpt('main',
+                default=['cinder', 'ceilometer', 'glance', 'heat',
+                         'horizon', 'keystone', 'neutron', 'nova',
+                         'swift'],
+                help='Main images'),
+    cfg.ListOpt('aux',
+                default=['aodh', 'designate', 'gnocchi', 'ironic',
+                         'magnum', 'mistral', 'trove,' 'zaqar'],
+                help='Aux Images'),
+    cfg.ListOpt('default',
+                default=['data', 'kolla-ansible', 'glance', 'haproxy',
+                         'heat', 'horizon', 'keepalived', 'keystone',
+                         'memcached', 'mariadb', 'neutron', 'nova',
+                         'openvswitch', 'rabbitmq', 'rsyslog'],
+                help='Default images'),
+    cfg.ListOpt('gate',
+                default=['ceph', 'cinder', 'data', 'dind', 'glance',
+                         'haproxy', 'heat', 'horizon', 'keepalived',
+                         'keystone', 'kolla-ansible', 'mariadb',
+                         'memcached', 'neutron', 'nova', 'openvswitch',
+                         'rabbitmq', 'rsyslog'],
+                help='Gate images')
+]
+
+_CLI_OPTS = [
+    cfg.StrOpt('base', short='b', default='centos',
+               deprecated_group='kolla-build',
+               help='The base distro to use when building'),
+    cfg.StrOpt('base_tag', default='latest',
+               deprecated_group='kolla-build',
+               help='The base distro image tag'),
+    cfg.BoolOpt('debug', short='d', default=False,
+                deprecated_group='kolla-build',
+                help='Turn on debugging log level'),
+    cfg.StrOpt('include-header', short='i',
+               deprecated_group='kolla-build',
+               help=('Path to custom file to be added at '
+                     'beginning of base Dockerfile')),
+    cfg.StrOpt('include-footer', short='I',
+               deprecated_group='kolla-build',
+               help=('Path to custom file to be added at '
+                     'end of Dockerfiles for final images')),
+    cfg.BoolOpt('keep', default=False,
+                deprecated_group='kolla-build',
+                help='Keep failed intermediate containers'),
+    cfg.StrOpt('namespace', short='n', default='kollaglue',
+               deprecated_group='kolla-build',
+               help='The Docker namespace name'),
+    cfg.BoolOpt('cache', default=True,
+                help='Use the Docker cache when building',
+                ),
+    cfg.BoolOpt('no-cache', default=False,
+                help='Do not use the Docker cache when building',
+                deprecated_for_removal=True),
+    cfg.MultiOpt('profile', types.String(), short='p',
+                 deprecated_group='kolla-build',
+                 help=('Build a pre-defined set of images, see [profiles]'
+                       ' section in config. The default profiles are:'
+                       ' {}'.format(', '.join(
+                           [opt.name for opt in _PROFILE_OPTS])
+                       ))),
+    cfg.BoolOpt('push', default=False,
+                deprecated_group='kolla-build',
+                help='Push images after building'),
+    cfg.IntOpt('push-threads', default=1, min=1,
+               deprecated_group='kolla-build',
+               help=('The number of threads to user while pushing'
+                     ' Images. Note: Docker can not handle threading'
+                     ' push properly.')),
+    cfg.IntOpt('retries', short='r', default=3, min=0,
+               deprecated_group='kolla-build',
+               help='The number of times to retry while building'),
+    cfg.MultiOpt('regex', types.String(), positional=True,
+                 help=('Build only images matching regex and its'
+                       ' dependencies')),
+    cfg.StrOpt('registry', deprecated_group='kolla-build',
+               help=('The docker registry host. The default registry host'
+                     ' is Docker Hub')),
+    cfg.StrOpt('type', short='t', default='binary',
+               choices=INSTALL_TYPE_CHOICES,
+               dest='install_type', deprecated_group='kolla-build',
+               help=('The method of the Openstack install. The valid'
+                     ' types are: {}'.format(
+                         ', '.join(INSTALL_TYPE_CHOICES)))),
+    cfg.IntOpt('threads', short='T', default=8, min=1,
+               deprecated_group='kolla-build',
+               help=('The number of threads to use while building.'
+                     ' (Note: setting to one will allow real time'
+                     ' logging.)')),
+    cfg.StrOpt('tag', default=version.version_info.cached_version_string(),
+               deprecated_group='kolla-build',
+               help='The Docker tag'),
+    cfg.BoolOpt('template-only', default=False,
+                deprecated_group='kolla-build',
+                help=("Don't build images. Generate Dockerfile only")),
+]
+
+_BASE_OPTS = [
+    cfg.StrOpt('maintainer', deprecated_group='kolla-build',
+               default='Kolla Project (https://launchpad.net/kolla)',
+               help='The MAINTAINER field'),
+    cfg.ListOpt('rpm_setup_config', default=[DELOREAN, DELOREAN_DEPS],
+                deprecated_group='kolla-build',
+                help=('Comma separated list of .rpm or .repo file(s)'
+                      'or URL(s) to install before building containers'))
+]
+
+
+SOURCES = {
+    'openstack-base': {
+        'type': 'url',
+        'location': ('https://github.com/openstack/requirements/tarball/'
+                     'master')},
+    'aodh-base': {
+        'type': 'url',
+        'location': 'http://tarballs.openstack.org/aodh/aodh-master.tar.gz'},
+    'ceilometer-base': {
+        'type': 'url',
+        'location': ('http://tarballs.openstack.org/ceilometer/'
+                     'ceilometer-master.tar.gz')},
+    'cinder-base': {
+        'type': 'url',
+        'location': ('http://tarballs.openstack.org/cinder/'
+                     'cinder-master.tar.gz')},
+    'designate-base': {
+        'type': 'url',
+        'location': ('http://tarballs.openstack.org/designate/'
+                     'designate-master.tar.gz')},
+    'glance-base': {
+        'type': 'url',
+        'location': ('http://tarballs.openstack.org/glance/'
+                     'glance-master.tar.gz')},
+    'gnocchi-base': {
+        'type': 'url',
+        'location': ('http://tarballs.openstack.org/gnocchi/'
+                     'gnocchi-master.tar.gz')},
+    'heat-base': {
+        'type': 'url',
+        'location': ('http://tarballs.openstack.org/heat/heat-master.tar.gz')},
+    'horizon': {
+        'type': 'url',
+        'location': ('http://tarballs.openstack.org/horizon/'
+                     'horizon-master.tar.gz')},
+    'ironic-base': {
+        'type': 'url',
+        'location': ('http://tarballs.openstack.org/ironic/'
+                     'ironic-master.tar.gz')},
+    'keystone': {
+        'type': 'url',
+        'location': ('http://tarballs.openstack.org/keystone/'
+                     'keystone-master.tar.gz')},
+    'magnum-base': {
+        'type': 'url',
+        'location': ('http://tarballs.openstack.org/magnum/'
+                     'magnum-master.tar.gz')},
+    'mistral-base': {
+        'type': 'url',
+        'location': ('http://tarballs.openstack.org/mistral/'
+                     'mistral-master.tar.gz')},
+    'murano-base': {
+        'type': 'url',
+        'location': ('http://tarballs.openstack.org/murano/'
+                     'murano-master.tar.gz')},
+    'neutron-base': {
+        'type': 'url',
+        'location': ('http://tarballs.openstack.org/neutron/'
+                     'neutron-master.tar.gz')},
+    'nova-base': {
+        'type': 'url',
+        'location': 'http://tarballs.openstack.org/nova/nova-master.tar.gz'},
+    'nova-spicehtml5proxy': {
+        'type': 'url',
+        'location': ('http://github.com/SPICE/spice-html5/tarball/'
+                     'spice-html5-0.1.6')},
+    'nova-novncproxy': {
+        'type': 'url',
+        'location': 'http://github.com/kanaka/noVNC/tarball/v0.5.1'},
+    'swift-base': {
+        'type': 'url',
+        'location': ('http://tarballs.openstack.org/swift/'
+                     'swift-master.tar.gz')},
+    'tempest': {
+        'type': 'url',
+        'location': ('http://tarballs.openstack.org/tempest/'
+                     'tempest-master.tar.gz')},
+    'zaqar': {
+        'type': 'url',
+        'location': ('http://tarballs.openstack.org/zaqar/'
+                     'zaqar-master.tar.gz')}
+
+}
+
+
+def _get_source_opt(type_, location, reference=None):
+    return [cfg.StrOpt('type', choices=['git', 'url'],
+                       default=type_,
+                       help='Source location type'),
+            cfg.StrOpt('location', default=location,
+                       help='The location for source install'),
+            cfg.StrOpt('reference', default=reference,
+                       help=('Git reference to pull, commit sha, tag'
+                             'or branch name'))]
+
+
+def gen_source_opts():
+    for name, params in SOURCES.iteritems():
+        type_ = params['type']
+        location = params['location']
+        reference = params.get('reference')
+        yield name, _get_source_opt(type_, location, reference)
+
+
+def list_opts():
+    return itertools.chain([(None, _CLI_OPTS),
+                            (None, _BASE_OPTS),
+                            ('profiles', _PROFILE_OPTS)],
+                           gen_source_opts(),
+                           )
+
+
+def parse(conf, args, usage=None, prog=None,
+          default_config_files=None):
+    conf.register_cli_opts(_CLI_OPTS)
+    conf.register_opts(_BASE_OPTS)
+    conf.register_opts(_PROFILE_OPTS, group='profiles')
+
+    conf(args=args,
+         project='kolla',
+         usage=usage,
+         prog=prog,
+         version=version.version_info.cached_version_string(),
+         default_config_files=default_config_files)

kolla/opts.py

@@ -0,0 +1,17 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from kolla.common import config
+
+
+def list_opts():
+    return config.list_opts()

kolla/version.py

@@ -0,0 +1,15 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pbr.version
+
+version_info = pbr.version.VersionInfo('kolla')

setup.cfg

@@ -37,6 +37,8 @@ scripts =
 [entry_points]
 console_scripts =
     kolla-build = kolla.cmd.build:main
+oslo.config.opts =
+    kolla = kolla.opts:list_opts
 
 [global]
 setup-hooks =

tox.ini

@@ -4,6 +4,7 @@ skipsdist = True
 envlist = functional,pep8
 
 [testenv]
+usedevelop=True
 install_command = pip install -U {opts} {packages}
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
@@ -102,6 +103,11 @@ commands =
   bash -c "if [ ! -d .testrepository ]; then testr init; fi"
   sudo -g docker testr run test_build.BuildTestUbuntuSource
 
+[testenv:genconfig]
+whitelist_externals = which
+commands=
+  oslo-config-generator --config-file etc/oslo-config-generator/kolla-build.conf
+
 [flake8]
 show-source = True
 exclude=.eggs,.git,.tox,doc