From 3b3d6df0c2ab33bffdb85006468519f62f1417ef Mon Sep 17 00:00:00 2001
From: Jeffrey Zhang <zhang.lei.fly@gmail.com>
Date: Thu, 24 Dec 2015 01:07:29 +0800
Subject: [PATCH] use oslo.config instead of raw argparse.ArgumentParser
* Use oslo.config library
* kolla-build section is deprecated in favor of DEFAULT section
* Remove the deprecated `--template` option
* `--no-cache` is deprecated in favor of `--cache` and `--nocache`
options, which are using oslo BoolOpt inverse feature[0]
[0] https://github.com/openstack/oslo.config/blob/master/oslo_config/cfg.py#L1074
Closes-Bug: #1528542
Change-Id: I62c5ca3c8e057917d2798256e9d9efc8f6578f60
---
etc/kolla/kolla-build.conf | 12 +-
kolla/cmd/build.py | 332 +++++++++++++++++++------------------
requirements.txt | 1 +
3 files changed, 178 insertions(+), 167 deletions(-)
diff --git a/etc/kolla/kolla-build.conf b/etc/kolla/kolla-build.conf
index 39d210925e..e75788ddc4 100644
--- a/etc/kolla/kolla-build.conf
+++ b/etc/kolla/kolla-build.conf
@@ -1,6 +1,6 @@
# Configuration for kolla build script
-[kolla-build]
+[DEFAULT]
#The MAINTAINER field
#maintainer = Kolla Project (https://launchpad.net/kolla)
@@ -58,11 +58,11 @@
# Preset build profiles can be set here to easily build common sets of images
[profiles]
-infra = ceph,data,mariadb,haproxy,keepalived,kolla-ansible,memcached,mongodb,openvswitch,rabbitmq,rsyslog
-main = cinder,ceilometer,glance,heat,horizon,keystone,neutron,nova,swift
-aux = aodh,designate,gnocchi,ironic,magnum,mistral,trove,zaqar
-default = data,kolla-ansible,glance,haproxy,heat,horizon,keepalived,keystone,memcached,mariadb,neutron,nova,openvswitch,rabbitmq,rsyslog
-gate = ceph,cinder,data,dind,glance,haproxy,heat,horizon,keepalived,keystone,kolla-ansible,mariadb,memcached,neutron,nova,openvswitch,rabbitmq,rsyslog
+#infra = ceph,data,mariadb,haproxy,keepalived,kolla-ansible,memcached,mongodb,openvswitch,rabbitmq,rsyslog
+#main = cinder,ceilometer,glance,heat,horizon,keystone,neutron,nova,swift
+#aux = aodh,designate,gnocchi,ironic,magnum,mistral,trove,zaqar
+#default = data,kolla-ansible,glance,haproxy,heat,horizon,keepalived,keystone,memcached,mariadb,neutron,nova,openvswitch,rabbitmq,rsyslog
+#gate = ceph,cinder,data,dind,glance,haproxy,heat,horizon,keepalived,keystone,kolla-ansible,mariadb,memcached,neutron,nova,openvswitch,rabbitmq,rsyslog
# Provide location of sources for source install builds.
# Example:
diff --git a/kolla/cmd/build.py b/kolla/cmd/build.py
index 33627a859f..8d5aefe242 100755
--- a/kolla/cmd/build.py
+++ b/kolla/cmd/build.py
@@ -14,7 +14,6 @@
# TODO(jpeeler): Add clean up handler for SIGINT
-import argparse
import datetime
import errno
import json
@@ -34,6 +33,8 @@ import time
import docker
import git
import jinja2
+from oslo_config import cfg
+from oslo_config import types
from requests.exceptions import ConnectionError
import six
@@ -46,6 +47,7 @@ signal.signal(signal.SIGINT, signal.SIG_DFL)
RDO_MIRROR = "http://trunk.rdoproject.org/centos7"
DELOREAN = "{}/current/delorean.repo".format(RDO_MIRROR)
DELOREAN_DEPS = "{}/delorean-deps.repo".format(RDO_MIRROR)
+INSTALL_TYPE_CHOICES = ['binary', 'source', 'rdo', 'rhos']
class KollaDirNotFoundException(Exception):
@@ -107,13 +109,12 @@ class PushThread(Thread):
class WorkerThread(Thread):
- def __init__(self, queue, push_queue, config):
- self.config = config
+ def __init__(self, queue, push_queue, conf):
+ self.conf = conf
self.queue = queue
self.push_queue = push_queue
- self.nocache = config['no_cache']
- self.forcerm = not config['keep']
- self.retries = config['retries']
+ self.nocache = not conf.cache or conf.no_cache
+ self.forcerm = not conf.keep
self.dc = docker_client()
super(WorkerThread, self).__init__()
@@ -133,7 +134,7 @@ class WorkerThread(Thread):
while True:
try:
image = self.queue.get()
- for _ in range(self.retries + 1):
+ for _ in range(self.conf.retries + 1):
self.builder(image)
if image['status'] in ['built', 'unmatched',
'parent_error']:
@@ -273,7 +274,7 @@ class WorkerThread(Thread):
image['status'] = "built"
LOG.info('%s:Built', image['name'])
- if self.config['push']:
+ if self.conf.push:
self.push_queue.put(image)
@@ -320,121 +321,25 @@ def find_config_file(filename):
)
-def merge_args_and_config(settings_from_config_file):
- parser = argparse.ArgumentParser(description='Kolla build script')
-
- defaults = {
- "base": "centos",
- "base_tag": "latest",
- "install_type": "binary",
- "keep": False,
- "maintainer": "Kolla Project (https://launchpad.net/kolla)",
- "namespace": "kollaglue",
- "no_cache": False,
- "push": False,
- "push_threads": 1,
- "registry": None,
- "retries": 3,
- "rpm_setup_config": '',
- "tag": get_kolla_version(),
- "threads": 8
- }
- defaults.update(settings_from_config_file.items('kolla-build'))
- parser.set_defaults(**defaults)
-
- parser.add_argument('-b', '--base',
- help='The base distro to use when building',
- type=str)
- parser.add_argument('--base-tag',
- help='The base distro image tag',
- type=str)
- parser.add_argument('-d', '--debug',
- help='Turn on debugging log level',
- action='store_true')
- parser.add_argument('-i', '--include-header',
- help=('Path to custom file to be added at '
- 'beginning of base Dockerfile'),
- type=str)
- parser.add_argument('-I', '--include-footer',
- help=('Path to custom file to be added at '
- 'end of Dockerfiles for final images'),
- type=str)
- parser.add_argument('--keep',
- help='Keep failed intermediate containers',
- action='store_true')
- parser.add_argument('-n', '--namespace',
- help='Set the Docker namespace name',
- type=str)
- parser.add_argument('--no-cache',
- help='Do not use the Docker cache when building',
- action='store_true')
- parser.add_argument('-p', '--profile',
- help=('Build a pre-defined set of images, see '
- '[profiles] section in '
- '{}'.format(
- find_config_file('kolla-build.conf'))),
- type=str,
- action='append')
- parser.add_argument('--push',
- help='Push images after building',
- action='store_true')
- parser.add_argument('--push-threads',
- help=('The number of threads to use while pushing'
- ' images.(NOTE: Docker can not handle'
- 'threading push properly.)'),
- type=int)
- parser.add_argument('-r', '--retries',
- help='The number of times to retry while building',
- type=int)
- parser.add_argument('regex',
- help=('Build only images matching '
- 'regex and its dependencies'),
- nargs='*')
- parser.add_argument('--registry',
- help=("the docker registry host"),
- type=str)
- parser.add_argument('-t', '--type',
- help='The method of the Openstack install: binary,'
- ' source, rdo, or rhos',
- type=str,
- dest='install_type')
- parser.add_argument('-T', '--threads',
- help='The number of threads to use while building.'
- ' (Note: setting to one will allow real time'
- ' logging.)',
- type=int)
- parser.add_argument('--tag',
- help='Set the Docker tag',
- type=str)
- parser.add_argument('--template',
- help='DEPRECATED: All Dockerfiles are templates',
- action='store_true',
- default=True)
- parser.add_argument('--template-only',
- help=("Don't build images. Generate Dockerfile only"),
- action='store_true')
- return vars(parser.parse_args())
-
-
class KollaWorker(object):
- def __init__(self, config):
+ def __init__(self, conf):
+ self.conf = conf
self.base_dir = os.path.abspath(find_base_dir())
LOG.debug("Kolla base directory: " + self.base_dir)
self.images_dir = os.path.join(self.base_dir, 'docker')
- self.registry = config['registry']
+
+ self.registry = conf.registry
if self.registry:
- self.namespace = self.registry + '/' + config['namespace']
+ self.namespace = self.registry + '/' + conf.namespace
else:
- self.namespace = config['namespace']
- self.base = config['base']
- self.base_tag = config['base_tag']
- self.install_type = config['install_type']
- self.tag = config['tag']
+ self.namespace = conf.namespace
+ self.base = conf.base
+ self.base_tag = conf.base_tag
+ self.install_type = conf.install_type
+ self.tag = conf.tag
self.images = list()
- rpm_setup_config = [cfg.strip()
- for cfg in config['rpm_setup_config'].split(',')
- if cfg]
+ rpm_setup_config = filter(None, conf.rpm_setup_config)
self.rpm_setup = self.build_rpm_setup(rpm_setup_config)
if self.install_type == 'binary':
@@ -454,17 +359,13 @@ class KollaWorker(object):
self.image_prefix = self.base + '-' + self.install_type + '-'
- self.tag = config['tag']
- self.include_header = config['include_header']
- self.include_footer = config['include_footer']
- self.regex = config['regex']
- self.profile = config['profile']
- self.source_location = six.moves.configparser.SafeConfigParser()
- self.source_location.read(find_config_file('kolla-build.conf'))
+ self.include_header = conf.include_header
+ self.include_footer = conf.include_footer
+ self.regex = conf.regex
self.image_statuses_bad = dict()
self.image_statuses_good = dict()
self.image_statuses_unmatched = dict()
- self.maintainer = config['maintainer']
+ self.maintainer = conf.maintainer
def build_rpm_setup(self, rpm_setup_config):
"""Generates a list of docker commands based on provided configuration.
@@ -474,9 +375,6 @@ class KollaWorker(object):
"""
rpm_setup = list()
- if not rpm_setup_config:
- rpm_setup_config = [DELOREAN, DELOREAN_DEPS]
-
for config in rpm_setup_config:
if config.endswith('.rpm'):
# RPM files can be installed with yum from file path or url
@@ -566,19 +464,9 @@ class KollaWorker(object):
if self.regex:
filter_ += self.regex
- if self.profile:
- for profile in self.profile:
- try:
- filter_ += self.source_location.get('profiles',
- profile
- ).split(',')
- except six.moves.configparser.NoSectionError:
- LOG.error('No [profiles] section found in %s',
- find_config_file('kolla-build.conf'))
- except six.moves.configparser.NoOptionError:
- LOG.error('No profile named "%s" found in %s',
- self.profile,
- find_config_file('kolla-build.conf'))
+ if self.conf.profile:
+ for profile in self.conf.profile:
+ filter_ += self.conf.profiles[profile]
if filter_:
patterns = re.compile(r"|".join(filter_).join('()'))
@@ -650,17 +538,14 @@ class KollaWorker(object):
def build_image_list(self):
def process_source_installation(image, section):
installation = dict()
- try:
- installation['type'] = \
- self.source_location.get(section, 'type')
- installation['source'] = \
- self.source_location.get(section, 'location')
+ if section not in self.conf.list_all_sections():
+ LOG.debug('%s:No source location found', section)
+ else:
+ installation['type'] = self.conf[section]['type']
+ installation['source'] = self.conf[section]['location']
installation['name'] = section
if installation['type'] == 'git':
- installation['reference'] = \
- self.source_location.get(section, 'reference')
- except six.moves.configparser.NoSectionError:
- LOG.debug('%s:No source location found', section)
+ installation['reference'] = self.conf[section]['reference']
return installation
for path in self.docker_build_paths:
@@ -681,12 +566,17 @@ class KollaWorker(object):
image['plugins'] = list()
if self.install_type == 'source':
- image['source'] = \
- process_source_installation(image, image['name'])
+ self.conf.register_opts([
+ cfg.StrOpt('type'),
+ cfg.StrOpt('location'),
+ cfg.StrOpt('reference')
+ ], image['name'])
+ image['source'] = process_source_installation(image,
+ image['name'])
for plugin in [match.group(0) for match in
(re.search('{}-plugin-.+'.format(image['name']),
section) for section in
- self.source_location.sections()) if match]:
+ self.conf.list_all_sections()) if match]:
image['plugins'].append(
process_source_installation(image, plugin))
@@ -725,19 +615,138 @@ class KollaWorker(object):
return queue
+def get_conf():
+ conf = cfg.ConfigOpts()
+
+ _kolla_profile_opts = [
+ cfg.ListOpt('infra',
+ default=['ceph', 'data', 'mariadb', 'haproxy',
+ 'keepalived', 'kolla-ansible', 'memcached',
+ 'mongodb', 'openvswitch', 'rabbitmq', 'rsyslog']),
+ cfg.ListOpt('main',
+ default=['cinder', 'ceilometer', 'glance', 'heat',
+ 'horizon', 'keystone', 'neutron', 'nova',
+ 'swift']),
+ cfg.ListOpt('aux',
+ default=['aodh', 'designate', 'gnocchi', 'ironic',
+ 'magnum', 'mistral', 'trove,' 'zaqar']),
+ cfg.ListOpt('default',
+ default=['data', 'kolla-ansible', 'glance', 'haproxy',
+ 'heat', 'horizon', 'keepalived', 'keystone',
+ 'memcached', 'mariadb', 'neutron', 'nova',
+ 'openvswitch', 'rabbitmq', 'rsyslog']),
+ cfg.ListOpt('gate',
+ default=['ceph', 'cinder', 'data', 'dind', 'glance',
+ 'haproxy', 'heat', 'horizon', 'keepalived',
+ 'keystone', 'kolla-ansible', 'mariadb',
+ 'memcached', 'neutron', 'nova', 'openvswitch',
+ 'rabbitmq', 'rsyslog'])
+ ]
+
+ _kolla_cli_opts = [
+ cfg.StrOpt('base', short='b', default='centos',
+ deprecated_group='kolla-build',
+ help='The base distro to use when building'),
+ cfg.StrOpt('base_tag', default='latest',
+ deprecated_group='kolla-build',
+ help='The base distro image tag'),
+ cfg.BoolOpt('debug', short='d', default=False,
+ deprecated_group='kolla-build',
+ help='Turn on debugging log level'),
+ cfg.StrOpt('include-header', short='i',
+ deprecated_group='kolla-build',
+ help=('Path to custom file to be added at '
+ 'beginning of base Dockerfile')),
+ cfg.StrOpt('include-footer', short='I',
+ deprecated_group='kolla-build',
+ help=('Path to custom file to be added at '
+ 'end of Dockerfiles for final images')),
+ cfg.BoolOpt('keep', default=False,
+ deprecated_group='kolla-build',
+ help='Keep failed intermediate containers'),
+ cfg.StrOpt('namespace', short='n', default='kollaglue',
+ deprecated_group='kolla-build',
+ help='The Docker namespace name'),
+ cfg.BoolOpt('cache', default=True,
+ help='Use the Docker cache when building',
+ ),
+ cfg.BoolOpt('no-cache', default=False,
+ help='Do not use the Docker cache when building',
+ deprecated_for_removal=True),
+ cfg.MultiOpt('profile', types.String(), short='p',
+ deprecated_group='kolla-build',
+ help=('Build a pre-defined set of images, see [profiles]'
+ ' section in {}. The default profiles are:'
+ ' {}'.format(
+ find_config_file('kolla-build.conf'),
+ ', '.join(
+ [opt.name for opt in _kolla_profile_opts])
+ ))),
+ cfg.BoolOpt('push', default=False,
+ deprecated_group='kolla-build',
+ help='Push images after building'),
+ cfg.IntOpt('push-threads', default=1, min=1,
+ deprecated_group='kolla-build',
+ help=('The number of threads to user while pushing'
+ ' Images. Note: Docker can not handle threading'
+ ' push properly.')),
+ cfg.IntOpt('retries', short='r', default=3, min=0,
+ deprecated_group='kolla-build',
+ help='The number of times to retry while building'),
+ cfg.MultiOpt('regex', types.String(), positional=True,
+ help=('Build only images matching regex and its'
+ ' dependencies')),
+ cfg.StrOpt('registry', deprecated_group='kolla-build',
+ help=('The docker registry host. The default registry host'
+ ' is Docker Hub')),
+ cfg.StrOpt('type', short='t', default='binary',
+ choices=INSTALL_TYPE_CHOICES,
+ dest='install_type', deprecated_group='kolla-build',
+ help=('The method of the Openstack install. The valid'
+ ' types are: {}'.format(
+ ', '.join(INSTALL_TYPE_CHOICES)))),
+ cfg.IntOpt('threads', short='T', default=8, min=1,
+ deprecated_group='kolla-build',
+ help=('The number of threads to use while building.'
+ ' (Note: setting to one will allow real time'
+ ' logging.)')),
+ cfg.StrOpt('tag', default=get_kolla_version(),
+ deprecated_group='kolla-build',
+ help='The Docker tag'),
+ cfg.BoolOpt('template-only', default=False,
+ deprecated_group='kolla-build',
+ help=("Don't build images. Generate Dockerfile only")),
+ ]
+
+ _kolla_base_opts = [
+ cfg.StrOpt('maintainer', deprecated_group='kolla-build',
+ default='Kolla Project (https://launchpad.net/kolla)',
+ help='The MAINTAINER field'),
+ cfg.ListOpt('rpm_setup_config', default=[DELOREAN, DELOREAN_DEPS],
+ deprecated_group='kolla-build',
+ help=('Comma separated list of .rpm or .repo file(s)'
+ 'or URL(s) to install before building containers'))
+ ]
+ conf.register_cli_opts(_kolla_cli_opts)
+ conf.register_opts(_kolla_profile_opts, group='profiles')
+ conf.register_opts(_kolla_base_opts)
+ conf(sys.argv[1:],
+ default_config_files=[find_config_file('kolla-build.conf')])
+ return conf
+
+
def main():
- build_config = six.moves.configparser.SafeConfigParser()
- build_config.read(find_config_file('kolla-build.conf'))
- config = merge_args_and_config(build_config)
- if config['debug']:
+ conf = get_conf()
+
+ if conf.debug:
LOG.setLevel(logging.DEBUG)
- kolla = KollaWorker(config)
+ kolla = KollaWorker(conf)
kolla.setup_working_dir()
kolla.find_dockerfiles()
kolla.create_dockerfiles()
- if config['template_only']:
+ if conf.template_only:
LOG.info('Dockerfiles are generated in %s', kolla.working_dir)
return
@@ -748,13 +757,13 @@ def main():
queue = kolla.build_queue()
push_queue = six.moves.queue.Queue()
- for x in six.moves.xrange(config['threads']):
- worker = WorkerThread(queue, push_queue, config)
+ for x in six.moves.xrange(conf.threads):
+ worker = WorkerThread(queue, push_queue, conf)
worker.setDaemon(True)
worker.start()
- for x in six.moves.xrange(config['push_threads']):
- push_thread = PushThread(config, push_queue)
+ for x in six.moves.xrange(conf.push_threads):
+ push_thread = PushThread(conf, push_queue)
push_thread.start()
# block until queue is empty
@@ -766,5 +775,6 @@ def main():
return kolla.get_image_statuses()
+
if __name__ == '__main__':
main()
diff --git a/requirements.txt b/requirements.txt
index f9c5e75d16..a1b735afd7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,3 +7,4 @@ Jinja2>=2.8 # BSD License (3 clause)
gitdb>=0.6.4 # BSD License (3 clause)
GitPython>=1.0.1 # BSD License (3 clause)
six>=1.9.0
+oslo.config>=2.7.0 # Apache-2.0