x
/
fuel-ccp
Code Issues Proposed changes

[BREAKING CLI] Remove oslo.config and oslo.log 

This change replaces oslo.config CLI parsing with dumb parser with
arguments that will be supplorted by cliff and doesn't provide
alternative for oslo.log, so logging will be broken until next change
with cliff lands. Add dumb logging.basicConfig until then.

Depends-On: Ibafdaf073bf973f05a721f8422e48e1f581dc4a4
Change-Id: I82169d36021201964a48d7e35161c51e10f30f4a
This commit is contained in:
Yuriy Taraday 2016-09-14 20:37:54 +03:00
parent b722dd7a28
commit ad989953c1
10 changed files with 28 additions and 330 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
fuel_ccp/config/__init__.py
View File

@ -1,13 +1,9 @@
import argparse
import logging
import sys
import itertools
import jsonschema
import os
from oslo_config import cfg
from oslo_log import _options as log_options
from oslo_log import log
import six
from fuel_ccp.config import _yaml
from fuel_ccp.config import builder
@ -22,36 +18,35 @@ LOG = logging.getLogger(__name__)
_REAL_CONF = None
def setup_config():
config_file, args = get_cli_config()
def setup_config(args=None):
if args is None:
args = sys.argv[1:]
config_file, args = get_cli_config(args)
if config_file is None:
config_file = find_config()
log.register_options(cfg.CONF)
yconf = get_config_defaults()
if config_file:
loaded_conf = _yaml.load_with_includes(config_file)
yconf._merge(loaded_conf)
set_oslo_defaults(cfg.CONF, yconf)
# Don't let oslo.config parse any config files
cfg.CONF(args, project='ccp', default_config_files=[])
log.setup(cfg.CONF, 'fuel-ccp')
action_dict = parse_args(args)
yconf._merge({'action': action_dict})
logging.basicConfig(level=logging.DEBUG)
if config_file:
LOG.debug('Loaded config from file %s', config_file)
else:
LOG.debug('No config file loaded')
copy_values_from_oslo(cfg.CONF, yconf)
validate_config(yconf)
global _REAL_CONF
_REAL_CONF = yconf
def get_cli_config():
def get_cli_config(args):
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument(
'--config-file',
metavar='PATH',
help=('Path to a config file to use.'))
args, rest = parser.parse_known_args()
args, rest = parser.parse_known_args(args)
if args.config_file:
config_file = os.path.abspath(os.path.expanduser(args.config_file))
else:
@ -74,43 +69,18 @@ def find_config():
return None
def set_oslo_defaults(oconf, yconf):
for key, value in six.iteritems(oconf):
if key == 'action':
continue
try:
yconf_value = yconf[key]
except KeyError:
continue
if isinstance(value, cfg.ConfigOpts.GroupAttr):
for subkey, subvalue in yconf_value._items():
oconf.set_default(group=key, name=subkey, default=subvalue)
else:
oconf.set_default(group=None, name=key, default=yconf_value)
def copy_values_from_oslo(oconf, yconf):
for key, value in six.iteritems(oconf):
if isinstance(value, (cfg.ConfigOpts.GroupAttr,
cfg.ConfigOpts.SubCommandAttr)):
try:
yconf_value = yconf[key]
except KeyError:
yconf_value = yconf[key] = _yaml.AttrDict()
if isinstance(value, cfg.ConfigOpts.SubCommandAttr):
yconf_items = set(yconf_value)
for skey in ['name', 'components', 'dry_run', 'export_dir',
'auth_url', 'skip_os_cleanup', 'types']:
try:
svalue = getattr(value, skey)
except cfg.NoSuchOptError:
continue
if skey not in yconf_items or svalue is not None:
yconf_value[skey] = svalue
else:
yconf_value._update(value)
else:
yconf[key] = value
def parse_args(args=None):
parser = argparse.ArgumentParser()
parser.add_argument('--debug', action='store_true', default=False)
parser.add_argument('--verbose', action='store_true', default=False)
parser.add_argument('--log-file', default=None)
subparsers = parser.add_subparsers(dest='action')
cli.add_parsers(subparsers)
action_dict = vars(parser.parse_args(args))
action_dict['name'] = action_dict.pop('action')
for name in ['debug', 'verbose', 'log_file']:
del action_dict[name]
return action_dict
class _Wrapper(object):
@ -141,12 +111,8 @@ def get_config_schema():
}
for module in [cli, builder, images, kubernetes, registry, repositories]:
schema['properties'].update(module.SCHEMA)
# Don't validate all options added from oslo.log and oslo.config
ignore_opts = ['config_file', 'config_dir']
for opt in itertools.chain(log_options.logging_cli_opts,
log_options.generic_log_opts,
log_options.log_opts):
ignore_opts.append(opt.name.replace('-', '_'))
# Don't validate all options used to be added from oslo.log and oslo.config
ignore_opts = ['debug', 'verbose', 'log_file']
for name in ignore_opts:
schema['properties'][name] = {}
# Also for now don't validate sections that used to be in deploy config

30
fuel_ccp/config/builder.py
View File

@ -1,35 +1,5 @@
import multiprocessing
from oslo_config import cfg
CONF = cfg.CONF
builder_opts = [
cfg.IntOpt('workers',
default=multiprocessing.cpu_count(),
help='Number of workers which build docker images'),
cfg.BoolOpt('keep-image-tree-consistency',
default=True,
help='Rebuild all descendant images to keep consistent state '
'of image tree'),
cfg.BoolOpt('build-base-images-if-not-exist',
default=True,
help='Make sure that all base images required for target '
'image building are ready and prebuild them if they are '
'not'),
cfg.BoolOpt('push',
default=False,
help='Push to the Docker registry'),
cfg.BoolOpt('no-cache',
default=False,
help='Dont use docker cache')
]
builder_opt_group = cfg.OptGroup(name='builder',
title='Images builder')
CONF.register_group(builder_opt_group)
CONF.register_cli_opts(builder_opts, builder_opt_group)
CONF.register_opts(builder_opts, builder_opt_group)
DEFAULTS = {
'builder': {
'workers': multiprocessing.cpu_count(),

13
fuel_ccp/config/cli.py
View File

@ -1,7 +1,3 @@
from oslo_config import cfg
CONF = cfg.CONF
def add_parsers(subparsers):
@ -48,15 +44,6 @@ def add_parsers(subparsers):
nargs='+',
help='CCP components to show dependencies')
CONF.register_cli_opt(cfg.SubCommandOpt('action',
handler=add_parsers))
common_opts = [
cfg.StrOpt('deploy-config', help='Cluster-wide configuration overrides')
]
CONF.register_cli_opts(common_opts)
DEFAULTS = {
'deploy_config': None,
'action': {

26
fuel_ccp/config/images.py
View File

@ -1,29 +1,3 @@
from oslo_config import cfg
CONF = cfg.CONF
images_opts = [
cfg.StrOpt('namespace',
default='ccp',
help='Namespace for Docker images'),
cfg.StrOpt('tag',
default='latest',
help='Tag for Docker images'),
cfg.StrOpt('base-distro',
default='debian',
help='Base distribution and image'),
cfg.StrOpt('base-tag',
default='jessie',
help='Tag of the base image'),
cfg.StrOpt('maintainer',
default='MOS Microservices <mos-microservices@mirantis.com>')
]
images_opt_group = cfg.OptGroup(name='images',
title='Docker images')
CONF.register_group(images_opt_group)
CONF.register_cli_opts(images_opts, images_opt_group)
CONF.register_opts(images_opts, images_opt_group)
DEFAULTS = {
'images': {
'namespace': 'ccp',

26
fuel_ccp/config/kubernetes.py
View File

@ -1,29 +1,3 @@
from oslo_config import cfg
CONF = cfg.CONF
kubernetes_opts = [
cfg.StrOpt('server',
default='http://localhost:8080',
help='The URL for the Kubernetes API server'),
cfg.StrOpt('namespace',
default='ccp',
help='The name of the namespace'),
cfg.StrOpt('ca-cert',
help='The location of the CA certificate file'),
cfg.StrOpt('key-file',
help='The location of the key file'),
cfg.StrOpt('cert-file',
help='The location of the certificate file'),
cfg.BoolOpt('insecure',
help='Disable certificate checking')
]
kubernetes_opt_group = cfg.OptGroup(name='kubernetes',
title='Kubernetes client')
CONF.register_group(kubernetes_opt_group)
CONF.register_cli_opts(kubernetes_opts, kubernetes_opt_group)
CONF.register_cli_opts(kubernetes_opts, kubernetes_opt_group)
DEFAULTS = {
'kubernetes': {
'server': 'http://localhost:8080',

27
fuel_ccp/config/registry.py
View File

@ -1,30 +1,3 @@
from oslo_config import cfg
CONF = cfg.CONF
registry_opts = [
cfg.StrOpt('address',
default='',
help='Docker registry address (host:port)'),
cfg.BoolOpt('insecure',
default=False,
help='Permit registry access without SSL'),
cfg.StrOpt('username',
default='',
help='Username for Docker registry'),
cfg.StrOpt('password',
default='',
help='Password for Docker registry'),
cfg.IntOpt('timeout',
default=300,
help='Registry request timeout, in seconds')
]
registry_opt_group = cfg.OptGroup(name='registry',
title='Docker registry data')
CONF.register_group(registry_opt_group)
CONF.register_cli_opts(registry_opts, registry_opt_group)
CONF.register_opts(registry_opts, registry_opt_group)
DEFAULTS = {
'registry': {
'address': '',

44
fuel_ccp/config/repositories.py
View File

@ -1,8 +1,6 @@
import multiprocessing
import os
from oslo_config import cfg
DEFAULT_REPOS = [
'fuel-ccp-cinder',
'fuel-ccp-debian-base',
@ -25,40 +23,6 @@ DEFAULT_REPOS = [
'fuel-ccp-stacklight',
]
CONF = cfg.CONF
repositories_opts = [
cfg.BoolOpt('clone',
default=True,
help='Automatic cloning of microservices repositories'),
cfg.IntOpt('clone-concurrency',
default=multiprocessing.cpu_count(),
help="Define how many 'git clone' processes will run "
"concurrently"),
cfg.BoolOpt('skip-empty',
default=True,
help='Skip repositories not containing Dockerfiles without '
'error'),
cfg.StrOpt('path',
default=os.path.expanduser('~/ccp-repos/'),
help='Path where the Git repositories are cloned'),
cfg.HostnameOpt('hostname',
default='review.openstack.org',
help='Git server hostname to pull repositories from'),
cfg.PortOpt('port', default=443, help='Git server port'),
cfg.StrOpt('protocol',
choices=['ssh', 'git', 'http', 'https'],
default='https',
help='Git access protocol'),
cfg.StrOpt('project',
default='openstack',
help='Gerrit project'),
cfg.StrOpt('username',
help='Username when using git or ssh scheme'),
cfg.ListOpt('names',
default=DEFAULT_REPOS,
help='List of repository names'),
]
DEFAULTS = {
'repositories': {
'clone': True,
@ -94,15 +58,7 @@ SCHEMA = {
}
for repo in DEFAULT_REPOS:
option = cfg.StrOpt(repo)
repositories_opts.append(option)
conf_name = repo.replace('-', '_')
SCHEMA['repositories']['properties'][conf_name] = \
{'anyOf': [{'type': 'string'}, {'type': 'null'}]}
DEFAULTS['repositories'][conf_name] = None
repositories_opt_group = cfg.OptGroup(name='repositories',
title='Git repositories for components')
CONF.register_group(repositories_opt_group)
CONF.register_cli_opts(repositories_opts, repositories_opt_group)
CONF.register_opts(repositories_opts, repositories_opt_group)

14
fuel_ccp/tests/conf_fixture.py
View File

@ -1,18 +1,10 @@
import fixtures
from oslo_config import cfg
from oslo_config import fixture as oslo_fixture
from oslo_log import log
from fuel_ccp import config
from fuel_ccp.config import _yaml
class Config(fixtures.Fixture):
def _setUp(self):
self.useFixture(oslo_fixture.Config())
log.register_options(cfg.CONF)
cfg.CONF(['build'], default_config_files=[])
self.conf = _yaml.AttrDict()
config.copy_values_from_oslo(cfg.CONF, self.conf)
self.useFixture(
fixtures.MockPatchObject(config, '_REAL_CONF', new=self.conf))
self.useFixture(fixtures.MockPatchObject(config, '_REAL_CONF'))
config.setup_config(['build'])
self.conf = config._REAL_CONF

94
fuel_ccp/tests/test_config.py
View File

@ -1,9 +1,5 @@
import collections
import functools
import fixtures
import jsonschema
from oslo_config import cfg
import six
import testscenarios
@ -38,9 +34,7 @@ class TestGetCLIConfig(testscenarios.WithScenarios, base.TestCase):
def test_get_cli_config(self):
self.useFixture(fixtures.MockPatch(
'argparse.ArgumentParser.error', side_effect=ArgumentParserError))
new_argv = ['ccp'] + self.argv
self.useFixture(fixtures.MockPatch('sys.argv', new=new_argv))
result = config.get_cli_config()
result = config.get_cli_config(self.argv)
self.assertEqual(result, self.expected_result)
@ -54,92 +48,6 @@ def nested_dict_to_attrdict(d):
return d
class TestSetOsloDefaults(testscenarios.WithScenarios, base.TestCase):
scenarios = [
('empty', {'yconf': {}, 'expected_defaults': {}}),
('bogus', {'yconf': {'bogus': 1}, 'expected_defaults': {}}),
('simple', {
'yconf': {'debug': False},
'expected_defaults': {None: {'debug': False}},
}),
('deep', {
'yconf': {'thegroup': {'count': 42}},
'expected_defaults': {'thegroup': {'count': 42}},
}),
]
yconf = None
expected_defaults = None
def setUp(self):
super(TestSetOsloDefaults, self).setUp()
self.conf = cfg.ConfigOpts()
self.conf.register_opt(cfg.BoolOpt('debug', default=False))
self.conf.register_opt(cfg.IntOpt('count'), group='thegroup')
def get_defaults(self):
res = collections.defaultdict(
functools.partial(collections.defaultdict, dict))
for opt_info, group in self.conf._all_opt_infos():
try:
default = opt_info['default']
except KeyError:
continue
if group is not None:
group = group.name
res[group][opt_info['opt'].name] = default
return res
def test_set_oslo_defaults(self):
yconf = nested_dict_to_attrdict(self.yconf)
config.set_oslo_defaults(self.conf, yconf)
self.assertEqual(self.get_defaults(), self.expected_defaults)
class TestCopyValuesFromOslo(testscenarios.WithScenarios, base.TestCase):
scenarios = [
('simple', {
'yconf': {},
'oconf': {None: {'debug': True}},
'expected_result': {'debug': True, 'thegroup': {'count': None}},
}),
('overwrite', {
'yconf': {'debug': False},
'oconf': {None: {'debug': True}},
'expected_result': {'debug': True, 'thegroup': {'count': None}},
}),
('deep', {
'yconf': {'debug': False},
'oconf': {'thegroup': {'count': 3}},
'expected_result': {'debug': False, 'thegroup': {'count': 3}},
}),
('deep_overwrite_with_bogus', {
'yconf': {'thegroup': {'bogus': 'value'}, 'other': 1},
'oconf': {'thegroup': {'count': 3}},
'expected_result': {
'debug': False,
'thegroup': {'count': 3, 'bogus': 'value'},
'other': 1,
},
}),
]
yconf = None
oconf = None
expected_result = None
def test_copy_values_from_oslo(self):
conf = cfg.ConfigOpts()
conf.register_opt(cfg.BoolOpt('debug', default=False))
conf.register_opt(cfg.IntOpt('count'), group='thegroup')
for group, values in six.iteritems(self.oconf):
for key, value in six.iteritems(values):
conf.set_default(group=group, name=key, default=value)
yconf = nested_dict_to_attrdict(self.yconf)
config.copy_values_from_oslo(conf, yconf)
self.assertEqual(yconf, self.expected_result)
class TestConfigSchema(base.TestCase):
def test_validate_config_schema(self):
schema = config.get_config_schema()

2
requirements.txt
View File

@ -9,8 +9,6 @@ docker-py>=1.6.0,<1.8.0 # Apache-2.0
GitPython>=1.0.1 # BSD License (3 clause)
Jinja2>=2.8 # BSD License (3 clause)
jsonschema>=2.0.0,<3.0.0,!=2.5.0 # MIT
oslo.config>=3.9.0 # Apache-2.0
oslo.log>=1.14.0 # Apache-2.0
PyYAML>=3.1.0 # MIT
six>=1.9.0 # MIT
pykube