From 390fab8da43339e04dae511b5a8bf4a6235cb8ea Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Mon, 4 May 2015 23:00:25 -0400 Subject: [PATCH] Use oslo-config-generator instead of generate_sample.sh Oslo team is actively getting rid of the old style generator.py in openstack/common/config. So we should switch to the newer oslo-config-generator which is the way forward. For simplicity's sake, i used one single list_opts in opts.py. However we can split this up and add entrypoints in setup.cfg if we decide to do that instead. blueprint oslo-config-generator Change-Id: Ib7c72af7be47df08983708ac6da0ce103518b78c --- etc/nova/nova-config-generator.conf | 23 ++ nova/api/opts.py | 147 ++++++++++ nova/cells/opts.py | 35 +++ nova/compute/opts.py | 44 +++ nova/network/opts.py | 44 +++ nova/openstack/common/config/__init__.py | 0 nova/openstack/common/config/generator.py | 320 ---------------------- nova/opts.py | 140 ++++++++++ nova/scheduler/opts.py | 67 +++++ nova/virt/opts.py | 110 ++++++++ setup.cfg | 15 + tools/config/README | 20 -- tools/config/analyze_opts.py | 81 ------ tools/config/check_uptodate.sh | 25 -- tools/config/generate_sample.sh | 119 -------- tools/config/oslo.config.generator.rc | 2 - tox.ini | 3 +- 17 files changed, 626 insertions(+), 569 deletions(-) create mode 100644 etc/nova/nova-config-generator.conf create mode 100644 nova/api/opts.py create mode 100644 nova/compute/opts.py create mode 100644 nova/network/opts.py delete mode 100644 nova/openstack/common/config/__init__.py delete mode 100644 nova/openstack/common/config/generator.py create mode 100644 nova/opts.py create mode 100644 nova/scheduler/opts.py create mode 100644 nova/virt/opts.py delete mode 100644 tools/config/README delete mode 100755 tools/config/analyze_opts.py delete mode 100755 tools/config/check_uptodate.sh delete mode 100755 tools/config/generate_sample.sh delete mode 100644 tools/config/oslo.config.generator.rc diff --git a/etc/nova/nova-config-generator.conf b/etc/nova/nova-config-generator.conf new file mode 100644 index 000000000000..698a671eeaa4 --- /dev/null +++ b/etc/nova/nova-config-generator.conf @@ -0,0 +1,23 @@ +[DEFAULT] +output_file = etc/nova/nova.conf.sample +wrap_width = 79 +namespace = nova +namespace = nova.api +namespace = nova.cells +namespace = nova.compute +namespace = nova.network +namespace = nova.scheduler +namespace = nova.virt +namespace = nova.openstack.common.eventlet_backdoor +namespace = nova.openstack.common.memorycache +namespace = nova.openstack.common.periodic_task +namespace = nova.openstack.common.policy +namespace = nova.openstack.common.sslutils +namespace = nova.openstack.common.versionutils +namespace = oslo.log +namespace = oslo.messaging +namespace = oslo.policy +namespace = oslo.db +namespace = oslo.middleware +namespace = oslo.concurrency +namespace = keystonemiddleware.auth_token \ No newline at end of file diff --git a/nova/api/opts.py b/nova/api/opts.py new file mode 100644 index 000000000000..7b4ed05b8f87 --- /dev/null +++ b/nova/api/opts.py @@ -0,0 +1,147 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools + +import nova.api.auth +import nova.api.ec2 +import nova.api.ec2.cloud +import nova.api.metadata.base +import nova.api.metadata.handler +import nova.api.metadata.vendordata_json +import nova.api.openstack +import nova.api.openstack.common +import nova.api.openstack.compute +import nova.api.openstack.compute.contrib +import nova.api.openstack.compute.contrib.fping +import nova.api.openstack.compute.contrib.os_tenant_networks +import nova.api.openstack.compute.extensions +import nova.api.openstack.compute.plugins.v3.hide_server_addresses +import nova.api.openstack.compute.servers +import nova.availability_zones +import nova.baserpc +import nova.cells.manager +import nova.cells.messaging +import nova.cells.opts +import nova.cells.rpc_driver +import nova.cells.rpcapi +import nova.cells.scheduler +import nova.cells.state +import nova.cells.weights.mute_child +import nova.cells.weights.ram_by_instance_type +import nova.cells.weights.weight_offset +import nova.cert.rpcapi +import nova.cloudpipe.pipelib +import nova.cmd.novnc +import nova.cmd.novncproxy +import nova.cmd.serialproxy +import nova.cmd.spicehtml5proxy +import nova.compute.api +import nova.compute.flavors +import nova.compute.manager +import nova.compute.monitors +import nova.compute.resource_tracker +import nova.compute.rpcapi +import nova.conductor.api +import nova.conductor.rpcapi +import nova.conductor.tasks.live_migrate +import nova.console.manager +import nova.console.rpcapi +import nova.console.serial +import nova.console.xvp +import nova.consoleauth +import nova.consoleauth.manager +import nova.consoleauth.rpcapi +import nova.crypto +import nova.db.api +import nova.db.base +import nova.db.sqlalchemy.api +import nova.exception +import nova.image.download.file +import nova.image.glance +import nova.image.s3 +import nova.ipv6.api +import nova.keymgr +import nova.keymgr.barbican +import nova.keymgr.conf_key_mgr +import nova.netconf +import nova.network +import nova.network.driver +import nova.network.floating_ips +import nova.network.ldapdns +import nova.network.linux_net +import nova.network.manager +import nova.network.neutronv2.api +import nova.network.rpcapi +import nova.network.security_group.openstack_driver +import nova.notifications +import nova.objects.network +import nova.objectstore.s3server +import nova.paths +import nova.pci.request +import nova.pci.whitelist +import nova.quota +import nova.rdp +import nova.scheduler.driver +import nova.scheduler.filter_scheduler +import nova.scheduler.filters.aggregate_image_properties_isolation +import nova.scheduler.filters.core_filter +import nova.scheduler.filters.disk_filter +import nova.scheduler.filters.io_ops_filter +import nova.scheduler.filters.isolated_hosts_filter +import nova.scheduler.filters.num_instances_filter +import nova.scheduler.filters.ram_filter +import nova.scheduler.filters.trusted_filter +import nova.scheduler.host_manager +import nova.scheduler.ironic_host_manager +import nova.scheduler.manager +import nova.scheduler.rpcapi +import nova.scheduler.scheduler_options +import nova.scheduler.utils +import nova.scheduler.weights.io_ops +import nova.scheduler.weights.metrics +import nova.scheduler.weights.ram +import nova.service +import nova.servicegroup.api +import nova.servicegroup.drivers.zk +import nova.spice +import nova.utils +import nova.vnc +import nova.vnc.xvp_proxy +import nova.volume +import nova.volume.cinder +import nova.wsgi + + +def list_opts(): + return [ + ('DEFAULT', + itertools.chain( + [nova.api.metadata.vendordata_json.file_opt], + [nova.api.openstack.compute.allow_instance_snapshots_opt], + nova.api.auth.auth_opts, + nova.api.ec2.cloud.ec2_opts, + nova.api.ec2.ec2_opts, + nova.api.metadata.base.metadata_opts, + nova.api.metadata.handler.metadata_opts, + nova.api.openstack.common.osapi_opts, + nova.api.openstack.compute.contrib.ext_opts, + nova.api.openstack.compute.contrib.fping.fping_opts, + nova.api.openstack.compute.contrib.os_tenant_networks. + os_network_opts, + nova.api.openstack.compute.extensions.ext_opts, + nova.api.openstack.compute.plugins.v3.hide_server_addresses.opts, + nova.api.openstack.compute.servers.server_opts, + )), + ('neutron', nova.api.metadata.handler.metadata_proxy_opts), + ('osapi_v3', nova.api.openstack.api_opts), + ] diff --git a/nova/cells/opts.py b/nova/cells/opts.py index 3b8225b0d743..4cd2f21a493b 100644 --- a/nova/cells/opts.py +++ b/nova/cells/opts.py @@ -16,7 +16,10 @@ Global cells config options """ +import itertools + from oslo_config import cfg +from oslo_utils import importutils cells_opts = [ @@ -66,3 +69,35 @@ def get_cell_type(): if not CONF.cells.enable: return return CONF.cells.cell_type + + +def list_opts(): + return [ + ('cells', + itertools.chain( + cells_opts, + importutils.import_module( + "nova.cells.manager").cell_manager_opts, + importutils.import_module( + "nova.cells.messaging").cell_messaging_opts, + importutils.import_module( + "nova.cells.rpc_driver").cell_rpc_driver_opts, + importutils.import_module( + "nova.cells.scheduler").cell_scheduler_opts, + importutils.import_module( + "nova.cells.state").cell_state_manager_opts, + importutils.import_module( + "nova.cells.weights.mute_child").mute_weigher_opts, + importutils.import_module( + "nova.cells.weights.ram_by_instance_type").ram_weigher_opts, + importutils.import_module( + "nova.cells.weights.weight_offset").weigher_opts + )), + ('upgrade_levels', + itertools.chain( + [importutils.import_module( + "nova.cells.rpc_driver").rpcapi_cap_opt], + [importutils.import_module( + "nova.cells.rpcapi").rpcapi_cap_opt], + )), + ] diff --git a/nova/compute/opts.py b/nova/compute/opts.py new file mode 100644 index 000000000000..b8bfe5734e7a --- /dev/null +++ b/nova/compute/opts.py @@ -0,0 +1,44 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools + +import nova.compute.api +import nova.compute.flavors +import nova.compute.manager +import nova.compute.monitors +import nova.compute.resource_tracker +import nova.compute.rpcapi + + +def list_opts(): + return [ + ('DEFAULT', + itertools.chain( + nova.compute.api.compute_opts, + nova.compute.flavors.flavor_opts, + nova.compute.manager.compute_opts, + nova.compute.manager.instance_cleaning_opts, + nova.compute.manager.interval_opts, + nova.compute.manager.running_deleted_opts, + nova.compute.manager.timeout_opts, + nova.compute.monitors.compute_monitors_opts, + nova.compute.resource_tracker.resource_tracker_opts, + nova.compute.rpcapi.rpcapi_opts, + )), + ('ephemeral_storage_encryption', + nova.compute.api.ephemeral_storage_encryption_opts), + ('upgrade_levels', + itertools.chain( + [nova.compute.rpcapi.rpcapi_cap_opt], + )), + ] diff --git a/nova/network/opts.py b/nova/network/opts.py new file mode 100644 index 000000000000..ac3560c503e8 --- /dev/null +++ b/nova/network/opts.py @@ -0,0 +1,44 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools + +import nova.network +import nova.network.driver +import nova.network.floating_ips +import nova.network.ldapdns +import nova.network.linux_net +import nova.network.manager +import nova.network.neutronv2.api +import nova.network.rpcapi +import nova.network.security_group.openstack_driver + + +def list_opts(): + return [ + ('DEFAULT', + itertools.chain( + nova.network._network_opts, + nova.network.driver.driver_opts, + nova.network.floating_ips.floating_opts, + nova.network.ldapdns.ldap_dns_opts, + nova.network.linux_net.linux_net_opts, + nova.network.manager.network_opts, + nova.network.rpcapi.rpcapi_opts, + nova.network.security_group.openstack_driver.security_group_opts, + )), + ('neutron', nova.network.neutronv2.api.neutron_opts), + ('upgrade_levels', + itertools.chain( + [nova.network.rpcapi.rpcapi_cap_opt], + )), + ] diff --git a/nova/openstack/common/config/__init__.py b/nova/openstack/common/config/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/nova/openstack/common/config/generator.py b/nova/openstack/common/config/generator.py deleted file mode 100644 index a4b2ab593850..000000000000 --- a/nova/openstack/common/config/generator.py +++ /dev/null @@ -1,320 +0,0 @@ -# Copyright 2012 SINA Corporation -# Copyright 2014 Cisco Systems, Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# - -"""Extracts OpenStack config option info from module(s).""" - -from __future__ import print_function - -import argparse -import imp -import os -import re -import socket -import sys -import textwrap - -from oslo_config import cfg -from oslo_utils import importutils -import six -import stevedore.named - -STROPT = "StrOpt" -BOOLOPT = "BoolOpt" -INTOPT = "IntOpt" -FLOATOPT = "FloatOpt" -LISTOPT = "ListOpt" -DICTOPT = "DictOpt" -MULTISTROPT = "MultiStrOpt" - -OPT_TYPES = { - STROPT: 'string value', - BOOLOPT: 'boolean value', - INTOPT: 'integer value', - FLOATOPT: 'floating point value', - LISTOPT: 'list value', - DICTOPT: 'dict value', - MULTISTROPT: 'multi valued', -} - -OPTION_REGEX = re.compile(r"(%s)" % "|".join([STROPT, BOOLOPT, INTOPT, - FLOATOPT, LISTOPT, DICTOPT, - MULTISTROPT])) - -PY_EXT = ".py" -BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__), - "../../../../")) -WORDWRAP_WIDTH = 60 - - -def raise_extension_exception(extmanager, ep, err): - raise - - -def generate(argv): - parser = argparse.ArgumentParser( - description='generate sample configuration file', - ) - parser.add_argument('-m', dest='modules', action='append') - parser.add_argument('-l', dest='libraries', action='append') - parser.add_argument('srcfiles', nargs='*') - parsed_args = parser.parse_args(argv) - - mods_by_pkg = dict() - for filepath in parsed_args.srcfiles: - pkg_name = filepath.split(os.sep)[1] - mod_str = '.'.join(['.'.join(filepath.split(os.sep)[:-1]), - os.path.basename(filepath).split('.')[0]]) - mods_by_pkg.setdefault(pkg_name, list()).append(mod_str) - # NOTE(lzyeval): place top level modules before packages - pkg_names = sorted(pkg for pkg in mods_by_pkg if pkg.endswith(PY_EXT)) - ext_names = sorted(pkg for pkg in mods_by_pkg if pkg not in pkg_names) - pkg_names.extend(ext_names) - - # opts_by_group is a mapping of group name to an options list - # The options list is a list of (module, options) tuples - opts_by_group = {'DEFAULT': []} - - if parsed_args.modules: - for module_name in parsed_args.modules: - module = _import_module(module_name) - if module: - for group, opts in _list_opts(module): - opts_by_group.setdefault(group, []).append((module_name, - opts)) - - # Look for entry points defined in libraries (or applications) for - # option discovery, and include their return values in the output. - # - # Each entry point should be a function returning an iterable - # of pairs with the group name (or None for the default group) - # and the list of Opt instances for that group. - if parsed_args.libraries: - loader = stevedore.named.NamedExtensionManager( - 'oslo_config.opts', - names=list(set(parsed_args.libraries)), - invoke_on_load=False, - on_load_failure_callback=raise_extension_exception - ) - for ext in loader: - for group, opts in ext.plugin(): - opt_list = opts_by_group.setdefault(group or 'DEFAULT', []) - opt_list.append((ext.name, opts)) - - for pkg_name in pkg_names: - mods = mods_by_pkg.get(pkg_name) - mods.sort() - for mod_str in mods: - if mod_str.endswith('.__init__'): - mod_str = mod_str[:mod_str.rfind(".")] - - mod_obj = _import_module(mod_str) - if not mod_obj: - raise RuntimeError("Unable to import module %s" % mod_str) - - for group, opts in _list_opts(mod_obj): - opts_by_group.setdefault(group, []).append((mod_str, opts)) - - print_group_opts('DEFAULT', opts_by_group.pop('DEFAULT', [])) - for group in sorted(opts_by_group.keys()): - print_group_opts(group, opts_by_group[group]) - - -def _import_module(mod_str): - try: - if mod_str.startswith('bin.'): - imp.load_source(mod_str[4:], os.path.join('bin', mod_str[4:])) - return sys.modules[mod_str[4:]] - else: - return importutils.import_module(mod_str) - except Exception as e: - sys.stderr.write("Error importing module %s: %s\n" % (mod_str, str(e))) - return None - - -def _is_in_group(opt, group): - "Check if opt is in group." - for value in group._opts.values(): - # NOTE(llu): Temporary workaround for bug #1262148, wait until - # newly released oslo.config support '==' operator. - if not(value['opt'] != opt): - return True - return False - - -def _guess_groups(opt, mod_obj): - # is it in the DEFAULT group? - if _is_in_group(opt, cfg.CONF): - return 'DEFAULT' - - # what other groups is it in? - for value in cfg.CONF.values(): - if isinstance(value, cfg.CONF.GroupAttr): - if _is_in_group(opt, value._group): - return value._group.name - - raise RuntimeError( - "Unable to find group for option %s, " - "maybe it's defined twice in the same group?" - % opt.name - ) - - -def _list_opts(obj): - def is_opt(o): - return (isinstance(o, cfg.Opt) and - not isinstance(o, cfg.SubCommandOpt)) - - opts = list() - - if 'list_opts' in dir(obj): - return getattr(obj, 'list_opts')() - - for attr_str in dir(obj): - attr_obj = getattr(obj, attr_str) - if is_opt(attr_obj): - opts.append(attr_obj) - elif (isinstance(attr_obj, list) and - all(map(lambda x: is_opt(x), attr_obj))): - opts.extend(attr_obj) - - ret = {} - for opt in opts: - ret.setdefault(_guess_groups(opt, obj), []).append(opt) - return ret.items() - - -def print_group_opts(group, opts_by_module): - print("[%s]" % group) - print('') - for mod, opts in opts_by_module: - print('#') - print('# Options defined in %s' % mod) - print('#') - print('') - for opt in opts: - _print_opt(opt) - print('') - - -def _get_my_ip(): - try: - csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - csock.connect(('8.8.8.8', 80)) - (addr, port) = csock.getsockname() - csock.close() - return addr - except socket.error: - return None - - -def _sanitize_default(name, value): - """Set up a reasonably sensible default for pybasedir, my_ip and host.""" - if value.startswith(sys.prefix): - # NOTE(jd) Don't use os.path.join, because it is likely to think the - # second part is an absolute pathname and therefore drop the first - # part. - value = os.path.normpath("/usr/" + value[len(sys.prefix):]) - elif value.startswith(BASEDIR): - return value.replace(BASEDIR, '/usr/lib/python/site-packages') - elif BASEDIR in value: - return value.replace(BASEDIR, '') - elif value == _get_my_ip(): - return '10.0.0.1' - elif value in (socket.gethostname(), socket.getfqdn()) and 'host' in name: - return 'nova' - elif value.strip() != value: - return '"%s"' % value - return value - - -def _get_choice_text(choice): - if choice is None: - return '' - elif choice == '': - return "''" - return six.text_type(choice) - - -def _print_opt(opt): - opt_name, opt_default, opt_help = opt.dest, opt.default, opt.help - if not opt_help: - sys.stderr.write('WARNING: "%s" is missing help string.\n' % opt_name) - opt_help = "" - opt_type = None - try: - opt_type = OPTION_REGEX.search(str(type(opt))).group(0) - except (ValueError, AttributeError) as err: - sys.stderr.write("%s\n" % str(err)) - sys.exit(1) - opt_help = u'%s (%s)' % (opt_help, - OPT_TYPES[opt_type]) - print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH))) - if opt.deprecated_opts: - for deprecated_opt in opt.deprecated_opts: - if deprecated_opt.name: - deprecated_group = (deprecated_opt.group if - deprecated_opt.group else "DEFAULT") - print('# Deprecated group/name - [%s]/%s' % - (deprecated_group, - deprecated_opt.name)) - try: - if opt_default is None: - print('#%s=' % opt_name) - elif opt_type == STROPT: - assert(isinstance(opt_default, six.string_types)) - if (getattr(opt, 'type', None) and - getattr(opt.type, 'choices', None)): - choices_text = ', '.join([_get_choice_text(choice) - for choice in opt.type.choices]) - print('# Allowed values: %s' % choices_text) - print('#%s=%s' % (opt_name, _sanitize_default(opt_name, - opt_default))) - elif opt_type == BOOLOPT: - assert(isinstance(opt_default, bool)) - print('#%s=%s' % (opt_name, str(opt_default).lower())) - elif opt_type == INTOPT: - assert(isinstance(opt_default, int) and - not isinstance(opt_default, bool)) - print('#%s=%s' % (opt_name, opt_default)) - elif opt_type == FLOATOPT: - assert(isinstance(opt_default, float)) - print('#%s=%s' % (opt_name, opt_default)) - elif opt_type == LISTOPT: - assert(isinstance(opt_default, list)) - print('#%s=%s' % (opt_name, ','.join(opt_default))) - elif opt_type == DICTOPT: - assert(isinstance(opt_default, dict)) - opt_default_strlist = [str(key) + ':' + str(value) - for (key, value) in opt_default.items()] - print('#%s=%s' % (opt_name, ','.join(opt_default_strlist))) - elif opt_type == MULTISTROPT: - assert(isinstance(opt_default, list)) - if not opt_default: - opt_default = [''] - for default in opt_default: - print('#%s=%s' % (opt_name, default)) - print('') - except Exception: - sys.stderr.write('Error in option "%s"\n' % opt_name) - sys.exit(1) - - -def main(): - generate(sys.argv[1:]) - -if __name__ == '__main__': - main() diff --git a/nova/opts.py b/nova/opts.py new file mode 100644 index 000000000000..ea72717a3174 --- /dev/null +++ b/nova/opts.py @@ -0,0 +1,140 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools + +import nova.availability_zones +import nova.baserpc +import nova.cert.rpcapi +import nova.cloudpipe.pipelib +import nova.cmd.novnc +import nova.cmd.novncproxy +import nova.cmd.serialproxy +import nova.cmd.spicehtml5proxy +import nova.conductor.api +import nova.conductor.rpcapi +import nova.conductor.tasks.live_migrate +import nova.console.manager +import nova.console.rpcapi +import nova.console.serial +import nova.console.xvp +import nova.consoleauth +import nova.consoleauth.manager +import nova.consoleauth.rpcapi +import nova.crypto +import nova.db.api +import nova.db.base +import nova.db.sqlalchemy.api +import nova.exception +import nova.image.download.file +import nova.image.glance +import nova.image.s3 +import nova.ipv6.api +import nova.keymgr +import nova.keymgr.barbican +import nova.keymgr.conf_key_mgr +import nova.netconf +import nova.notifications +import nova.objects.network +import nova.objectstore.s3server +import nova.paths +import nova.pci.request +import nova.pci.whitelist +import nova.quota +import nova.rdp +import nova.service +import nova.servicegroup.api +import nova.servicegroup.drivers.zk +import nova.spice +import nova.utils +import nova.vnc +import nova.vnc.xvp_proxy +import nova.volume +import nova.volume.cinder +import nova.wsgi + + +def list_opts(): + return [ + ('DEFAULT', + itertools.chain( + [nova.conductor.tasks.live_migrate.migrate_opt], + [nova.consoleauth.consoleauth_topic_opt], + [nova.db.base.db_driver_opt], + [nova.ipv6.api.ipv6_backend_opt], + [nova.servicegroup.api.servicegroup_driver_opt], + nova.availability_zones.availability_zone_opts, + nova.cert.rpcapi.rpcapi_opts, + nova.cloudpipe.pipelib.cloudpipe_opts, + nova.cmd.novnc.opts, + nova.cmd.novncproxy.opts, + nova.cmd.serialproxy.opts, + nova.cmd.spicehtml5proxy.opts, + nova.console.manager.console_manager_opts, + nova.console.rpcapi.rpcapi_opts, + nova.console.xvp.xvp_opts, + nova.consoleauth.manager.consoleauth_opts, + nova.crypto.crypto_opts, + nova.db.api.db_opts, + nova.db.sqlalchemy.api.db_opts, + nova.exception.exc_log_opts, + nova.image.s3.s3_opts, + nova.netconf.netconf_opts, + nova.notifications.notify_opts, + nova.objects.network.network_opts, + nova.objectstore.s3server.s3_opts, + nova.paths.path_opts, + nova.pci.request.pci_alias_opts, + nova.pci.whitelist.pci_opts, + nova.quota.quota_opts, + nova.service.service_opts, + nova.utils.monkey_patch_opts, + nova.utils.utils_opts, + nova.vnc.vnc_opts, + nova.vnc.xvp_proxy.xvp_proxy_opts, + nova.volume._volume_opts, + nova.wsgi.wsgi_opts, + )), + ('barbican', nova.keymgr.barbican.barbican_opts), + ('cinder', nova.volume.cinder.cinder_opts), + ('api_database', nova.db.sqlalchemy.api.api_db_opts), + ('conductor', nova.conductor.api.conductor_opts), + ('database', nova.db.sqlalchemy.api.oslo_db_options.database_opts), + ('glance', nova.image.glance.glance_opts), + ('image_file_url', [nova.image.download.file.opt_group]), + ('keymgr', + itertools.chain( + nova.keymgr.conf_key_mgr.key_mgr_opts, + nova.keymgr.keymgr_opts, + )), + ('rdp', nova.rdp.rdp_opts), + ('serial_console', + itertools.chain( + nova.cmd.serialproxy.opts, + nova.console.serial.serial_opts, + )), + ('spice', + itertools.chain( + nova.cmd.spicehtml5proxy.opts, + nova.spice.spice_opts, + )), + ('upgrade_levels', + itertools.chain( + [nova.baserpc.rpcapi_cap_opt], + [nova.cert.rpcapi.rpcapi_cap_opt], + [nova.conductor.rpcapi.rpcapi_cap_opt], + [nova.console.rpcapi.rpcapi_cap_opt], + [nova.consoleauth.rpcapi.rpcapi_cap_opt], + )), + ('workarounds', nova.utils.workarounds_opts), + ('zookeeper', nova.servicegroup.drivers.zk.zk_driver_opts) + ] diff --git a/nova/scheduler/opts.py b/nova/scheduler/opts.py new file mode 100644 index 000000000000..dea1f6477028 --- /dev/null +++ b/nova/scheduler/opts.py @@ -0,0 +1,67 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools + +import nova.scheduler.driver +import nova.scheduler.filter_scheduler +import nova.scheduler.filters.aggregate_image_properties_isolation +import nova.scheduler.filters.core_filter +import nova.scheduler.filters.disk_filter +import nova.scheduler.filters.io_ops_filter +import nova.scheduler.filters.isolated_hosts_filter +import nova.scheduler.filters.num_instances_filter +import nova.scheduler.filters.ram_filter +import nova.scheduler.filters.trusted_filter +import nova.scheduler.host_manager +import nova.scheduler.ironic_host_manager +import nova.scheduler.manager +import nova.scheduler.rpcapi +import nova.scheduler.scheduler_options +import nova.scheduler.utils +import nova.scheduler.weights.io_ops +import nova.scheduler.weights.metrics +import nova.scheduler.weights.ram + + +def list_opts(): + return [ + ('DEFAULT', + itertools.chain( + [nova.scheduler.filters.core_filter.cpu_allocation_ratio_opt], + [nova.scheduler.filters.disk_filter.disk_allocation_ratio_opt], + [nova.scheduler.filters.io_ops_filter.max_io_ops_per_host_opt], + [nova.scheduler.filters.num_instances_filter. + max_instances_per_host_opt], + [nova.scheduler.filters.ram_filter.ram_allocation_ratio_opt], + [nova.scheduler.scheduler_options. + scheduler_json_config_location_opt], + nova.scheduler.driver.scheduler_driver_opts, + nova.scheduler.filter_scheduler.filter_scheduler_opts, + nova.scheduler.filters.aggregate_image_properties_isolation.opts, + nova.scheduler.filters.isolated_hosts_filter.isolated_opts, + nova.scheduler.host_manager.host_manager_opts, + nova.scheduler.ironic_host_manager.host_manager_opts, + nova.scheduler.manager.scheduler_driver_opts, + nova.scheduler.rpcapi.rpcapi_opts, + nova.scheduler.utils.scheduler_opts, + nova.scheduler.weights.io_ops.io_ops_weight_opts, + nova.scheduler.weights.ram.ram_weight_opts, + )), + ('metrics', nova.scheduler.weights.metrics.metrics_weight_opts), + ('trusted_computing', + nova.scheduler.filters.trusted_filter.trusted_opts), + ('upgrade_levels', + itertools.chain( + [nova.scheduler.rpcapi.rpcapi_cap_opt], + )), + ] diff --git a/nova/virt/opts.py b/nova/virt/opts.py new file mode 100644 index 000000000000..4b2cbd4c4544 --- /dev/null +++ b/nova/virt/opts.py @@ -0,0 +1,110 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools + +import nova.virt.configdrive +import nova.virt.disk.api +import nova.virt.disk.mount.nbd +import nova.virt.disk.vfs.guestfs +import nova.virt.driver +import nova.virt.firewall +import nova.virt.hardware +import nova.virt.hyperv.pathutils +import nova.virt.hyperv.utilsfactory +import nova.virt.hyperv.vif +import nova.virt.hyperv.vmops +import nova.virt.hyperv.volumeops +import nova.virt.imagecache +import nova.virt.images +import nova.virt.ironic.driver +import nova.virt.libvirt.driver +import nova.virt.libvirt.imagebackend +import nova.virt.libvirt.imagecache +import nova.virt.libvirt.lvm +import nova.virt.libvirt.utils +import nova.virt.libvirt.vif +import nova.virt.libvirt.volume +import nova.virt.netutils +import nova.virt.vmwareapi.driver +import nova.virt.vmwareapi.images +import nova.virt.vmwareapi.vif +import nova.virt.vmwareapi.vim_util +import nova.virt.vmwareapi.vm_util +import nova.virt.vmwareapi.vmops +import nova.virt.xenapi.agent +import nova.virt.xenapi.client.session +import nova.virt.xenapi.driver +import nova.virt.xenapi.image.bittorrent +import nova.virt.xenapi.pool +import nova.virt.xenapi.vif +import nova.virt.xenapi.vm_utils +import nova.virt.xenapi.vmops +import nova.virt.xenapi.volume_utils + + +def list_opts(): + return [ + ('DEFAULT', + itertools.chain( + nova.virt.configdrive.configdrive_opts, + nova.virt.disk.api.disk_opts, + nova.virt.disk.mount.nbd.nbd_opts, + nova.virt.driver.driver_opts, + nova.virt.firewall.firewall_opts, + nova.virt.hardware.virt_cpu_opts, + nova.virt.imagecache.imagecache_opts, + nova.virt.images.image_opts, + nova.virt.netutils.netutils_opts, + )), + ('guestfs', nova.virt.disk.vfs.guestfs.guestfs_opts), + ('hyperv', + itertools.chain( + nova.virt.hyperv.pathutils.hyperv_opts, + nova.virt.hyperv.utilsfactory.hyper_opts, + nova.virt.hyperv.vif.hyperv_opts, + nova.virt.hyperv.vmops.hyperv_opts, + nova.virt.hyperv.volumeops.hyper_volumeops_opts, + )), + ('ironic', nova.virt.ironic.driver.opts), + ('libvirt', + itertools.chain( + nova.virt.libvirt.driver.libvirt_opts, + nova.virt.libvirt.imagebackend.__imagebackend_opts, + nova.virt.libvirt.imagecache.imagecache_opts, + nova.virt.libvirt.lvm.lvm_opts, + nova.virt.libvirt.utils.libvirt_opts, + nova.virt.libvirt.vif.libvirt_vif_opts, + nova.virt.libvirt.volume.volume_opts, + )), + ('vmware', + itertools.chain( + [nova.virt.vmwareapi.vim_util.vmware_opts], + nova.virt.vmwareapi.driver.spbm_opts, + nova.virt.vmwareapi.driver.vmwareapi_opts, + nova.virt.vmwareapi.vif.vmwareapi_vif_opts, + nova.virt.vmwareapi.vm_util.vmware_utils_opts, + nova.virt.vmwareapi.vmops.vmops_opts, + )), + ('xenserver', + itertools.chain( + [nova.virt.xenapi.vif.xenapi_ovs_integration_bridge_opt], + nova.virt.xenapi.agent.xenapi_agent_opts, + nova.virt.xenapi.client.session.xenapi_session_opts, + nova.virt.xenapi.driver.xenapi_opts, + nova.virt.xenapi.image.bittorrent.xenapi_torrent_opts, + nova.virt.xenapi.pool.xenapi_pool_opts, + nova.virt.xenapi.vm_utils.xenapi_vm_utils_opts, + nova.virt.xenapi.vmops.xenapi_vmops_opts, + nova.virt.xenapi.volume_utils.xenapi_volume_utils_opts, + )), + ] diff --git a/setup.cfg b/setup.cfg index e7d1d692231a..2c94e874dac4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,6 +26,21 @@ packages = nova [entry_points] +oslo.config.opts = + nova = nova.opts:list_opts + nova.api = nova.api.opts:list_opts + nova.cells = nova.cells.opts:list_opts + nova.compute = nova.compute.opts:list_opts + nova.network = nova.network.opts:list_opts + nova.scheduler = nova.scheduler.opts:list_opts + nova.virt = nova.virt.opts:list_opts + nova.openstack.common.eventlet_backdoor = nova.openstack.common.eventlet_backdoor:list_opts + nova.openstack.common.memorycache = nova.openstack.common.memorycache:list_opts + nova.openstack.common.periodic_task = nova.openstack.common.periodic_task:list_opts + nova.openstack.common.policy = nova.openstack.common.policy:list_opts + nova.openstack.common.sslutils = nova.openstack.common.sslutils:list_opts + nova.openstack.common.versionutils = nova.openstack.common.versionutils:list_opts + nova.compute.resources = vcpu = nova.compute.resources.vcpu:VCPU nova.image.download.modules = diff --git a/tools/config/README b/tools/config/README deleted file mode 100644 index 0d5bd5740363..000000000000 --- a/tools/config/README +++ /dev/null @@ -1,20 +0,0 @@ -This generate_sample.sh tool is used to generate etc/nova/nova.conf.sample - -Run it from the top-level working directory i.e. - - $> ./tools/config/generate_sample.sh -b ./ -p nova -o etc/nova - -Watch out for warnings about modules like libvirt, qpid and zmq not -being found - these warnings are significant because they result -in options not appearing in the generated config file. - - -The analyze_opts.py tool is used to find options which appear in -/etc/nova/nova.conf but not in etc/nova/nova.conf.sample -This helps identify options in the nova.conf file which are not used by nova. -The tool also identifies any options which are set to the default value. - -Run it from the top-level working directory i.e. - - $> ./tools/config/analyze_opts.py - diff --git a/tools/config/analyze_opts.py b/tools/config/analyze_opts.py deleted file mode 100755 index a2481203ca8e..000000000000 --- a/tools/config/analyze_opts.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2012, Cloudscaling -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -''' -find_unused_options.py - -Compare the nova.conf file with the nova.conf.sample file to find any unused -options or default values in nova.conf -''' - -from __future__ import print_function - -import argparse -import os -import sys - -sys.path.append(os.getcwd()) -from oslo_config import iniparser - - -class PropertyCollecter(iniparser.BaseParser): - def __init__(self): - super(PropertyCollecter, self).__init__() - self.key_value_pairs = {} - - def assignment(self, key, value): - self.key_value_pairs[key] = value - - def new_section(self, section): - pass - - @classmethod - def collect_properties(cls, lineiter, sample_format=False): - def clean_sample(f): - for line in f: - if line.startswith("#") and not line.startswith("# "): - line = line[1:] - yield line - pc = cls() - if sample_format: - lineiter = clean_sample(lineiter) - pc.parse(lineiter) - return pc.key_value_pairs - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='''Compare the nova.conf - file with the nova.conf.sample file to find any unused options or - default values in nova.conf''') - - parser.add_argument('-c', action='store', - default='/etc/nova/nova.conf', - help='path to nova.conf\ - (defaults to /etc/nova/nova.conf)') - parser.add_argument('-s', default='./etc/nova/nova.conf.sample', - help='path to nova.conf.sample\ - (defaults to ./etc/nova/nova.conf.sample') - options = parser.parse_args() - - conf_file_options = PropertyCollecter.collect_properties(open(options.c)) - sample_conf_file_options = PropertyCollecter.collect_properties( - open(options.s), sample_format=True) - - for k, v in sorted(conf_file_options.items()): - if k not in sample_conf_file_options: - print("Unused:", k) - for k, v in sorted(conf_file_options.items()): - if k in sample_conf_file_options and v == sample_conf_file_options[k]: - print("Default valued:", k) diff --git a/tools/config/check_uptodate.sh b/tools/config/check_uptodate.sh deleted file mode 100755 index 48add47875ef..000000000000 --- a/tools/config/check_uptodate.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -PROJECT_NAME=${PROJECT_NAME:-nova} -CFGFILE_NAME=${PROJECT_NAME}.conf.sample - -if [ -e etc/${PROJECT_NAME}/${CFGFILE_NAME} ]; then - CFGFILE=etc/${PROJECT_NAME}/${CFGFILE_NAME} -elif [ -e etc/${CFGFILE_NAME} ]; then - CFGFILE=etc/${CFGFILE_NAME} -else - echo "${0##*/}: can not find config file" - exit 1 -fi - -TEMPDIR=`mktemp -d /tmp/${PROJECT_NAME}.XXXXXX` -trap "rm -rf $TEMPDIR" EXIT - -tools/config/generate_sample.sh -b ./ -p ${PROJECT_NAME} -o ${TEMPDIR} - -if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE} -then - echo "${0##*/}: ${PROJECT_NAME}.conf.sample is not up to date." - echo "${0##*/}: Please run ${0%%${0##*/}}generate_sample.sh." - exit 1 -fi diff --git a/tools/config/generate_sample.sh b/tools/config/generate_sample.sh deleted file mode 100755 index 1de1b7ebcd11..000000000000 --- a/tools/config/generate_sample.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env bash - -print_hint() { - echo "Try \`${0##*/} --help' for more information." >&2 -} - -PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:m:l:o: \ - --long help,base-dir:,package-name:,output-dir:,module:,library: -- "$@") - -if [ $? != 0 ] ; then print_hint ; exit 1 ; fi - -eval set -- "$PARSED_OPTIONS" - -while true; do - case "$1" in - -h|--help) - echo "${0##*/} [options]" - echo "" - echo "options:" - echo "-h, --help show brief help" - echo "-b, --base-dir=DIR project base directory" - echo "-p, --package-name=NAME project package name" - echo "-o, --output-dir=DIR file output directory" - echo "-m, --module=MOD extra python module to interrogate for options" - echo "-l, --library=LIB extra library that registers options for discovery" - exit 0 - ;; - -b|--base-dir) - shift - BASEDIR=`echo $1 | sed -e 's/\/*$//g'` - shift - ;; - -p|--package-name) - shift - PACKAGENAME=`echo $1` - shift - ;; - -o|--output-dir) - shift - OUTPUTDIR=`echo $1 | sed -e 's/\/*$//g'` - shift - ;; - -m|--module) - shift - MODULES="$MODULES -m $1" - shift - ;; - -l|--library) - shift - LIBRARIES="$LIBRARIES -l $1" - shift - ;; - --) - break - ;; - esac -done - -BASEDIR=${BASEDIR:-`pwd`} -if ! [ -d $BASEDIR ] -then - echo "${0##*/}: missing project base directory" >&2 ; print_hint ; exit 1 -elif [[ $BASEDIR != /* ]] -then - BASEDIR=$(cd "$BASEDIR" && pwd) -fi - -PACKAGENAME=${PACKAGENAME:-${BASEDIR##*/}} -TARGETDIR=$BASEDIR/$PACKAGENAME -if ! [ -d $TARGETDIR ] -then - echo "${0##*/}: invalid project package name" >&2 ; print_hint ; exit 1 -fi - -OUTPUTDIR=${OUTPUTDIR:-$BASEDIR/etc} -# NOTE(bnemec): Some projects put their sample config in etc/, -# some in etc/$PACKAGENAME/ -if [ -d $OUTPUTDIR/$PACKAGENAME ] -then - OUTPUTDIR=$OUTPUTDIR/$PACKAGENAME -elif ! [ -d $OUTPUTDIR ] -then - echo "${0##*/}: cannot access \`$OUTPUTDIR': No such file or directory" >&2 - exit 1 -fi - -BASEDIRESC=`echo $BASEDIR | sed -e 's/\//\\\\\//g'` -find $TARGETDIR -type f -name "*.pyc" -delete -FILES=$(find $TARGETDIR -type f -name "*.py" ! -path "*/tests/*" \ - -exec grep -l "Opt(" {} + | sed -e "s/^$BASEDIRESC\///g" | sort -u) - -RC_FILE="`dirname $0`/oslo.config.generator.rc" -if test -r "$RC_FILE" -then - source "$RC_FILE" -fi - -for mod in ${NOVA_CONFIG_GENERATOR_EXTRA_MODULES}; do - MODULES="$MODULES -m $mod" -done - -for lib in ${NOVA_CONFIG_GENERATOR_EXTRA_LIBRARIES}; do - LIBRARIES="$LIBRARIES -l $lib" -done - -export EVENTLET_NO_GREENDNS=yes - -OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs) -[ "$OS_VARS" ] && eval "unset \$OS_VARS" -DEFAULT_CONFIG_GENERATOR=nova.openstack.common.config.generator -CONFIG_GENERATOR=${CONFIG_GENERATOR:-$DEFAULT_CONFIG_GENERATOR} -OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample -python -m $CONFIG_GENERATOR $MODULES $LIBRARIES $FILES > $OUTPUTFILE - -# Hook to allow projects to append custom config file snippets -CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null) -for CONCAT_FILE in $CONCAT_FILES; do - cat $CONCAT_FILE >> $OUTPUTFILE -done diff --git a/tools/config/oslo.config.generator.rc b/tools/config/oslo.config.generator.rc deleted file mode 100644 index f59f28a4d629..000000000000 --- a/tools/config/oslo.config.generator.rc +++ /dev/null @@ -1,2 +0,0 @@ -NOVA_CONFIG_GENERATOR_EXTRA_LIBRARIES="oslo.messaging oslo_db oslo.concurrency" -NOVA_CONFIG_GENERATOR_EXTRA_MODULES=keystonemiddleware.auth_token diff --git a/tox.ini b/tox.ini index 04a3ec0f295d..5a78887200e2 100644 --- a/tox.ini +++ b/tox.ini @@ -57,8 +57,7 @@ commands = bash tools/pretty_tox.sh '{posargs}' [testenv:genconfig] -commands = - bash tools/config/generate_sample.sh -b . -p nova -o etc/nova +commands = oslo-config-generator --config-file=etc/nova/nova-config-generator.conf [testenv:cover] # Also do not run test_coverage_ext tests while gathering coverage as those