From e436173ae2f4f9dd88910b75a0151a717b6c498a Mon Sep 17 00:00:00 2001 From: Gauvain Pocentek Date: Fri, 8 Jan 2016 15:31:25 +0100 Subject: [PATCH] Move the config items in the manuals repository Modifying configuration options (ignored modules, extra repositories, hooks) for autohelp requires an update of the doc-tools repository, which doesn't make much sense since the goal is to update the manuals repository. This patch removes the configuration options from the repository and updates the scripts to use the configuration items in openstack-manuals. Change-Id: I630d2a2fea690f6360039aa100331678468d18dc --- autogenerate_config_docs/autohelp-wrapper | 23 ++-- autogenerate_config_docs/autohelp.ignore | 19 ---- autogenerate_config_docs/autohelp.py | 11 +- .../extra_repos/neutron-master.txt | 3 - autogenerate_config_docs/hooks.py | 106 ------------------ .../conf-in-manuals-f2d57e04ee35741f.yaml | 4 + .../update-can-create-097dd0790567eeb5.yaml | 2 +- .../requirements/cinder-juno.txt | 2 - .../requirements/neutron-juno.txt | 1 - .../requirements/neutron-kilo.txt | 43 ------- 10 files changed, 23 insertions(+), 191 deletions(-) delete mode 100644 autogenerate_config_docs/autohelp.ignore delete mode 100644 autogenerate_config_docs/extra_repos/neutron-master.txt delete mode 100644 autogenerate_config_docs/hooks.py create mode 100644 autogenerate_config_docs/releasenotes/notes/conf-in-manuals-f2d57e04ee35741f.yaml delete mode 100644 autogenerate_config_docs/requirements/cinder-juno.txt delete mode 100644 autogenerate_config_docs/requirements/neutron-juno.txt delete mode 100644 autogenerate_config_docs/requirements/neutron-kilo.txt diff --git a/autogenerate_config_docs/autohelp-wrapper b/autogenerate_config_docs/autohelp-wrapper index e98e5ffe..9ebc0879 100755 --- a/autogenerate_config_docs/autohelp-wrapper +++ b/autogenerate_config_docs/autohelp-wrapper @@ -17,6 +17,7 @@ HERE=$(pwd) VENVDIR=$HERE/venv SOURCESDIR=$HERE/sources MANUALSREPO=$SOURCESDIR/openstack-manuals +MAPPINGS_DIR=$MANUALSREPO/tools/autogenerate-config-flagmappings AUTOHELP="python $HERE/autohelp.py" EXTRACT_SWIFT="python $HERE/extract_swift_flags.py" GITBASE=git://git.openstack.org/openstack @@ -75,10 +76,10 @@ get_project() { fi git clone $git_url/$project $SOURCESDIR/$project - if [ -e extra_repos/$project-$RELEASE.txt ]; then + if [ -e $MAPPINGS_DIR/$project.extra_repos ]; then while read extra; do git clone $git_url/$extra $SOURCESDIR/$extra - done < extra_repos/$project-$RELEASE.txt + done < $MAPPINGS_DIR/$project.extra_repos fi else @@ -86,10 +87,10 @@ get_project() { (cd $SOURCESDIR/$project && git pull) fi - if [ -e extra_repos/$project-$RELEASE.txt ]; then + if [ -e $MAPPINGS_DIR/$project.extra_repos ]; then while read extra; do (cd $SOURCESDIR/$extra && git pull) - done < extra_repos/$project-$RELEASE.txt + done < $MAPPINGS_DIR/$project.extra_repos fi fi } @@ -162,8 +163,8 @@ if [ "$FAST" -eq 0 ] ; then for project in $PROJECTS; do setup_venv $project setup_tools - if [ -e requirements/$project-$RELEASE.txt ]; then - pip install -r requirements/$project-$RELEASE.txt \ + if [ -e $MAPPINGS_DIR/$project.requirements ]; then + pip install -r $MAPPINGS_DIR/$project.requirements \ --allow-all-external fi get_project $project @@ -183,7 +184,7 @@ if [ "$FAST" -eq 0 ] ; then python setup.py install popd - if [ -e extra_repos/$project-$RELEASE.txt ]; then + if [ -e $MAPPINGS_DIR/$project.extra_repos ]; then while read extra; do ( cd $SOURCESDIR/$extra @@ -191,7 +192,7 @@ if [ "$FAST" -eq 0 ] ; then [ -e "test-requirements.txt" ] && pip install -rtest-requirements.txt python setup.py install ) - done < extra_repos/$project-$RELEASE.txt + done < $MAPPINGS_DIR/$project.extra_repos fi ) done @@ -204,7 +205,7 @@ for project in $PROJECTS; do break fi - if [ -e extra_repos/$project-$RELEASE.txt ]; then + if [ -e $MAPPINGS_DIR/$project.extra_repos ]; then extra_flags= while read extra; do package=$(echo $extra | tr - _) @@ -223,10 +224,10 @@ for project in $PROJECTS; do cp -r $SOURCESDIR/networking-zvm/neutron/plugins/ml2/drivers/zvm $SOURCESDIR/neutron/neutron/plugins/ml2/drivers fi extra_flags="$extra_flags -i $SOURCESDIR/$extra/$package" - done < extra_repos/$project-$RELEASE.txt + done < $MAPPINGS_DIR/$project.extra_repos fi - cd $MANUALSREPO/tools/autogenerate-config-flagmappings + cd $MAPPINGS_DIR case $ACTION in update) diff --git a/autogenerate_config_docs/autohelp.ignore b/autogenerate_config_docs/autohelp.ignore deleted file mode 100644 index 0a429a5d..00000000 --- a/autogenerate_config_docs/autohelp.ignore +++ /dev/null @@ -1,19 +0,0 @@ -cinder.wsgi.wsgi -designate.api.v1.extensions.quotas -designate.api.v1.extensions.reports -designate.api.v1.extensions.sync -designate.api.v1.extensions.touch -glance.contrib.plugins.artifacts_sample.setup -glance.contrib.plugins.image_artifact.setup -keystone.assignment.role_backends.sql -keystone.assignment.V8_backends.sql -manila.db.migrations.alembic.env -trove.guestagent.datastore.experimental.postgresql.manager -trove.guestagent.datastore.experimental.postgresql.service.database -trove.guestagent.datastore.experimental.postgresql.service.root -trove.guestagent.datastore.experimental.postgresql.service.users -trove.guestagent.datastore.postgresql.manager -trove.guestagent.datastore.postgresql.service.database -trove.guestagent.datastore.postgresql.service.root -trove.guestagent.datastore.postgresql.service.users -zaqar.transport.wsgi.app diff --git a/autogenerate_config_docs/autohelp.py b/autogenerate_config_docs/autohelp.py index b13b749d..28b0679e 100755 --- a/autogenerate_config_docs/autohelp.py +++ b/autogenerate_config_docs/autohelp.py @@ -30,6 +30,7 @@ import sys import jinja2 import stevedore +sys.path.insert(0, '.') from hooks import HOOKS # noqa @@ -42,10 +43,6 @@ EXTENSIONS = ['oslo.cache', 'oslo.policy', 'oslo.service'] -IGNORE = open(os.path.join(os.path.dirname(__file__), - 'autohelp.ignore')).read().split('\n') - - register_re = re.compile(r'''^ +.*\.register_opts\((?P[^,)]+)''' r'''(, (group=)?["'](?P.*)["'])?\)''') @@ -57,6 +54,10 @@ def import_modules(repo_location, package_name, verbose=0): populate the configuration object (cfg.CONF) created from Oslo. """ + with open('ignore.list') as fd: + ignore_list = [l for l in fd.read().split('\n') + if l and (l[0] != '#')] + pkg_location = os.path.join(repo_location, package_name) for root, dirs, files in os.walk(pkg_location): skipdir = False @@ -77,7 +78,7 @@ def import_modules(repo_location, package_name, verbose=0): modname = '.'.join(modname) if modname.endswith('.__init__'): modname = modname[:modname.rfind(".")] - if modname in IGNORE: + if modname in ignore_list: continue try: module = importlib.import_module(modname) diff --git a/autogenerate_config_docs/extra_repos/neutron-master.txt b/autogenerate_config_docs/extra_repos/neutron-master.txt deleted file mode 100644 index 38a2ceb4..00000000 --- a/autogenerate_config_docs/extra_repos/neutron-master.txt +++ /dev/null @@ -1,3 +0,0 @@ -neutron-fwaas -neutron-lbaas -neutron-vpnaas diff --git a/autogenerate_config_docs/hooks.py b/autogenerate_config_docs/hooks.py deleted file mode 100644 index 8747de94..00000000 --- a/autogenerate_config_docs/hooks.py +++ /dev/null @@ -1,106 +0,0 @@ -# -# A collection of shared functions for managing help flag mapping files. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -"""Hooks to handle configuration options not handled on module import or with a -call to _register_runtime_opts(). The HOOKS dict associate hook functions with -a module path.""" - - -def aodh_config(): - # Aodh uses a local conf object, therefore we need to use the same method - # here to populate the global cfg.CONF object used by the script. - import aodh.opts as opts - from oslo_config import cfg - - cfg.CONF = cfg.ConfigOpts() - for group, options in opts.list_opts(): - cfg.CONF.register_opts(list(options), - group=None if group == "DEFAULT" else group) - - -def glance_store_config(): - try: - import glance_store - from oslo_config import cfg - - glance_store.backend.register_opts(cfg.CONF) - except ImportError: - # glance_store is not available before Juno - pass - - -def keystone_config(): - from keystone.common import config - - config.configure() - - -def neutron_misc(): - try: - # These imports are needed for kilo only - import bsnstacklib.plugins.bigswitch.config - import networking_cisco.plugins.cisco.cfg_agent.device_status # noqa - import networking_l2gw.services.l2gateway.common.config as l2gw - import networking_vsphere.common.config - from oslo_config import cfg - import vmware_nsx.neutron.plugins.vmware.common.config # noqa - - bsnstacklib.plugins.bigswitch.config.register_config() - networking_vsphere.common.config.register_options() - l2gw.register_l2gw_opts_helper() - l2gw.register_ovsdb_opts_helper(cfg.CONF) - except Exception: - pass - - -def nova_spice(): - import os - # nova.cmd.__init__ before kilo requires to be imported before eventlet is. - # Since we can't make sure of that, we define this envvar to let nova know - # that the import is OK (see nova/cmd/__init__.py) - os.environ['EVENTLET_NO_GREENDNS'] = 'yes' - import nova.cmd.spicehtml5proxy # noqa - - -def zaqar_config(): - # Zaqar registers most of it's options in lazy way - # Let's find all these options and manually register them - from oslo_config import cfg - import pkg_resources - - zaqar_opts_sets = [] - for p in pkg_resources.iter_entry_points(group='oslo.config.opts'): - if p.name.startswith('zaqar'): - zaqar_opts_sets.append(p.load()) - - for opts_set in zaqar_opts_sets: - for group, options in opts_set(): - for opt in options: - try: - cfg.CONF.register_opt(opt, group=group) - except cfg.DuplicateOptError: - pass - - # Let's unregister options that were automatically registered from embedded - # zaqar-bench tool during import of Zaqar's modules - from zaqar.bench import config as bench_config - cfg.CONF.unregister_opts(bench_config._CLI_OPTIONS) - -HOOKS = {'aodh': aodh_config, - 'glance.common.config': glance_store_config, - 'keystone.common.config': keystone_config, - 'neutron': neutron_misc, - 'nova.spice': nova_spice, - 'zaqar': zaqar_config} diff --git a/autogenerate_config_docs/releasenotes/notes/conf-in-manuals-f2d57e04ee35741f.yaml b/autogenerate_config_docs/releasenotes/notes/conf-in-manuals-f2d57e04ee35741f.yaml new file mode 100644 index 00000000..c6803dc7 --- /dev/null +++ b/autogenerate_config_docs/releasenotes/notes/conf-in-manuals-f2d57e04ee35741f.yaml @@ -0,0 +1,4 @@ +--- +other: + - The configuration items for autohelp are now in the openstack-manuals + repository. diff --git a/autogenerate_config_docs/releasenotes/notes/update-can-create-097dd0790567eeb5.yaml b/autogenerate_config_docs/releasenotes/notes/update-can-create-097dd0790567eeb5.yaml index 77322caa..dea2aa29 100644 --- a/autogenerate_config_docs/releasenotes/notes/update-can-create-097dd0790567eeb5.yaml +++ b/autogenerate_config_docs/releasenotes/notes/update-can-create-097dd0790567eeb5.yaml @@ -1,3 +1,3 @@ --- other: - - ``autohelp.py update`` will create the flagmappings file if it doesn't exist yet. + - autohelp.py update will create the flagmappings file if it doesn't exist yet. diff --git a/autogenerate_config_docs/requirements/cinder-juno.txt b/autogenerate_config_docs/requirements/cinder-juno.txt deleted file mode 100644 index 81994279..00000000 --- a/autogenerate_config_docs/requirements/cinder-juno.txt +++ /dev/null @@ -1,2 +0,0 @@ -hp3parclient -hplefthandclient diff --git a/autogenerate_config_docs/requirements/neutron-juno.txt b/autogenerate_config_docs/requirements/neutron-juno.txt deleted file mode 100644 index 8f7e407e..00000000 --- a/autogenerate_config_docs/requirements/neutron-juno.txt +++ /dev/null @@ -1 +0,0 @@ -ryu diff --git a/autogenerate_config_docs/requirements/neutron-kilo.txt b/autogenerate_config_docs/requirements/neutron-kilo.txt deleted file mode 100644 index cff5090f..00000000 --- a/autogenerate_config_docs/requirements/neutron-kilo.txt +++ /dev/null @@ -1,43 +0,0 @@ -ryu - -# lbaas -kemptech-openstack-lbaas -a10-neutron-lbaas -neutron_radware_lbaas -neutron_radware_lbaas_v2 - -# for vmware --e git://git.openstack.org/openstack/nova.git@stable/kilo#egg=nova - -# for a bunch of plugins --e git://git.openstack.org/openstack/neutron.git@stable/kilo#egg=neutron --e git://git.openstack.org/openstack/neutron-fwaas.git@stable/kilo#egg=neutron-fwaas --e git://git.openstack.org/openstack/neutron-lbaas.git@stable/kilo#egg=neutron-lbaas --e git://git.openstack.org/openstack/neutron-vpnaas.git@stable/kilo#egg=neutron-vpnaas - -# for midonet --e git://github.com/midonet/python-midonetclient.git#egg=midonetclient - --e git+https://git.openstack.org/openstack/networking-arista.git@stable/kilo#egg=networking-arista --e git+https://git.openstack.org/openstack/networking-bagpipe.git@stable/kilo#egg=networking-bagpipe --e git+https://git.openstack.org/openstack/networking-bgpvpn.git#egg=networking-bgpvpn --e git+https://git.openstack.org/stackforge/networking-bigswitch.git#egg=networking-bigswitch --e git+https://git.openstack.org/stackforge/networking-brocade.git#egg=networking-brocade --e git+https://git.openstack.org/openstack/networking-cisco.git@stable/kilo#egg=networking-cisco --e git+https://git.openstack.org/stackforge/networking-edge-vpn.git#egg=networking-edge-vpn --e git+https://git.openstack.org/stackforge/networking-hyperv.git@stable/kilo#egg=networking-hyperv --e git+https://git.openstack.org/stackforge/networking-ibm.git#egg=networking-ibm --e git+https://git.openstack.org/openstack/networking-l2gw.git@stable/kilo#egg=networking-l2gw -# pip dependency installation failure --e git+https://git.openstack.org/openstack/networking-midonet.git@stable/kilo#egg=networking-midonet --e git+https://git.openstack.org/stackforge/networking-mlnx.git@stable/kilo#egg=networking-mlnx --e git+https://git.openstack.org/stackforge/networking-nec.git@stable/kilo#egg=networking-nec --e git+https://git.openstack.org/openstack/networking-odl.git@stable/kilo#egg=networking-odl --e git+https://git.openstack.org/openstack/networking-ofagent.git@stable/kilo#egg=networking-ofagent --e git+https://git.openstack.org/openstack/networking-ovn.git#egg=networking-ovn --e git+https://git.openstack.org/stackforge/networking-ovs-dpdk.git@stable/kilo#egg=networking-ovs-dpdk --e git+https://git.openstack.org/openstack/networking-plumgrid.git#egg=networking-plumgrid --e git+https://git.openstack.org/stackforge/networking-portforwarding.git#egg=networking-portforwarding --e git+https://git.openstack.org/openstack/networking-vsphere.git@stable/kilo#egg=networking-vsphere --e git+https://git.openstack.org/openstack/networking-powervm.git#egg=networking-powervm --e git+https://git.openstack.org/openstack/vmware-nsx.git@stable/kilo#egg=vmware-nsx