Provide a script to gerenate options changes
diff_branches.py generates a listing of the configuration options changes that occured between 2 openstack releases. This involves a few changes in other tools: - the 'dump' subcommand for autohelp.py generates the serialized dict of options - add a special case for the 'bindir' option to avoid getting different default values in different virtual environments - the autohelp-wrapper -e switch builds the needed venv without running autohelp.py commands Change-Id: I80da172b91b8d2f0a15f89f4c812864da2fea471
This commit is contained in:
Gauvain Pocentek
parent
db87448679
commit
bdb2f2f003
|
|
@ -26,6 +26,7 @@ ChangeLog
|
|||
# Autohelp
|
||||
autogenerate_config_docs/venv
|
||||
autogenerate_config_docs/sources
|
||||
autogenerate_config_docs/*-conf-changes-*.xml
|
||||
|
||||
# sitemap
|
||||
sitemap/sitemap_docs.openstack.org.xml
|
||||
|
|
|
|||
|
|
@ -101,6 +101,11 @@ Release notes
|
|||
----
|
||||
|
||||
* ``openstack-doc-test``: Fix handling of ignore-dir parameter.
|
||||
* ``autohelp-wrapper``: New tool to simplify the setup of an autohelp.py
|
||||
environment
|
||||
* ``diff_branches.py``: Generates a listing of the configuration options
|
||||
changes that occured between 2 openstack releases.
|
||||
* ``autohelp.py``: add the 'dump' subcommand
|
||||
|
||||
0.15
|
||||
----
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ usage() {
|
|||
echo "Options:"
|
||||
echo " -b BRANCH: Work on this branch (defaults to master)"
|
||||
echo " -c: Recreate the virtual environment"
|
||||
echo " -e PATH: Create the virtualenv in PATH"
|
||||
}
|
||||
|
||||
setup_venv() {
|
||||
|
|
@ -59,7 +60,7 @@ setup_tools() {
|
|||
get_project openstack-manuals
|
||||
|
||||
(cd $SOURCESDIR/oslo-incubator && python setup.py install)
|
||||
pip install GitPython>=0.3.2.RC1
|
||||
pip install "GitPython>=0.3.2.RC1"
|
||||
|
||||
# For some reason the ceilometer installation fails without these 2
|
||||
# packages pre-installed
|
||||
|
|
@ -86,11 +87,11 @@ setup_tools() {
|
|||
# late in the icehouse release cycle to let the doc team handle the changes
|
||||
# properly in the documentation.
|
||||
if echo $BRANCH | grep -q stable/; then
|
||||
pip install python-keystoneclient==0.7
|
||||
pip install "python-keystoneclient==0.7"
|
||||
fi
|
||||
}
|
||||
|
||||
while getopts :b:m:c opt; do
|
||||
while getopts :b:e:c opt; do
|
||||
case $opt in
|
||||
b)
|
||||
BRANCH=$OPTARG
|
||||
|
|
@ -100,6 +101,10 @@ while getopts :b:m:c opt; do
|
|||
rm -rf $VENVDIR
|
||||
shift
|
||||
;;
|
||||
e)
|
||||
VENVDIR=$OPTARG
|
||||
shift 2
|
||||
;;
|
||||
\?)
|
||||
usage
|
||||
exit 1
|
||||
|
|
@ -145,7 +150,16 @@ for project in $PROJECTS; do
|
|||
python setup.py install
|
||||
)
|
||||
|
||||
cd $MANUALSREPO/tools/autogenerate-config-flagmappings
|
||||
if [ "$ACTION" = "setup" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
if [ -d $MANUALSREPO/tools/autogenerate-config-flagmappings ]; then
|
||||
cd $MANUALSREPO/tools/autogenerate-config-flagmappings
|
||||
else
|
||||
# for havana
|
||||
$MANUALSREPO/tools/autogenerate-config-docs
|
||||
fi
|
||||
|
||||
case $ACTION in
|
||||
update)
|
||||
|
|
@ -155,8 +169,5 @@ for project in $PROJECTS; do
|
|||
docbook)
|
||||
$AUTOHELP docbook $project -i $SOURCESDIR/$project
|
||||
;;
|
||||
setup)
|
||||
# The work is already done
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from oslo.config import cfg
|
|||
import argparse
|
||||
import importlib
|
||||
import os
|
||||
import pickle
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
|
@ -123,6 +124,13 @@ def import_modules(repo_location, package_name, verbose=0):
|
|||
if verbose >= 2:
|
||||
print(e)
|
||||
continue
|
||||
except cfg.NoSuchGroupError as e:
|
||||
"""
|
||||
If a group doesn't exist, we ignore the import.
|
||||
"""
|
||||
if verbose >= 2:
|
||||
print(e)
|
||||
continue
|
||||
_register_runtime_opts(module, abs_path, verbose)
|
||||
_run_hook(modname)
|
||||
|
||||
|
|
@ -205,6 +213,11 @@ class OptionsCache(object):
|
|||
if self._verbose >= 2:
|
||||
print ("Duplicate option name %s" % optname)
|
||||
else:
|
||||
if opt.name == 'bindir':
|
||||
venv = os.environ.get('VIRTUAL_ENV')
|
||||
if venv is not None and opt.default.startswith(venv):
|
||||
opt.default = opt.default.replace(venv, '/usr/local')
|
||||
|
||||
self._opts_by_name[optname] = (group, opt)
|
||||
self._opt_names.append(optname)
|
||||
|
||||
|
|
@ -250,7 +263,8 @@ class OptionsCache(object):
|
|||
return cmp(x, y)
|
||||
|
||||
|
||||
def write_docbook(package_name, options, verbose=0, target='./'):
|
||||
def write_docbook(package_name, options, verbose=0,
|
||||
target='../../doc/common/tables/'):
|
||||
"""Write DocBook tables.
|
||||
|
||||
Prints a docbook-formatted table for every group of options.
|
||||
|
|
@ -323,13 +337,13 @@ def write_docbook(package_name, options, verbose=0, target='./'):
|
|||
groups_file.close()
|
||||
|
||||
|
||||
def write_docbook_rootwrap(package_name, repo, verbose=0, target='./'):
|
||||
def write_docbook_rootwrap(package_name, repo, verbose=0,
|
||||
target='../../doc/common/tables/'):
|
||||
"""Write a DocBook table for rootwrap options.
|
||||
|
||||
Prints a docbook-formatted table for options in a project's
|
||||
rootwrap.conf configuration file.
|
||||
"""
|
||||
|
||||
# The sample rootwrap.conf path is not the same in all projects. It is
|
||||
# either in etc/ or in etc/<project>/, so we check both locations.
|
||||
conffile = os.path.join(repo, 'etc', package_name, 'rootwrap.conf')
|
||||
|
|
@ -445,13 +459,21 @@ def update_flagmappings(package_name, options, verbose=0):
|
|||
print(line)
|
||||
|
||||
|
||||
def dump_options(options):
|
||||
"""Dumps the list of options with their attributes.
|
||||
|
||||
This output is consumed by the diff_branches script.
|
||||
"""
|
||||
print(pickle.dumps(options._opts_by_name))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Manage flag files, to aid in updating documentation.',
|
||||
usage='%(prog)s <cmd> <package> [options]')
|
||||
parser.add_argument('subcommand',
|
||||
help='Action (create, update, verify).',
|
||||
choices=['create', 'update', 'docbook'])
|
||||
help='Action (create, update, verify, dump).',
|
||||
choices=['create', 'update', 'docbook', 'dump'])
|
||||
parser.add_argument('package',
|
||||
help='Name of the top-level package.')
|
||||
parser.add_argument('-v', '--verbose',
|
||||
|
|
@ -466,9 +488,11 @@ def main():
|
|||
type=str,)
|
||||
parser.add_argument('-o', '--output',
|
||||
dest='target',
|
||||
help='Directory in which xml files are generated.',
|
||||
help='Directory or file in which data will be saved.\n'
|
||||
'Defaults to ../../doc/common/tables/ '
|
||||
'for "docbook".\n'
|
||||
'Defaults to stdout for "dump"',
|
||||
required=False,
|
||||
default='../../doc/common/tables/',
|
||||
type=str,)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
|
@ -505,6 +529,8 @@ def main():
|
|||
write_docbook_rootwrap(package_name, args.repo,
|
||||
verbose=args.verbose,
|
||||
target=args.target)
|
||||
elif args.subcommand == 'dump':
|
||||
dump_options(options)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -0,0 +1,283 @@
|
|||
#!/usr/bin/env python
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
# A collection of tools for working with flags from OpenStack
|
||||
# packages and documentation.
|
||||
#
|
||||
# For an example of usage, run this program with the -h switch.
|
||||
#
|
||||
import argparse
|
||||
import os
|
||||
import pickle
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import git
|
||||
from lxml import etree
|
||||
|
||||
|
||||
PROJECTS = ['ceilometer', 'cinder', 'glance', 'heat', 'keystone', 'neutron',
|
||||
'nova', 'trove']
|
||||
|
||||
|
||||
def setup_venv(branch, novenvupdate):
|
||||
"""Uses the autohelp-wrapper script to generate a virtualenv for a given
|
||||
branch.
|
||||
"""
|
||||
dirname = os.path.join('venv', branch.replace('/', '_'))
|
||||
if novenvupdate and os.path.exists(dirname):
|
||||
return
|
||||
if not os.path.exists('venv'):
|
||||
os.mkdir('venv')
|
||||
args = ["./autohelp-wrapper", "-b", branch, "-e", dirname, "setup"]
|
||||
if subprocess.call(args) != 0:
|
||||
print("Impossible to create the %s environment." % branch)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def get_options(project, branch, args):
|
||||
"""Calls the autohelp script in a venv to get the list of known
|
||||
options.
|
||||
"""
|
||||
print("Working on %(project)s (%(branch)s)" % {'project': project,
|
||||
'branch': branch})
|
||||
# Checkout the required branch
|
||||
repo_path = os.path.join(args.sources, project)
|
||||
repo = git.Repo(repo_path)
|
||||
repo.heads[branch].checkout()
|
||||
|
||||
# And run autohelp script to get a serialized dict of the discovered
|
||||
# options
|
||||
dirname = os.path.abspath(os.path.join('venv', branch.replace('/', '_')))
|
||||
cmd = ("python autohelp.py dump %(project)s -i %(path)s" %
|
||||
{'project': project, 'path': repo_path})
|
||||
path = os.environ.get("PATH")
|
||||
bin_path = os.path.abspath(os.path.join(dirname, "bin"))
|
||||
path = "%s:%s" % (bin_path, path)
|
||||
serialized = subprocess.check_output(cmd, shell=True,
|
||||
env={'VIRTUAL_ENV': dirname,
|
||||
'PATH': path})
|
||||
return pickle.loads(serialized)
|
||||
|
||||
|
||||
def _cmpopts(x, y):
|
||||
"""Compare to option names.
|
||||
|
||||
The options can be of 2 forms: option_name or group/option_name. Options
|
||||
without a group always comes first. Options are sorted alphabetically
|
||||
inside a group.
|
||||
"""
|
||||
if '/' in x and '/' in y:
|
||||
prex = x[:x.find('/')]
|
||||
prey = y[:y.find('/')]
|
||||
if prex != prey:
|
||||
return cmp(prex, prey)
|
||||
return cmp(x, y)
|
||||
elif '/' in x:
|
||||
return 1
|
||||
elif '/' in y:
|
||||
return -1
|
||||
else:
|
||||
return cmp(x, y)
|
||||
|
||||
|
||||
def dbk_append_table(parent, title, cols):
|
||||
"""Create a docbook table and append it to `parent`.
|
||||
|
||||
:param parent: the element to which the table is added
|
||||
:param title: the table title
|
||||
:param cols: the number of columns in this table
|
||||
"""
|
||||
table = etree.Element("table")
|
||||
parent.append(table)
|
||||
caption = etree.Element("caption")
|
||||
caption.text = title
|
||||
table.append(caption)
|
||||
for i in range(cols):
|
||||
# We cast to int for python 3
|
||||
width = "%d%%" % int(100 / cols)
|
||||
table.append(etree.Element("col", width=width))
|
||||
return table
|
||||
|
||||
|
||||
def dbk_append_row(parent, cells):
|
||||
"""Append a row to a table.
|
||||
|
||||
:param parent: the table
|
||||
:param cells: a list of strings, one string per column
|
||||
"""
|
||||
tr = etree.Element("tr")
|
||||
for text in cells:
|
||||
td = etree.Element("td")
|
||||
td.text = str(text)
|
||||
tr.append(td)
|
||||
parent.append(tr)
|
||||
|
||||
|
||||
def dbk_append_header(parent, cells):
|
||||
"""Append a header to a table.
|
||||
|
||||
:param parent: the table
|
||||
:param cells: a list of strings, one string per column
|
||||
"""
|
||||
thead = etree.Element("thead")
|
||||
dbk_append_row(thead, cells)
|
||||
parent.append(thead)
|
||||
|
||||
|
||||
def diff(old_list, new_list):
|
||||
"""Compare the old and new lists of options to generate lists of modified
|
||||
options.
|
||||
"""
|
||||
new_opts = []
|
||||
changed_default = []
|
||||
deprecated_opts = []
|
||||
for name, (group, option) in new_list.items():
|
||||
# Find the new options
|
||||
if name not in old_list.viewkeys():
|
||||
new_opts.append(name)
|
||||
|
||||
# Find the options for which the default value has changed
|
||||
elif option.default != old_list[name][1].default:
|
||||
changed_default.append(name)
|
||||
|
||||
# Find options that have been deprecated in the new release.
|
||||
# If an option name is a key in the old_list dict, it means that it
|
||||
# wasn't deprecated.
|
||||
for deprecated in option.deprecated_opts:
|
||||
# deprecated_opts is a list which always holds at least 1 invalid
|
||||
# dict. Forget it.
|
||||
if deprecated.name is None:
|
||||
continue
|
||||
|
||||
if deprecated.group in [None, 'DEFAULT']:
|
||||
full_name = deprecated.name
|
||||
else:
|
||||
full_name = deprecated.group + '/' + deprecated.name
|
||||
|
||||
if full_name in old_list.viewkeys():
|
||||
deprecated_opts.append((full_name, name))
|
||||
|
||||
return new_opts, changed_default, deprecated_opts
|
||||
|
||||
|
||||
def generate_docbook(project, new_branch, old_list, new_list):
|
||||
"""Generate the diff between the 2 options lists for `project`."""
|
||||
new_opts, changed_default, deprecated_opts = diff(old_list, new_list)
|
||||
|
||||
XMLNS = '{http://www.w3.org/XML/1998/namespace}'
|
||||
DOCBOOKMAP = {None: "http://docbook.org/ns/docbook"}
|
||||
|
||||
section = etree.Element("section", nsmap=DOCBOOKMAP, version="5.0")
|
||||
id = "%(project)s-conf-changes-%(branch)s" % {'project': project,
|
||||
'branch': new_branch}
|
||||
section.set(XMLNS + 'id', id)
|
||||
section.append(etree.Comment(" Warning: Do not edit this file. It is "
|
||||
"automatically generated and your changes "
|
||||
"will be overwritten. The tool to do so "
|
||||
"lives in the openstack-doc-tools "
|
||||
"repository. "))
|
||||
title = etree.Element("title")
|
||||
title.text = "New, updated and deprecated options for %s" % project
|
||||
section.append(title)
|
||||
|
||||
# New options
|
||||
table = dbk_append_table(section, "New options", 2)
|
||||
dbk_append_header(table, ["Option = default value", "(Type) Help string"])
|
||||
for name in sorted(new_opts, _cmpopts):
|
||||
opt = new_list[name][1]
|
||||
type = opt.__class__.__name__.split('.')[-1]
|
||||
cells = ["%(name)s = %(default)s" % {'name': name,
|
||||
'default': opt.default},
|
||||
"(%(type)s) %(help)s" % {'type': type, 'help': opt.help}]
|
||||
dbk_append_row(table, cells)
|
||||
|
||||
table = dbk_append_table(section, "New default values", 3)
|
||||
dbk_append_header(table, ["Option", "Previous default value",
|
||||
"New default value"])
|
||||
for name in sorted(changed_default, _cmpopts):
|
||||
old_default = old_list[name][1].default
|
||||
new_default = new_list[name][1].default
|
||||
if isinstance(old_default, list):
|
||||
old_default = ", ".join(old_default)
|
||||
if isinstance(new_default, list):
|
||||
new_default = ", ".join(new_default)
|
||||
cells = [name, old_default, new_default]
|
||||
dbk_append_row(table, cells)
|
||||
|
||||
table = dbk_append_table(section, "Deprecated options", 2)
|
||||
dbk_append_header(table, ["Deprecated option", "New Option"])
|
||||
for deprecated, new in deprecated_opts:
|
||||
dbk_append_row(table, [deprecated, new])
|
||||
|
||||
return etree.tostring(section, pretty_print=True, xml_declaration=True,
|
||||
encoding="UTF-8")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Generate a summary of configuration option changes.',
|
||||
usage='%(prog)s <old_branch> <new_branch> [options]')
|
||||
parser.add_argument('old_branch',
|
||||
help='Name of the old branch.')
|
||||
parser.add_argument('new_branch',
|
||||
help='Name of the new branch.')
|
||||
parser.add_argument('-i', '--input',
|
||||
dest='sources',
|
||||
help='Path to a folder containing the git '
|
||||
'repositories.',
|
||||
required=False,
|
||||
default='./sources',
|
||||
type=str,)
|
||||
parser.add_argument('-o', '--output',
|
||||
dest='target',
|
||||
help='Directory or file in which data will be saved.\n'
|
||||
'Defaults to "."',
|
||||
required=False,
|
||||
default='.',
|
||||
type=str,)
|
||||
parser.add_argument('-n', '--no-venv-update',
|
||||
dest='novenvupdate',
|
||||
help='Don\'t update the virtual envs.',
|
||||
required=False,
|
||||
action='store_true',
|
||||
default=False,)
|
||||
args = parser.parse_args()
|
||||
|
||||
# Blacklist trove if we diff between havana and icehouse: autohelp.py fails
|
||||
# with trove on havana
|
||||
if args.old_branch == "stable/havana":
|
||||
PROJECTS.remove('trove')
|
||||
|
||||
setup_venv(args.old_branch, args.novenvupdate)
|
||||
setup_venv(args.new_branch, args.novenvupdate)
|
||||
|
||||
for project in PROJECTS:
|
||||
old_list = get_options(project, args.old_branch, args)
|
||||
new_list = get_options(project, args.new_branch, args)
|
||||
|
||||
release = args.new_branch.replace('stable/', '')
|
||||
xml = generate_docbook(project, release, old_list, new_list)
|
||||
filename = ("%(project)s-conf-changes-%(release)s.xml" %
|
||||
{'project': project, 'release': release})
|
||||
if not os.path.exists(args.target):
|
||||
os.makedirs(args.target)
|
||||
dest = os.path.join(args.target, filename)
|
||||
with open(dest, 'w') as fd:
|
||||
fd.write(xml)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Reference in New Issue