Browse Source

Merge "Provide a script to gerenate options changes"

changes/79/99079/3
Jenkins 8 years ago committed by Gerrit Code Review
parent
commit
8bfbbf01a1
  1. 1
      .gitignore
  2. 5
      README.rst
  3. 25
      autogenerate_config_docs/autohelp-wrapper
  4. 40
      autogenerate_config_docs/autohelp.py
  5. 283
      autogenerate_config_docs/diff_branches.py

1
.gitignore vendored

@ -26,6 +26,7 @@ ChangeLog
# Autohelp
autogenerate_config_docs/venv
autogenerate_config_docs/sources
autogenerate_config_docs/*-conf-changes-*.xml
# sitemap
sitemap/sitemap_docs.openstack.org.xml

5
README.rst

@ -101,6 +101,11 @@ Release notes
----
* ``openstack-doc-test``: Fix handling of ignore-dir parameter.
* ``autohelp-wrapper``: New tool to simplify the setup of an autohelp.py
environment
* ``diff_branches.py``: Generates a listing of the configuration options
changes that occured between 2 openstack releases.
* ``autohelp.py``: add the 'dump' subcommand
0.15
----

25
autogenerate_config_docs/autohelp-wrapper

@ -35,6 +35,7 @@ usage() {
echo "Options:"
echo " -b BRANCH: Work on this branch (defaults to master)"
echo " -c: Recreate the virtual environment"
echo " -e PATH: Create the virtualenv in PATH"
}
setup_venv() {
@ -59,7 +60,7 @@ setup_tools() {
get_project openstack-manuals
(cd $SOURCESDIR/oslo-incubator && python setup.py install)
pip install GitPython>=0.3.2.RC1
pip install "GitPython>=0.3.2.RC1"
# For some reason the ceilometer installation fails without these 2
# packages pre-installed
@ -86,11 +87,11 @@ setup_tools() {
# late in the icehouse release cycle to let the doc team handle the changes
# properly in the documentation.
if echo $BRANCH | grep -q stable/; then
pip install python-keystoneclient==0.7
pip install "python-keystoneclient==0.7"
fi
}
while getopts :b:m:c opt; do
while getopts :b:e:c opt; do
case $opt in
b)
BRANCH=$OPTARG
@ -100,6 +101,10 @@ while getopts :b:m:c opt; do
rm -rf $VENVDIR
shift
;;
e)
VENVDIR=$OPTARG
shift 2
;;
\?)
usage
exit 1
@ -145,7 +150,16 @@ for project in $PROJECTS; do
python setup.py install
)
cd $MANUALSREPO/tools/autogenerate-config-flagmappings
if [ "$ACTION" = "setup" ]; then
continue
fi
if [ -d $MANUALSREPO/tools/autogenerate-config-flagmappings ]; then
cd $MANUALSREPO/tools/autogenerate-config-flagmappings
else
# for havana
$MANUALSREPO/tools/autogenerate-config-docs
fi
case $ACTION in
update)
@ -155,8 +169,5 @@ for project in $PROJECTS; do
docbook)
$AUTOHELP docbook $project -i $SOURCESDIR/$project
;;
setup)
# The work is already done
;;
esac
done

40
autogenerate_config_docs/autohelp.py

@ -23,6 +23,7 @@ from oslo.config import cfg
import argparse
import importlib
import os
import pickle
import re
import sys
@ -123,6 +124,13 @@ def import_modules(repo_location, package_name, verbose=0):
if verbose >= 2:
print(e)
continue
except cfg.NoSuchGroupError as e:
"""
If a group doesn't exist, we ignore the import.
"""
if verbose >= 2:
print(e)
continue
_register_runtime_opts(module, abs_path, verbose)
_run_hook(modname)
@ -205,6 +213,11 @@ class OptionsCache(object):
if self._verbose >= 2:
print ("Duplicate option name %s" % optname)
else:
if opt.name == 'bindir':
venv = os.environ.get('VIRTUAL_ENV')
if venv is not None and opt.default.startswith(venv):
opt.default = opt.default.replace(venv, '/usr/local')
self._opts_by_name[optname] = (group, opt)
self._opt_names.append(optname)
@ -250,7 +263,8 @@ class OptionsCache(object):
return cmp(x, y)
def write_docbook(package_name, options, verbose=0, target='./'):
def write_docbook(package_name, options, verbose=0,
target='../../doc/common/tables/'):
"""Write DocBook tables.
Prints a docbook-formatted table for every group of options.
@ -323,13 +337,13 @@ def write_docbook(package_name, options, verbose=0, target='./'):
groups_file.close()
def write_docbook_rootwrap(package_name, repo, verbose=0, target='./'):
def write_docbook_rootwrap(package_name, repo, verbose=0,
target='../../doc/common/tables/'):
"""Write a DocBook table for rootwrap options.
Prints a docbook-formatted table for options in a project's
rootwrap.conf configuration file.
"""
# The sample rootwrap.conf path is not the same in all projects. It is
# either in etc/ or in etc/<project>/, so we check both locations.
conffile = os.path.join(repo, 'etc', package_name, 'rootwrap.conf')
@ -445,13 +459,21 @@ def update_flagmappings(package_name, options, verbose=0):
print(line)
def dump_options(options):
"""Dumps the list of options with their attributes.
This output is consumed by the diff_branches script.
"""
print(pickle.dumps(options._opts_by_name))
def main():
parser = argparse.ArgumentParser(
description='Manage flag files, to aid in updating documentation.',
usage='%(prog)s <cmd> <package> [options]')
parser.add_argument('subcommand',
help='Action (create, update, verify).',
choices=['create', 'update', 'docbook'])
help='Action (create, update, verify, dump).',
choices=['create', 'update', 'docbook', 'dump'])
parser.add_argument('package',
help='Name of the top-level package.')
parser.add_argument('-v', '--verbose',
@ -466,9 +488,11 @@ def main():
type=str,)
parser.add_argument('-o', '--output',
dest='target',
help='Directory in which xml files are generated.',
help='Directory or file in which data will be saved.\n'
'Defaults to ../../doc/common/tables/ '
'for "docbook".\n'
'Defaults to stdout for "dump"',
required=False,
default='../../doc/common/tables/',
type=str,)
args = parser.parse_args()
@ -505,6 +529,8 @@ def main():
write_docbook_rootwrap(package_name, args.repo,
verbose=args.verbose,
target=args.target)
elif args.subcommand == 'dump':
dump_options(options)
if __name__ == "__main__":

283
autogenerate_config_docs/diff_branches.py

@ -0,0 +1,283 @@
#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# A collection of tools for working with flags from OpenStack
# packages and documentation.
#
# For an example of usage, run this program with the -h switch.
#
import argparse
import os
import pickle
import subprocess
import sys
import git
from lxml import etree
PROJECTS = ['ceilometer', 'cinder', 'glance', 'heat', 'keystone', 'neutron',
'nova', 'trove']
def setup_venv(branch, novenvupdate):
"""Uses the autohelp-wrapper script to generate a virtualenv for a given
branch.
"""
dirname = os.path.join('venv', branch.replace('/', '_'))
if novenvupdate and os.path.exists(dirname):
return
if not os.path.exists('venv'):
os.mkdir('venv')
args = ["./autohelp-wrapper", "-b", branch, "-e", dirname, "setup"]
if subprocess.call(args) != 0:
print("Impossible to create the %s environment." % branch)
sys.exit(1)
def get_options(project, branch, args):
"""Calls the autohelp script in a venv to get the list of known
options.
"""
print("Working on %(project)s (%(branch)s)" % {'project': project,
'branch': branch})
# Checkout the required branch
repo_path = os.path.join(args.sources, project)
repo = git.Repo(repo_path)
repo.heads[branch].checkout()
# And run autohelp script to get a serialized dict of the discovered
# options
dirname = os.path.abspath(os.path.join('venv', branch.replace('/', '_')))
cmd = ("python autohelp.py dump %(project)s -i %(path)s" %
{'project': project, 'path': repo_path})
path = os.environ.get("PATH")
bin_path = os.path.abspath(os.path.join(dirname, "bin"))
path = "%s:%s" % (bin_path, path)
serialized = subprocess.check_output(cmd, shell=True,
env={'VIRTUAL_ENV': dirname,
'PATH': path})
return pickle.loads(serialized)
def _cmpopts(x, y):
"""Compare to option names.
The options can be of 2 forms: option_name or group/option_name. Options
without a group always comes first. Options are sorted alphabetically
inside a group.
"""
if '/' in x and '/' in y:
prex = x[:x.find('/')]
prey = y[:y.find('/')]
if prex != prey:
return cmp(prex, prey)
return cmp(x, y)
elif '/' in x:
return 1
elif '/' in y:
return -1
else:
return cmp(x, y)
def dbk_append_table(parent, title, cols):
"""Create a docbook table and append it to `parent`.
:param parent: the element to which the table is added
:param title: the table title
:param cols: the number of columns in this table
"""
table = etree.Element("table")
parent.append(table)
caption = etree.Element("caption")
caption.text = title
table.append(caption)
for i in range(cols):
# We cast to int for python 3
width = "%d%%" % int(100 / cols)
table.append(etree.Element("col", width=width))
return table
def dbk_append_row(parent, cells):
"""Append a row to a table.
:param parent: the table
:param cells: a list of strings, one string per column
"""
tr = etree.Element("tr")
for text in cells:
td = etree.Element("td")
td.text = str(text)
tr.append(td)
parent.append(tr)
def dbk_append_header(parent, cells):
"""Append a header to a table.
:param parent: the table
:param cells: a list of strings, one string per column
"""
thead = etree.Element("thead")
dbk_append_row(thead, cells)
parent.append(thead)
def diff(old_list, new_list):
"""Compare the old and new lists of options to generate lists of modified
options.
"""
new_opts = []
changed_default = []
deprecated_opts = []
for name, (group, option) in new_list.items():
# Find the new options
if name not in old_list.viewkeys():
new_opts.append(name)
# Find the options for which the default value has changed
elif option.default != old_list[name][1].default:
changed_default.append(name)
# Find options that have been deprecated in the new release.
# If an option name is a key in the old_list dict, it means that it
# wasn't deprecated.
for deprecated in option.deprecated_opts:
# deprecated_opts is a list which always holds at least 1 invalid
# dict. Forget it.
if deprecated.name is None:
continue
if deprecated.group in [None, 'DEFAULT']:
full_name = deprecated.name
else:
full_name = deprecated.group + '/' + deprecated.name
if full_name in old_list.viewkeys():
deprecated_opts.append((full_name, name))
return new_opts, changed_default, deprecated_opts
def generate_docbook(project, new_branch, old_list, new_list):
"""Generate the diff between the 2 options lists for `project`."""
new_opts, changed_default, deprecated_opts = diff(old_list, new_list)
XMLNS = '{http://www.w3.org/XML/1998/namespace}'
DOCBOOKMAP = {None: "http://docbook.org/ns/docbook"}
section = etree.Element("section", nsmap=DOCBOOKMAP, version="5.0")
id = "%(project)s-conf-changes-%(branch)s" % {'project': project,
'branch': new_branch}
section.set(XMLNS + 'id', id)
section.append(etree.Comment(" Warning: Do not edit this file. It is "
"automatically generated and your changes "
"will be overwritten. The tool to do so "
"lives in the openstack-doc-tools "
"repository. "))
title = etree.Element("title")
title.text = "New, updated and deprecated options for %s" % project
section.append(title)
# New options
table = dbk_append_table(section, "New options", 2)
dbk_append_header(table, ["Option = default value", "(Type) Help string"])
for name in sorted(new_opts, _cmpopts):
opt = new_list[name][1]
type = opt.__class__.__name__.split('.')[-1]
cells = ["%(name)s = %(default)s" % {'name': name,
'default': opt.default},
"(%(type)s) %(help)s" % {'type': type, 'help': opt.help}]
dbk_append_row(table, cells)
table = dbk_append_table(section, "New default values", 3)
dbk_append_header(table, ["Option", "Previous default value",
"New default value"])
for name in sorted(changed_default, _cmpopts):
old_default = old_list[name][1].default
new_default = new_list[name][1].default
if isinstance(old_default, list):
old_default = ", ".join(old_default)
if isinstance(new_default, list):
new_default = ", ".join(new_default)
cells = [name, old_default, new_default]
dbk_append_row(table, cells)
table = dbk_append_table(section, "Deprecated options", 2)
dbk_append_header(table, ["Deprecated option", "New Option"])
for deprecated, new in deprecated_opts:
dbk_append_row(table, [deprecated, new])
return etree.tostring(section, pretty_print=True, xml_declaration=True,
encoding="UTF-8")
def main():
parser = argparse.ArgumentParser(
description='Generate a summary of configuration option changes.',
usage='%(prog)s <old_branch> <new_branch> [options]')
parser.add_argument('old_branch',
help='Name of the old branch.')
parser.add_argument('new_branch',
help='Name of the new branch.')
parser.add_argument('-i', '--input',
dest='sources',
help='Path to a folder containing the git '
'repositories.',
required=False,
default='./sources',
type=str,)
parser.add_argument('-o', '--output',
dest='target',
help='Directory or file in which data will be saved.\n'
'Defaults to "."',
required=False,
default='.',
type=str,)
parser.add_argument('-n', '--no-venv-update',
dest='novenvupdate',
help='Don\'t update the virtual envs.',
required=False,
action='store_true',
default=False,)
args = parser.parse_args()
# Blacklist trove if we diff between havana and icehouse: autohelp.py fails
# with trove on havana
if args.old_branch == "stable/havana":
PROJECTS.remove('trove')
setup_venv(args.old_branch, args.novenvupdate)
setup_venv(args.new_branch, args.novenvupdate)
for project in PROJECTS:
old_list = get_options(project, args.old_branch, args)
new_list = get_options(project, args.new_branch, args)
release = args.new_branch.replace('stable/', '')
xml = generate_docbook(project, release, old_list, new_list)
filename = ("%(project)s-conf-changes-%(release)s.xml" %
{'project': project, 'release': release})
if not os.path.exists(args.target):
os.makedirs(args.target)
dest = os.path.join(args.target, filename)
with open(dest, 'w') as fd:
fd.write(xml)
return 0
if __name__ == "__main__":
sys.exit(main())
Loading…
Cancel
Save