Merge "Remove swift tooling"
This commit is contained in:
commit
121ddfebd1
@ -29,7 +29,7 @@ On Ubuntu:
|
|||||||
libxml2-dev libxslt1-dev zlib1g-dev \
|
libxml2-dev libxslt1-dev zlib1g-dev \
|
||||||
libmysqlclient-dev libpq-dev libffi-dev \
|
libmysqlclient-dev libpq-dev libffi-dev \
|
||||||
libsqlite3-dev libldap2-dev libsasl2-dev \
|
libsqlite3-dev libldap2-dev libsasl2-dev \
|
||||||
libjpeg-dev liberasurecode-dev
|
libjpeg-dev
|
||||||
|
|
||||||
On RHEL 7 and CentOS 7:
|
On RHEL 7 and CentOS 7:
|
||||||
|
|
||||||
@ -41,11 +41,10 @@ On RHEL 7 and CentOS 7:
|
|||||||
libxml2-devel libxslt-devel zlib-devel \
|
libxml2-devel libxslt-devel zlib-devel \
|
||||||
mariadb-devel postgresql-devel libffi-devel \
|
mariadb-devel postgresql-devel libffi-devel \
|
||||||
sqlite-devel openldap-devel cyrus-sasl-devel \
|
sqlite-devel openldap-devel cyrus-sasl-devel \
|
||||||
libjpeg-turbo-devel liberasurecode-devel gcc git
|
libjpeg-turbo-devel gcc git
|
||||||
|
|
||||||
.. note::
|
.. note::
|
||||||
* libjpeg is needed for ironic
|
* libjpeg is needed for ironic
|
||||||
* liberasurecode is needed for swift
|
|
||||||
|
|
||||||
The workflow is:
|
The workflow is:
|
||||||
|
|
||||||
@ -111,19 +110,11 @@ to work on another branch:
|
|||||||
which will be left untouched (no ``git branch``, no ``git update``).
|
which will be left untouched (no ``git branch``, no ``git update``).
|
||||||
|
|
||||||
|
|
||||||
Updating swift options
|
|
||||||
----------------------
|
|
||||||
|
|
||||||
Swift configuration tables are generated using the ``extract_swift_flags.py``
|
|
||||||
script. This script doesn't use a mapping file, but organize the tables using
|
|
||||||
the various configuration files and sections. Most of the options must be
|
|
||||||
described manually at the moment.
|
|
||||||
|
|
||||||
Generate configuration difference
|
Generate configuration difference
|
||||||
---------------------------------
|
---------------------------------
|
||||||
|
|
||||||
To generate "New, updated, and deprecated options" for each service,
|
To generate "New, updated, and deprecated options" for each service,
|
||||||
run diff_branches.py. For example:
|
run ``diff_branches.py``. For example:
|
||||||
|
|
||||||
.. code-block:: console
|
.. code-block:: console
|
||||||
|
|
||||||
|
@ -19,11 +19,10 @@ SOURCESDIR=$HERE/sources
|
|||||||
MANUALSREPO=$SOURCESDIR/openstack-manuals
|
MANUALSREPO=$SOURCESDIR/openstack-manuals
|
||||||
MAPPINGS_DIR=$MANUALSREPO/tools/autogenerate-config-flagmappings
|
MAPPINGS_DIR=$MANUALSREPO/tools/autogenerate-config-flagmappings
|
||||||
AUTOHELP="python $HERE/autohelp.py"
|
AUTOHELP="python $HERE/autohelp.py"
|
||||||
EXTRACT_SWIFT="python $HERE/extract_swift_flags.py"
|
|
||||||
GITBASE=${GITBASE:-git://git.openstack.org/openstack}
|
GITBASE=${GITBASE:-git://git.openstack.org/openstack}
|
||||||
GITPROJ=${GITPROJ:-git://git.openstack.org/openstack}
|
GITPROJ=${GITPROJ:-git://git.openstack.org/openstack}
|
||||||
PROJECTS="aodh ceilometer cinder glance heat ironic keystone manila \
|
PROJECTS="aodh ceilometer cinder glance heat ironic keystone manila \
|
||||||
murano neutron nova sahara senlin swift trove zaqar"
|
murano neutron nova sahara senlin trove zaqar"
|
||||||
MANUALS_PROJECTS="openstack-manuals"
|
MANUALS_PROJECTS="openstack-manuals"
|
||||||
BRANCH=master
|
BRANCH=master
|
||||||
FAST=0
|
FAST=0
|
||||||
@ -254,30 +253,21 @@ for project in $PROJECTS; do
|
|||||||
|
|
||||||
case $ACTION in
|
case $ACTION in
|
||||||
update)
|
update)
|
||||||
[ "$project" = "swift" ] && continue
|
|
||||||
$AUTOHELP update $project -i $SOURCESDIR/$project/$project \
|
$AUTOHELP update $project -i $SOURCESDIR/$project/$project \
|
||||||
$extra_flags $AUTOOPT
|
$extra_flags $AUTOOPT
|
||||||
mv $project.flagmappings.new $project.flagmappings
|
mv $project.flagmappings.new $project.flagmappings
|
||||||
;;
|
;;
|
||||||
rst)
|
rst)
|
||||||
if [ "$project" = "swift" ]; then
|
$AUTOHELP rst $project -i $SOURCESDIR/$project/$project \
|
||||||
$EXTRACT_SWIFT rst -m $MANUALSREPO -s $SOURCESDIR/swift
|
$extra_flags $AUTOOPT
|
||||||
else
|
|
||||||
$AUTOHELP rst $project -i $SOURCESDIR/$project/$project \
|
|
||||||
$extra_flags $AUTOOPT
|
|
||||||
fi
|
|
||||||
;;
|
;;
|
||||||
dump)
|
dump)
|
||||||
if [ $QUIET -eq 1 ]; then
|
if [ $QUIET -eq 1 ]; then
|
||||||
exec 1>&3
|
exec 1>&3
|
||||||
exec 2>&4
|
exec 2>&4
|
||||||
fi
|
fi
|
||||||
if [ "$project" = "swift" ]; then
|
$AUTOHELP dump $project -i $SOURCESDIR/$project/$project \
|
||||||
$EXTRACT_SWIFT dump -m $MANUALSREPO -s $SOURCESDIR/swift
|
$extra_flags $AUTOOPT
|
||||||
else
|
|
||||||
$AUTOHELP dump $project -i $SOURCESDIR/$project/$project \
|
|
||||||
$extra_flags $AUTOOPT
|
|
||||||
fi
|
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
@ -26,8 +26,7 @@ import jinja2
|
|||||||
|
|
||||||
|
|
||||||
PROJECTS = ['aodh', 'ceilometer', 'cinder', 'glance', 'heat', 'ironic',
|
PROJECTS = ['aodh', 'ceilometer', 'cinder', 'glance', 'heat', 'ironic',
|
||||||
'keystone', 'manila', 'neutron', 'nova', 'sahara', 'swift',
|
'keystone', 'manila', 'neutron', 'nova', 'sahara', 'trove']
|
||||||
'trove']
|
|
||||||
MASTER_RELEASE = 'Ocata'
|
MASTER_RELEASE = 'Ocata'
|
||||||
CODENAME_TITLE = {'aodh': 'Alarming',
|
CODENAME_TITLE = {'aodh': 'Alarming',
|
||||||
'ceilometer': 'Telemetry',
|
'ceilometer': 'Telemetry',
|
||||||
@ -42,7 +41,6 @@ CODENAME_TITLE = {'aodh': 'Alarming',
|
|||||||
'nova': 'Compute',
|
'nova': 'Compute',
|
||||||
'sahara': 'Data Processing service',
|
'sahara': 'Data Processing service',
|
||||||
'senlin': 'Clustering service',
|
'senlin': 'Clustering service',
|
||||||
'swift': 'Object Storage service',
|
|
||||||
'trove': 'Database service',
|
'trove': 'Database service',
|
||||||
'zaqar': 'Message service'}
|
'zaqar': 'Message service'}
|
||||||
|
|
||||||
@ -165,14 +163,7 @@ def format_option_name(name):
|
|||||||
# name without a section ('log_dir')
|
# name without a section ('log_dir')
|
||||||
return "[DEFAULT] %s" % name
|
return "[DEFAULT] %s" % name
|
||||||
|
|
||||||
try:
|
return "[%s] %s" % (section, name)
|
||||||
# If we're dealing with swift, we'll have a filename to extract
|
|
||||||
# ('proxy-server|filter:tempurl/use')
|
|
||||||
filename, section = section.split('|')
|
|
||||||
return "%s.conf: [%s] %s" % (filename, section, name)
|
|
||||||
except ValueError:
|
|
||||||
# section but no filename ('database/connection')
|
|
||||||
return "[%s] %s" % (section, name)
|
|
||||||
|
|
||||||
|
|
||||||
def release_from_branch(branch):
|
def release_from_branch(branch):
|
||||||
|
@ -1,273 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
# not use this file except in compliance with the License. You may obtain
|
|
||||||
# a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
||||||
# License for the specific language governing permissions and limitations
|
|
||||||
# under the License.
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import glob
|
|
||||||
import os
|
|
||||||
import pickle
|
|
||||||
import sys
|
|
||||||
|
|
||||||
from docutils import core
|
|
||||||
from docutils import io
|
|
||||||
from docutils import nodes
|
|
||||||
import jinja2
|
|
||||||
from oslo_config import cfg
|
|
||||||
|
|
||||||
from autohelp import OptionsCache # noqa
|
|
||||||
|
|
||||||
# Swift configuration example files live in
|
|
||||||
# swift/etc/*.conf-sample
|
|
||||||
# and contain sections enclosed in [], with
|
|
||||||
# options one per line containing =
|
|
||||||
# and generally only having a single entry
|
|
||||||
# after the equals (the default value)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_line(line):
|
|
||||||
"""Parse a line.
|
|
||||||
|
|
||||||
Takes a line from a swift sample configuration file and attempts
|
|
||||||
to separate the lines with actual configuration option and default
|
|
||||||
value from the rest. Returns None if the line doesn't appear to
|
|
||||||
contain a valid configuration option = default value pair, and
|
|
||||||
a pair of the config and its default if it does.
|
|
||||||
"""
|
|
||||||
|
|
||||||
if '=' not in line:
|
|
||||||
return None
|
|
||||||
temp_line = line.strip('#').strip()
|
|
||||||
config, default = temp_line.split('=', 1)
|
|
||||||
config = config.strip()
|
|
||||||
if ' ' in config and config[0:3] != 'set':
|
|
||||||
if len(default.split()) > 1 or config[0].isupper():
|
|
||||||
return None
|
|
||||||
if len(config) < 2 or '.' in config or '<' in config or '>' in config:
|
|
||||||
return None
|
|
||||||
return config, default.strip()
|
|
||||||
|
|
||||||
|
|
||||||
def get_existing_options(optfiles):
|
|
||||||
"""Parse an existing RST table to compile a list of existing options."""
|
|
||||||
options = {}
|
|
||||||
for optfile in optfiles:
|
|
||||||
input_string = open(optfile).read().replace(':ref:', '')
|
|
||||||
output, pub = core.publish_programmatically(
|
|
||||||
source_class=io.StringInput, source=input_string,
|
|
||||||
source_path=optfile, destination_class=io.NullOutput,
|
|
||||||
destination=None, destination_path='/dev/null', reader=None,
|
|
||||||
reader_name='standalone', parser=None,
|
|
||||||
parser_name='restructuredtext', writer=None, writer_name='null',
|
|
||||||
settings=None, settings_spec=None, settings_overrides=None,
|
|
||||||
config_section=None, enable_exit_status=None)
|
|
||||||
doc = pub.writer.document
|
|
||||||
data = dict(doc.traverse(nodes.row, include_self=False)[1:])
|
|
||||||
for a, b in data.items():
|
|
||||||
option = str(a.traverse(nodes.literal)[0].children[0])
|
|
||||||
helptext = str(b.traverse(nodes.paragraph)[0].children[0])
|
|
||||||
|
|
||||||
if option not in options or 'No help text' in options[option]:
|
|
||||||
# options[option.split('=',1)[0]] = helptext
|
|
||||||
options[option] = helptext.strip()
|
|
||||||
|
|
||||||
return options
|
|
||||||
|
|
||||||
|
|
||||||
def extract_descriptions_from_devref(swift_repo, options):
|
|
||||||
"""Extract descriptions from devref RST files.
|
|
||||||
|
|
||||||
Loop through the devref RST files, looking for lines formatted
|
|
||||||
such that they might contain a description of a particular
|
|
||||||
option.
|
|
||||||
"""
|
|
||||||
|
|
||||||
option_descs = {}
|
|
||||||
rsts = glob.glob(swift_repo + '/doc/source/*.rst')
|
|
||||||
for rst in rsts:
|
|
||||||
rst_file = open(rst, 'r')
|
|
||||||
in_option_block = False
|
|
||||||
prev_option = None
|
|
||||||
for line in rst_file:
|
|
||||||
if 'Option ' in line:
|
|
||||||
in_option_block = True
|
|
||||||
if in_option_block:
|
|
||||||
if '========' in line:
|
|
||||||
in_option_block = False
|
|
||||||
continue
|
|
||||||
if line[0] == ' ' and prev_option is not None:
|
|
||||||
option_descs[prev_option] = (option_descs[prev_option]
|
|
||||||
+ ' ' + line.strip())
|
|
||||||
for option in options:
|
|
||||||
line_parts = line.strip().split(None, 2)
|
|
||||||
if (' ' in line and
|
|
||||||
len(line_parts) == 3 and
|
|
||||||
option == line_parts[0] and
|
|
||||||
line_parts[1] != '=' and
|
|
||||||
option != 'use' and
|
|
||||||
(option not in option_descs or
|
|
||||||
len(option_descs[option]) < len(line_parts[2]))):
|
|
||||||
option_descs[option] = line_parts[2]
|
|
||||||
prev_option = option
|
|
||||||
return option_descs
|
|
||||||
|
|
||||||
|
|
||||||
def write_files(options, manuals_repo):
|
|
||||||
"""Create new reStructuredText tables.
|
|
||||||
|
|
||||||
Writes a set of reStructuredText-formatted tables, one per section in swift
|
|
||||||
configuration files.
|
|
||||||
"""
|
|
||||||
all_options = {}
|
|
||||||
names = options.get_option_names()
|
|
||||||
for full_option in sorted(names, OptionsCache._cmpopts):
|
|
||||||
section, optname = full_option.split('/')
|
|
||||||
oslo_opt = options.get_option(full_option)[1]
|
|
||||||
all_options.setdefault(section, [])
|
|
||||||
|
|
||||||
helptext = oslo_opt.help.strip().replace('\n', ' ')
|
|
||||||
helptext = ' '.join(helptext.split())
|
|
||||||
all_options[section].append((oslo_opt.name,
|
|
||||||
oslo_opt.default,
|
|
||||||
helptext))
|
|
||||||
|
|
||||||
for full_section, options in all_options.items():
|
|
||||||
sample_filename, section = full_section.split('|')
|
|
||||||
tmpl_file = os.path.join(os.path.dirname(__file__),
|
|
||||||
'templates/swift.rst.j2')
|
|
||||||
with open(tmpl_file) as fd:
|
|
||||||
template = jinja2.Template(fd.read(), trim_blocks=True)
|
|
||||||
output = template.render(filename=sample_filename,
|
|
||||||
section=section,
|
|
||||||
options=options)
|
|
||||||
|
|
||||||
tgt = (manuals_repo + '/doc/config-reference/source/tables/' +
|
|
||||||
'swift-' + sample_filename + '-' + section + '.rst')
|
|
||||||
|
|
||||||
with open(tgt, 'w') as fd:
|
|
||||||
fd.write(output)
|
|
||||||
|
|
||||||
|
|
||||||
def read_options(swift_repo, manuals_repo, verbose):
|
|
||||||
"""Find swift configuration options.
|
|
||||||
|
|
||||||
Uses existing tables and swift devref as a source of truth in that order to
|
|
||||||
determine helptext for options found in sample config files.
|
|
||||||
"""
|
|
||||||
|
|
||||||
options = get_existing_options(glob.glob(
|
|
||||||
manuals_repo + '/doc/config-reference/source/tables/swift*rst'))
|
|
||||||
option_descs = extract_descriptions_from_devref(swift_repo, options)
|
|
||||||
conf_samples = glob.glob(swift_repo + '/etc/*conf-sample')
|
|
||||||
for sample in conf_samples:
|
|
||||||
current_section = None
|
|
||||||
sample_file = open(sample, 'r')
|
|
||||||
for line in sample_file:
|
|
||||||
if '[' in line and ']\n' in line and '=' not in line:
|
|
||||||
# It's a header line in the conf file, open a new table file
|
|
||||||
# for this section and close any existing one
|
|
||||||
new_line = line.strip('#').strip()
|
|
||||||
if current_section != new_line:
|
|
||||||
current_section = new_line
|
|
||||||
|
|
||||||
base_section = os.path.basename(sample).split('.conf')[0]
|
|
||||||
extra_section = current_section[1:-1].replace(':', '-')
|
|
||||||
full_section = "%s|%s" % (base_section, extra_section)
|
|
||||||
|
|
||||||
continue
|
|
||||||
|
|
||||||
# All the swift files start with a section, except the rsync
|
|
||||||
# sample. The first items are not important for us.
|
|
||||||
if current_section is None:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# It's a config option line in the conf file, find out the
|
|
||||||
# help text and write to the table file.
|
|
||||||
parsed_line = parse_line(line)
|
|
||||||
if parsed_line is not None:
|
|
||||||
if (parsed_line[0] in options.keys()
|
|
||||||
and 'No help text' not in options[parsed_line[0]]):
|
|
||||||
# use the help text from existing tables
|
|
||||||
option_desc = options[parsed_line[0]]
|
|
||||||
elif parsed_line[0] in option_descs:
|
|
||||||
# use the help text from the devref
|
|
||||||
option_desc = option_descs[parsed_line[0]]
|
|
||||||
else:
|
|
||||||
option_desc = 'No help text available for this option.'
|
|
||||||
if verbose > 0:
|
|
||||||
print(parsed_line[0] + " has no help text")
|
|
||||||
|
|
||||||
# \xa0 is a non-breacking space
|
|
||||||
name = parsed_line[0]
|
|
||||||
option_desc = option_desc.replace(u'\xa0', u' ')
|
|
||||||
default = parsed_line[1]
|
|
||||||
|
|
||||||
o = cfg.StrOpt(name=name, default=default, help=option_desc)
|
|
||||||
try:
|
|
||||||
cfg.CONF.register_opt(o, full_section)
|
|
||||||
except cfg.DuplicateOptError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def dump_options(options):
|
|
||||||
"""Dump the list of options with their attributes.
|
|
||||||
|
|
||||||
This output is consumed by the diff_branches script.
|
|
||||||
"""
|
|
||||||
print(pickle.dumps(options._opts_by_name))
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
"""Parse and write the Swift configuration options."""
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Update the swift options tables.",
|
|
||||||
usage=("%(prog)s rst|dump [-v] [-s swift_repo] "
|
|
||||||
"[-m manuals_repo]"))
|
|
||||||
parser.add_argument('subcommand',
|
|
||||||
help='Action (rst, dump).',
|
|
||||||
choices=['dump', 'rst'])
|
|
||||||
parser.add_argument('-s', '--swift-repo',
|
|
||||||
dest='swift_repo',
|
|
||||||
help="Location of the swift git repository.",
|
|
||||||
required=False,
|
|
||||||
default="./sources/swift")
|
|
||||||
parser.add_argument('-m', '--manuals-repo',
|
|
||||||
dest='manuals_repo',
|
|
||||||
help="Location of the manuals git repository.",
|
|
||||||
required=False,
|
|
||||||
default="./sources/openstack-manuals")
|
|
||||||
parser.add_argument('-v', '--verbose',
|
|
||||||
action='count',
|
|
||||||
default=0,
|
|
||||||
dest='verbose',
|
|
||||||
required=False)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# Avoid cluttering the pickle output, otherwise it's not usable
|
|
||||||
if args.subcommand == 'dump':
|
|
||||||
args.verbose = 0
|
|
||||||
|
|
||||||
read_options(args.swift_repo,
|
|
||||||
args.manuals_repo,
|
|
||||||
args.verbose)
|
|
||||||
options = OptionsCache()
|
|
||||||
|
|
||||||
if args.subcommand == 'rst':
|
|
||||||
write_files(options, args.manuals_repo)
|
|
||||||
else:
|
|
||||||
options.dump()
|
|
||||||
|
|
||||||
return 0
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
@ -1,15 +0,0 @@
|
|||||||
..
|
|
||||||
Warning: Do not edit this file. It is automatically generated and your
|
|
||||||
changes will be overwritten. The tool to do so lives in the
|
|
||||||
openstack-doc-tools repository.
|
|
||||||
|
|
||||||
.. list-table:: Description of configuration options for ``[{{ section }}]`` in ``{{ filename }}.conf``
|
|
||||||
:header-rows: 1
|
|
||||||
:class: config-ref-table
|
|
||||||
|
|
||||||
* - Configuration option = Default value
|
|
||||||
- Description
|
|
||||||
{% for option in options %}
|
|
||||||
* - ``{{ option[0] }}`` = ``{{ option[1]|default(' ', true) }}``
|
|
||||||
- {{ option[2] }}
|
|
||||||
{% endfor %}
|
|
@ -0,0 +1,9 @@
|
|||||||
|
---
|
||||||
|
upgrade:
|
||||||
|
- |
|
||||||
|
The `extract_swift_flags.py` script has been removed, and the `autohelp.py`
|
||||||
|
and `diff_branches.py` scripts no longer build config file documentation
|
||||||
|
for the swift project. The swift dev team had been manually maintaining
|
||||||
|
their config files in-tree and to avoid duplication, doc and swift have
|
||||||
|
agreed to link the config ref out to the dev docs. As such, there is
|
||||||
|
therefore no reason to keep this tooling around.
|
Loading…
Reference in New Issue
Block a user