diff --git a/autogenerate_config_docs/README.rst b/autogenerate_config_docs/README.rst index 9cb05f24..201d2644 100644 --- a/autogenerate_config_docs/README.rst +++ b/autogenerate_config_docs/README.rst @@ -29,7 +29,7 @@ On Ubuntu: libxml2-dev libxslt1-dev zlib1g-dev \ libmysqlclient-dev libpq-dev libffi-dev \ libsqlite3-dev libldap2-dev libsasl2-dev \ - libjpeg-dev liberasurecode-dev + libjpeg-dev On RHEL 7 and CentOS 7: @@ -41,11 +41,10 @@ On RHEL 7 and CentOS 7: libxml2-devel libxslt-devel zlib-devel \ mariadb-devel postgresql-devel libffi-devel \ sqlite-devel openldap-devel cyrus-sasl-devel \ - libjpeg-turbo-devel liberasurecode-devel gcc git + libjpeg-turbo-devel gcc git .. note:: * libjpeg is needed for ironic - * liberasurecode is needed for swift The workflow is: @@ -111,19 +110,11 @@ to work on another branch: which will be left untouched (no ``git branch``, no ``git update``). -Updating swift options ----------------------- - -Swift configuration tables are generated using the ``extract_swift_flags.py`` -script. This script doesn't use a mapping file, but organize the tables using -the various configuration files and sections. Most of the options must be -described manually at the moment. - Generate configuration difference --------------------------------- To generate "New, updated, and deprecated options" for each service, -run diff_branches.py. For example: +run ``diff_branches.py``. For example: .. code-block:: console diff --git a/autogenerate_config_docs/autohelp-wrapper b/autogenerate_config_docs/autohelp-wrapper index 0fc61cea..da643005 100755 --- a/autogenerate_config_docs/autohelp-wrapper +++ b/autogenerate_config_docs/autohelp-wrapper @@ -19,11 +19,10 @@ SOURCESDIR=$HERE/sources MANUALSREPO=$SOURCESDIR/openstack-manuals MAPPINGS_DIR=$MANUALSREPO/tools/autogenerate-config-flagmappings AUTOHELP="python $HERE/autohelp.py" -EXTRACT_SWIFT="python $HERE/extract_swift_flags.py" GITBASE=${GITBASE:-git://git.openstack.org/openstack} GITPROJ=${GITPROJ:-git://git.openstack.org/openstack} PROJECTS="aodh ceilometer cinder glance heat ironic keystone manila \ - murano neutron nova sahara senlin swift trove zaqar" + murano neutron nova sahara senlin trove zaqar" MANUALS_PROJECTS="openstack-manuals" BRANCH=master FAST=0 @@ -254,30 +253,21 @@ for project in $PROJECTS; do case $ACTION in update) - [ "$project" = "swift" ] && continue $AUTOHELP update $project -i $SOURCESDIR/$project/$project \ $extra_flags $AUTOOPT mv $project.flagmappings.new $project.flagmappings ;; rst) - if [ "$project" = "swift" ]; then - $EXTRACT_SWIFT rst -m $MANUALSREPO -s $SOURCESDIR/swift - else - $AUTOHELP rst $project -i $SOURCESDIR/$project/$project \ - $extra_flags $AUTOOPT - fi + $AUTOHELP rst $project -i $SOURCESDIR/$project/$project \ + $extra_flags $AUTOOPT ;; dump) if [ $QUIET -eq 1 ]; then exec 1>&3 exec 2>&4 fi - if [ "$project" = "swift" ]; then - $EXTRACT_SWIFT dump -m $MANUALSREPO -s $SOURCESDIR/swift - else - $AUTOHELP dump $project -i $SOURCESDIR/$project/$project \ - $extra_flags $AUTOOPT - fi + $AUTOHELP dump $project -i $SOURCESDIR/$project/$project \ + $extra_flags $AUTOOPT ;; esac done diff --git a/autogenerate_config_docs/diff_branches.py b/autogenerate_config_docs/diff_branches.py index 2fd3b2e4..356df504 100755 --- a/autogenerate_config_docs/diff_branches.py +++ b/autogenerate_config_docs/diff_branches.py @@ -26,8 +26,7 @@ import jinja2 PROJECTS = ['aodh', 'ceilometer', 'cinder', 'glance', 'heat', 'ironic', - 'keystone', 'manila', 'neutron', 'nova', 'sahara', 'swift', - 'trove'] + 'keystone', 'manila', 'neutron', 'nova', 'sahara', 'trove'] MASTER_RELEASE = 'Ocata' CODENAME_TITLE = {'aodh': 'Alarming', 'ceilometer': 'Telemetry', @@ -42,7 +41,6 @@ CODENAME_TITLE = {'aodh': 'Alarming', 'nova': 'Compute', 'sahara': 'Data Processing service', 'senlin': 'Clustering service', - 'swift': 'Object Storage service', 'trove': 'Database service', 'zaqar': 'Message service'} @@ -165,14 +163,7 @@ def format_option_name(name): # name without a section ('log_dir') return "[DEFAULT] %s" % name - try: - # If we're dealing with swift, we'll have a filename to extract - # ('proxy-server|filter:tempurl/use') - filename, section = section.split('|') - return "%s.conf: [%s] %s" % (filename, section, name) - except ValueError: - # section but no filename ('database/connection') - return "[%s] %s" % (section, name) + return "[%s] %s" % (section, name) def release_from_branch(branch): diff --git a/autogenerate_config_docs/extract_swift_flags.py b/autogenerate_config_docs/extract_swift_flags.py deleted file mode 100644 index 2e4969ba..00000000 --- a/autogenerate_config_docs/extract_swift_flags.py +++ /dev/null @@ -1,273 +0,0 @@ -#!/usr/bin/env python -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import argparse -import glob -import os -import pickle -import sys - -from docutils import core -from docutils import io -from docutils import nodes -import jinja2 -from oslo_config import cfg - -from autohelp import OptionsCache # noqa - -# Swift configuration example files live in -# swift/etc/*.conf-sample -# and contain sections enclosed in [], with -# options one per line containing = -# and generally only having a single entry -# after the equals (the default value) - - -def parse_line(line): - """Parse a line. - - Takes a line from a swift sample configuration file and attempts - to separate the lines with actual configuration option and default - value from the rest. Returns None if the line doesn't appear to - contain a valid configuration option = default value pair, and - a pair of the config and its default if it does. - """ - - if '=' not in line: - return None - temp_line = line.strip('#').strip() - config, default = temp_line.split('=', 1) - config = config.strip() - if ' ' in config and config[0:3] != 'set': - if len(default.split()) > 1 or config[0].isupper(): - return None - if len(config) < 2 or '.' in config or '<' in config or '>' in config: - return None - return config, default.strip() - - -def get_existing_options(optfiles): - """Parse an existing RST table to compile a list of existing options.""" - options = {} - for optfile in optfiles: - input_string = open(optfile).read().replace(':ref:', '') - output, pub = core.publish_programmatically( - source_class=io.StringInput, source=input_string, - source_path=optfile, destination_class=io.NullOutput, - destination=None, destination_path='/dev/null', reader=None, - reader_name='standalone', parser=None, - parser_name='restructuredtext', writer=None, writer_name='null', - settings=None, settings_spec=None, settings_overrides=None, - config_section=None, enable_exit_status=None) - doc = pub.writer.document - data = dict(doc.traverse(nodes.row, include_self=False)[1:]) - for a, b in data.items(): - option = str(a.traverse(nodes.literal)[0].children[0]) - helptext = str(b.traverse(nodes.paragraph)[0].children[0]) - - if option not in options or 'No help text' in options[option]: - # options[option.split('=',1)[0]] = helptext - options[option] = helptext.strip() - - return options - - -def extract_descriptions_from_devref(swift_repo, options): - """Extract descriptions from devref RST files. - - Loop through the devref RST files, looking for lines formatted - such that they might contain a description of a particular - option. - """ - - option_descs = {} - rsts = glob.glob(swift_repo + '/doc/source/*.rst') - for rst in rsts: - rst_file = open(rst, 'r') - in_option_block = False - prev_option = None - for line in rst_file: - if 'Option ' in line: - in_option_block = True - if in_option_block: - if '========' in line: - in_option_block = False - continue - if line[0] == ' ' and prev_option is not None: - option_descs[prev_option] = (option_descs[prev_option] - + ' ' + line.strip()) - for option in options: - line_parts = line.strip().split(None, 2) - if (' ' in line and - len(line_parts) == 3 and - option == line_parts[0] and - line_parts[1] != '=' and - option != 'use' and - (option not in option_descs or - len(option_descs[option]) < len(line_parts[2]))): - option_descs[option] = line_parts[2] - prev_option = option - return option_descs - - -def write_files(options, manuals_repo): - """Create new reStructuredText tables. - - Writes a set of reStructuredText-formatted tables, one per section in swift - configuration files. - """ - all_options = {} - names = options.get_option_names() - for full_option in sorted(names, OptionsCache._cmpopts): - section, optname = full_option.split('/') - oslo_opt = options.get_option(full_option)[1] - all_options.setdefault(section, []) - - helptext = oslo_opt.help.strip().replace('\n', ' ') - helptext = ' '.join(helptext.split()) - all_options[section].append((oslo_opt.name, - oslo_opt.default, - helptext)) - - for full_section, options in all_options.items(): - sample_filename, section = full_section.split('|') - tmpl_file = os.path.join(os.path.dirname(__file__), - 'templates/swift.rst.j2') - with open(tmpl_file) as fd: - template = jinja2.Template(fd.read(), trim_blocks=True) - output = template.render(filename=sample_filename, - section=section, - options=options) - - tgt = (manuals_repo + '/doc/config-reference/source/tables/' + - 'swift-' + sample_filename + '-' + section + '.rst') - - with open(tgt, 'w') as fd: - fd.write(output) - - -def read_options(swift_repo, manuals_repo, verbose): - """Find swift configuration options. - - Uses existing tables and swift devref as a source of truth in that order to - determine helptext for options found in sample config files. - """ - - options = get_existing_options(glob.glob( - manuals_repo + '/doc/config-reference/source/tables/swift*rst')) - option_descs = extract_descriptions_from_devref(swift_repo, options) - conf_samples = glob.glob(swift_repo + '/etc/*conf-sample') - for sample in conf_samples: - current_section = None - sample_file = open(sample, 'r') - for line in sample_file: - if '[' in line and ']\n' in line and '=' not in line: - # It's a header line in the conf file, open a new table file - # for this section and close any existing one - new_line = line.strip('#').strip() - if current_section != new_line: - current_section = new_line - - base_section = os.path.basename(sample).split('.conf')[0] - extra_section = current_section[1:-1].replace(':', '-') - full_section = "%s|%s" % (base_section, extra_section) - - continue - - # All the swift files start with a section, except the rsync - # sample. The first items are not important for us. - if current_section is None: - continue - - # It's a config option line in the conf file, find out the - # help text and write to the table file. - parsed_line = parse_line(line) - if parsed_line is not None: - if (parsed_line[0] in options.keys() - and 'No help text' not in options[parsed_line[0]]): - # use the help text from existing tables - option_desc = options[parsed_line[0]] - elif parsed_line[0] in option_descs: - # use the help text from the devref - option_desc = option_descs[parsed_line[0]] - else: - option_desc = 'No help text available for this option.' - if verbose > 0: - print(parsed_line[0] + " has no help text") - - # \xa0 is a non-breacking space - name = parsed_line[0] - option_desc = option_desc.replace(u'\xa0', u' ') - default = parsed_line[1] - - o = cfg.StrOpt(name=name, default=default, help=option_desc) - try: - cfg.CONF.register_opt(o, full_section) - except cfg.DuplicateOptError: - pass - - -def dump_options(options): - """Dump the list of options with their attributes. - - This output is consumed by the diff_branches script. - """ - print(pickle.dumps(options._opts_by_name)) - - -def main(): - """Parse and write the Swift configuration options.""" - - parser = argparse.ArgumentParser( - description="Update the swift options tables.", - usage=("%(prog)s rst|dump [-v] [-s swift_repo] " - "[-m manuals_repo]")) - parser.add_argument('subcommand', - help='Action (rst, dump).', - choices=['dump', 'rst']) - parser.add_argument('-s', '--swift-repo', - dest='swift_repo', - help="Location of the swift git repository.", - required=False, - default="./sources/swift") - parser.add_argument('-m', '--manuals-repo', - dest='manuals_repo', - help="Location of the manuals git repository.", - required=False, - default="./sources/openstack-manuals") - parser.add_argument('-v', '--verbose', - action='count', - default=0, - dest='verbose', - required=False) - args = parser.parse_args() - - # Avoid cluttering the pickle output, otherwise it's not usable - if args.subcommand == 'dump': - args.verbose = 0 - - read_options(args.swift_repo, - args.manuals_repo, - args.verbose) - options = OptionsCache() - - if args.subcommand == 'rst': - write_files(options, args.manuals_repo) - else: - options.dump() - - return 0 - -if __name__ == "__main__": - sys.exit(main()) diff --git a/autogenerate_config_docs/templates/swift.rst.j2 b/autogenerate_config_docs/templates/swift.rst.j2 deleted file mode 100644 index 4deaef62..00000000 --- a/autogenerate_config_docs/templates/swift.rst.j2 +++ /dev/null @@ -1,15 +0,0 @@ -.. - Warning: Do not edit this file. It is automatically generated and your - changes will be overwritten. The tool to do so lives in the - openstack-doc-tools repository. - -.. list-table:: Description of configuration options for ``[{{ section }}]`` in ``{{ filename }}.conf`` - :header-rows: 1 - :class: config-ref-table - - * - Configuration option = Default value - - Description -{% for option in options %} - * - ``{{ option[0] }}`` = ``{{ option[1]|default(' ', true) }}`` - - {{ option[2] }} -{% endfor %} diff --git a/releasenotes/notes/remove-swift-support-54f2d2d9c78cbb58.yaml b/releasenotes/notes/remove-swift-support-54f2d2d9c78cbb58.yaml new file mode 100644 index 00000000..9dec2af4 --- /dev/null +++ b/releasenotes/notes/remove-swift-support-54f2d2d9c78cbb58.yaml @@ -0,0 +1,9 @@ +--- +upgrade: + - | + The `extract_swift_flags.py` script has been removed, and the `autohelp.py` + and `diff_branches.py` scripts no longer build config file documentation + for the swift project. The swift dev team had been manually maintaining + their config files in-tree and to avoid duplication, doc and swift have + agreed to link the config ref out to the dev docs. As such, there is + therefore no reason to keep this tooling around.