diff --git a/tools/README-USE-openstack-doc-tools.txt b/tools/README-USE-openstack-doc-tools.txt
index aed4a8375e..d81b03cb26 100644
--- a/tools/README-USE-openstack-doc-tools.txt
+++ b/tools/README-USE-openstack-doc-tools.txt
@@ -1,5 +1,6 @@
-The tools directory has been moved to a separate
-repository openstack-doc-tools:
+With the exception of the autogenerate-config-flagmappings directory,
+the tools directory has been moved to a separate repository
+openstack-doc-tools:
https://github.com/openstack/openstack-doc-tools
@@ -8,3 +9,7 @@ unless you need those for gating jobs.
This directory will be removed once all the gating jobs are setup
correctly.
+
+Exception: the directory autogenerate-config-flagmappings contains
+data that will stay here.
+
diff --git a/tools/autogenerate-config-docs/.gitignore b/tools/autogenerate-config-docs/.gitignore
deleted file mode 100644
index 0ad0c6b53a..0000000000
--- a/tools/autogenerate-config-docs/.gitignore
+++ /dev/null
@@ -1,9 +0,0 @@
-*.DS_Store
-*.egg*
-*.log
-*.mo
-*.pyc
-*.swo
-*.swp
-*.sqlite
-*~
diff --git a/tools/autogenerate-config-docs/autohelp.py b/tools/autogenerate-config-docs/autohelp.py
deleted file mode 100755
index 5b6e0e5ee9..0000000000
--- a/tools/autogenerate-config-docs/autohelp.py
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env python
-#
-# A collection of tools for working with flags from OpenStack
-# packages and documentation.
-#
-# For an example of usage, run this program with the -h switch.
-#
-
-import os
-import sys
-
-# this is for the internationalisation function in gettext
-import __builtin__
-__builtin__.__dict__['_'] = lambda x: x
-
-import common
-
-
-def main(action, file, format, repo, verbose=0, name=False, test=False):
- package_name = common.git_check(repo)
-
- sys.path.append(repo)
- try:
- __import__(package_name)
- except ImportError as e:
- if verbose >= 1:
- print str(e)
- print "Failed to import: %s (%s)" % (package_name, e)
-
- if verbose >= 1:
- flags = common.extract_flags(repo, package_name, verbose)
- else:
- flags = common.extract_flags(repo, package_name)
-
- print "%s flags imported from package %s." % (len(flags),
- str(package_name))
- if action == "update":
- common.update(file, flags, True, verbose)
- return
-
- if format == "names":
- if verbose >= 1:
- common.write_flags(file, flags, True, verbose)
- else:
- common.write_flags(file, flags, True)
-
- if format == "docbook":
- groups = common.populate_groups(file)
- print "%s groups" % len(groups)
- if verbose >= 1:
- common.write_docbook('.', flags, groups, package_name, verbose)
- else:
- common.write_docbook('.', flags, groups, package_name)
-
- sys.exit(0)
-
-if __name__ == "__main__":
- args = common.parse_me_args()
- main(args['action'],
- args['file'],
- args['format'],
- args['repo'],
- args['verbose'],
- args['name'],
- args['test'])
diff --git a/tools/autogenerate-config-docs/common.py b/tools/autogenerate-config-docs/common.py
deleted file mode 100644
index f412d13654..0000000000
--- a/tools/autogenerate-config-docs/common.py
+++ /dev/null
@@ -1,401 +0,0 @@
-#
-# A collection of shared functions for managing help flag mapping files.
-#
-
-import os
-import string
-import sys
-import pkgutil
-import glob
-
-from collections import defaultdict
-from xml.sax.saxutils import escape
-from oslo.config import cfg
-
-# gettext internationalisation function requisite:
-import __builtin__
-__builtin__.__dict__['_'] = lambda x: x
-
-
-def git_check(repo_path):
- from git import Repo
- """
- Check a passed directory to verify it is a valid git repository.
- """
- try:
- repo = Repo(repo_path)
- assert repo.bare is False
- package_name = os.path.basename(repo.remotes.origin.url).rstrip('.git')
- except:
- print "\nThere is a problem verifying that the directory passed in"
- print "is a valid git repository. Please try again.\n"
- sys.exit(1)
- return package_name
-
-
-def populate_groups(filepath):
- """
- Takes a file formatted with lines of config option and group
- separated by a space and constructs a dictionary indexed by
- group, which is returned..
- """
- groups = defaultdict(list)
- groups_file = open(os.path.expanduser(filepath), 'r')
- for line in groups_file:
- try:
- option, group = line.split(None, 1)
- except ValueError:
- print "Couldn't read groups file line:%s" % line
- print "Check for formatting errors - did you add the group?"
- sys.exit(1)
- groups[group.strip()].append(option)
- return groups
-
-
-def extract_flags(repo_location, module_name, verbose=0, names_only=True):
- """
- Loops through the repository, importing module by module to
- populate the configuration object (cfg.CONF) created from Oslo.
- """
- usable_dirs = []
- module_location = os.path.dirname(repo_location + '/' + module_name)
- for root, dirs, files in os.walk(module_location + '/' + module_name):
- for name in dirs:
- abs_path = os.path.join(root.split(module_location)[1][1:], name)
- if ('/tests' not in abs_path and '/locale' not in abs_path and
- '/cmd' not in abs_path and '/db/migration' not in abs_path and
- '/transfer' not in abs_path):
- usable_dirs.append(os.path.join(root.split(module_location)[1][1:], name))
-
- for directory in usable_dirs:
- for python_file in glob.glob(module_location + '/' + directory + "/*.py"):
- if '__init__' not in python_file:
- usable_dirs.append(os.path.splitext(python_file)[0][len(module_location) + 1:])
-
- package_name = directory.replace('/', '.')
- try:
- __import__(package_name)
- if verbose >= 1:
- print "imported %s" % package_name
-
- except ImportError as e:
- """
- work around modules that don't like being imported in this way
- FIXME This could probably be better, but does not affect the
- configuration options found at this stage
- """
- if verbose >= 2:
- print str(e)
- print "Failed to import: %s (%s)" % (package_name, e)
-
- continue
-
- flags = cfg.CONF._opts.items()
-
- #extract group information
- for group in cfg.CONF._groups.keys():
- flags = flags + cfg.CONF._groups[group]._opts.items()
- flags.sort()
-
- return flags
-
-
-def extract_flags_test(repo_loc, module, verbose=0):
- """
- TEST TEST TEST TEST TEST TEST
- TEST TEST TEST TEST TEST TEST
- Loops through the repository, importing module by module to
- populate the configuration object (cfg.CONF) created from Oslo.
- TEST TEST TEST TEST TEST TEST
- TEST TEST TEST TEST TEST TEST
- """
- flag_data = {}
- flag_files = []
- usable_dirs = []
- module_location = os.path.dirname(repo_loc + '/' + module)
- for root, dirs, files in os.walk(module_location + '/' + module):
- for name in dirs:
- abs_path = os.path.join(root.split(module_location)[1][1:], name)
- if ('/tests' not in abs_path and '/locale' not in abs_path and
- '/cmd' not in abs_path and '/db/migration' not in abs_path):
- usable_dirs.append(os.path.join(root.split(module_location)[1][1:], name))
-
- for directory in usable_dirs:
- for python_file in glob.glob(module_location + '/' + directory + "/*.py"):
- if '__init__' not in python_file:
- usable_dirs.append(os.path.splitext(python_file)[0][len(module_location) + 1:])
-
- package_name = directory.replace('/', '.')
- try:
- __import__(package_name)
- if verbose >= 1:
- print "imported %s" % package_name
- flag_data[str(package_name)] = sorted(cfg.CONF._opts.items())
-
- except ImportError as e:
- """
- work around modules that don't like being imported in this way
- FIXME This could probably be better, but does not affect the
- configuration options found at this stage
- """
- if verbose >= 2:
- print str(e)
- print "Failed to import: %s (%s)" % (package_name, e)
-
- continue
-
- return flag_data
-
-
-def write_test(file, repo_dir, pkg_name):
- """
- """
- file1 = file + ".test"
- flags = extract_flags_test(repo_dir, pkg_name)
- with open(file1, 'a+') as f:
- f.write("\n")
- for filename, flag_info in flags.iteritems():
- f.write("\n -- start file name area --\n")
- f.write(filename)
- f.write("\n -- end file name area --\n")
- print "\n -- start file name area --\n"
- print filename
- print "\n -- end file name area --\n"
- print len(flag_info)
- for name, value in flag_info:
- opt = value['opt']
- #print type(opt)
- #print opt
- #print name
- #print value
- f.write(name)
- f.write("\n")
-
-
-def write_header(filepath, verbose=0):
- """
- Write header to output flag file.
- """
- pass
-
-
-def write_buffer(file, flags, verbose=0):
- """
- Write flag data to file. (The header is written with the write_header function.)
- """
- pass
- #with open(os.path.expanduser(filepath), 'wb') as f:
-
-
-def write_flags(filepath, flags, name_only=True, verbose=0):
- """
- write out the list of flags in the cfg.CONF object to filepath
- if name_only is True - write only a list of names, one per line,
- otherwise use MediaWiki syntax to write out the full table with
- help text and default values.
- """
- with open(os.path.expanduser(filepath), 'wb') as f:
- if not name_only:
- f.write("{|\n") # start table
- # print headers
- f.write("!")
- f.write("!!".join(["name", "default", "description"]))
- f.write("\n|-\n")
-
- for name, value in flags:
- opt = value['opt']
- if not opt.help:
- opt.help = "No help text available for this option"
- if not name_only:
- f.write("|")
- f.write("||".join([string.strip(name),
- string.strip(str(opt.default)),
- string.strip(opt.help.replace("\n", " "))]))
- f.write("\n|-\n")
- else:
- f.write(name + "\n")
-
- if not name_only:
- f.write("|}\n") # end table
-
-
-def write_docbook(directory, flags, groups, package_name, verbose=0):
- """
- Prints a docbook-formatted table for every group of options.
- """
- count = 0
- for group in groups.items():
- groups_file = open(package_name + '-' + group[0] + '.xml', 'w')
- groups_file.write('\n\
- \n\
- \n\
-
\n\
-
Description of configuration options for ' + group[0] +
- '
\n\
-
\n\
-
\n\
- \n\
-
\n\
-
Configuration option=Default value
\n\
-
Description
\n\
-
\n\
- \n\
- ')
- for flag_name in group[1]:
- for flag in flags:
- if flag[0] == flag_name:
- count = count + 1
- opt = flag[1]["opt"]
- if not opt.help:
- opt.help = "No help text available for this option"
- if type(opt).__name__ == "ListOpt" and opt.default is not None:
- opt.default = ",".join(opt.default)
- groups_file.write('\n
\n\
- ')
- groups_file.close()
-
-
-def create(flag_file, repo_path):
- """
- Create new flag mappings file, containing help information for
- the project whose repo location has been passed in at the command line.
- """
-
- # flag_file testing.
- #try:
- # Test for successful creation of flag_file.
- #except:
- # If the test(s) fail, exit noting the problem(s).
-
- # repo_path git repo validity testing.
- #try:
- # Test to be sure the repo_path passed in is a valid directory
- # and that directory is a valid existing git repo.
- #except:
- # If the test(s) fail, exit noting the problem(s).
-
- # get as much help as possible, searching recursively through the
- # entire repo source directory tree.
- #help_data = get_help(repo_path)
-
- # Write this information to the file.
- #write_file(flag_file, help_data)
-
-
-def update(filepath, flags, name_only=True, verbose=0):
- """
- Update flag mappings file, adding or removing entries as needed.
- This will update the file content, essentially overriding the data.
- The primary difference between create and update is that create will
- make a new file, and update will just work with the data that is
- data that is already there.
- """
- original_flags = []
- updated_flags = []
- write_flags(filepath + '.new', flags, name_only=True, verbose=0)
- original_flag_file = open(filepath)
- updated_flag_file = open(filepath + '.new', 'r')
- for line in original_flag_file:
- original_flags.append(line.split()[0])
- for line in updated_flag_file:
- updated_flags.append(line.rstrip())
- updated_flag_file.close()
-
- removed_flags = set(original_flags) - set(updated_flags)
- added_flags = set(updated_flags) - set(original_flags)
-
- print "\nRemoved Flags\n"
- for line in sorted(removed_flags):
- print line
-
- print "\nAdded Flags\n"
- for line in sorted(added_flags):
- print line
-
- updated_flag_file = open(filepath + '.new', 'wb')
- original_flag_file.seek(0)
- for line in original_flag_file:
- flag_name = line.split()[0]
- if flag_name not in removed_flags:
- for added_flag in added_flags:
- if flag_name > added_flag:
- updated_flag_file.write(added_flag + ' Unknown\n')
- added_flags.remove(added_flag)
- break
- updated_flag_file.write(line)
-
-
-def verify(flag_file):
- """
- Verify flag file contents. No actions are taken.
- """
- pass
-
-
-def usage():
- print "\nUsage: %s docbook " % sys.argv[0]
- print "\nGenerate a list of all flags for package in source loc and"\
- "\nwrites them in a docbook table format, grouped by the groups"\
- "\nin the groups file, one file per group.\n"
- print "\n %s names " % sys.argv[0]
- print "\nGenerate a list of all flags names for the package in"\
- "\nsource loc and writes them to names file, one per line \n"
-
-
-def parse_me_args():
- import argparse
- parser = argparse.ArgumentParser(
- description='Manage flag files, to aid in updatingdocumentation.',
- epilog='Example: %(prog)s -a create -in ./nova.flagfile -fmt docbook\
- -p /nova',
- usage='%(prog)s [options]')
- parser.add_argument('-a', '--action',
- choices=['create', 'update', 'verify'],
- dest='action',
- help='action (create, update, verify) [REQUIRED]',
- required=True,
- type=str,)
- # trying str data type... instead of file.
- parser.add_argument('-i', '-in', '--input',
- dest='file',
- help='flag file being worked with [REQUIRED]',
- required=True,
- type=str,)
- parser.add_argument('-f', '-fmt', '--format', '-o', '-out',
- dest='format',
- help='file output format (options: docbook, names)',
- required=False,
- type=str,)
- # ..tried having 'dir' here for the type, but the git.Repo function
- # requires a string is passed to it.. a directory won't work.
- parser.add_argument('-p', '--path',
- dest='repo',
- help='path to valid git repository [REQUIRED]',
- required=True,
- type=str,)
- parser.add_argument('-v', '--verbose',
- action='count',
- default=0,
- dest='verbose',
- required=False,)
- parser.add_argument('-no', '--name_only',
- action='store_true',
- dest='name',
- help='whether output should contain names only',
- required=False,)
- parser.add_argument('-test',
- action='store_true',
- dest='test',
- help=argparse.SUPPRESS,
- required=False,)
- args = vars(parser.parse_args())
- return args
diff --git a/tools/autogenerate-config-docs/extract_swift_flags.py b/tools/autogenerate-config-docs/extract_swift_flags.py
deleted file mode 100644
index 9e3440b939..0000000000
--- a/tools/autogenerate-config-docs/extract_swift_flags.py
+++ /dev/null
@@ -1,191 +0,0 @@
-#!/usr/bin/env python
-import sys
-from os import path
-import glob
-from xml.dom import minidom
-from xml.sax.saxutils import escape
-
-#Swift configuration example files live in
-# swift/etc/*.conf-sample
-# and contain sections enclosed in [], with
-# options one per line containing =
-# and generally only having a single entry
-# after the equals (the default value)
-
-
-def parse_line(line):
- """
- takes a line from a swift sample configuration file and attempts
- to separate the lines with actual configuration option and default
- value from the rest. Returns None if the line doesn't appear to
- contain a valid configuration option = default value pair, and
- a pair of the config and its default if it does.
- """
- if '=' not in line:
- return None
- temp_line = line.strip('#').strip()
- config, default = temp_line.split('=', 1)
- config = config.strip()
- if ' ' in config and config[0:3] != 'set':
- if len(default.split()) > 1 or config[0].isupper():
- return None
- if len(config) < 2 or '.' in config or '<' in config or '>' in config:
- return None
- return config, default.strip()
-
-
-def get_existing_options(optfiles):
- """
- parses an existing XML table to compile a list of existing options
- """
- options = {}
- for optfile in optfiles:
- xmldoc = minidom.parse(optfile)
- tbody = xmldoc.getElementsByTagName('tbody')[0]
- trlist = tbody.getElementsByTagName('tr')
- for tr in trlist:
- try:
- optentry = tr.childNodes[1].childNodes[0]
- option, default = optentry.nodeValue.split('=', 1)
- helptext = tr.childNodes[2].childNodes[0].nodeValue
- except IndexError:
- continue
- if option not in options or 'No help text' in options[option]:
- #options[option.split('=',1)[0]] = helptext
- options[option] = helptext
- return options
-
-
-def extract_descriptions_from_devref(repo, options):
- """
- loop through the devref RST files, looking for lines formatted
- such that they might contain a description of a particular
- option
- """
- option_descs = {}
- rsts = glob.glob(repo + '/doc/source/*.rst')
- for rst in rsts:
- rst_file = open(rst, 'r')
- in_option_block = False
- prev_option = None
- for line in rst_file:
- if 'Option ' in line:
- in_option_block = True
- if in_option_block:
- if '========' in line:
- in_option_block = False
- continue
- if line[0] == ' ' and prev_option is not None:
- option_descs[prev_option] = (option_descs[prev_option]
- + ' ' + line.strip())
- for option in options:
- line_parts = line.strip().split(None, 2)
- if (' ' in line and len(line_parts) == 3
- and option == line_parts[0]
- and line_parts[1] != '=' and option != 'use'
- and (option not in option_descs or
- len(option_descs[option]) < len(line_parts[2]))):
- option_descs[option] = line_parts[2]
- prev_option = option
- return option_descs
-
-
-def new_section_file(sample, current_section):
- section_filename = ('swift-' +
- path.basename(sample).split('.conf')[0]
- + '-'
- + current_section.replace('[', '').replace(']', '').replace(':', '-')
- + '.xml')
- section_file = open(section_filename, 'w')
- section_file.write('\n\
- \n\
- \n\
-
\n\
-
Description of configuration options for '
- + current_section + ' in ' + path.basename(sample) +
- '
\n\
-
\n\
-
\n\
- \n\
-
\n\
-
Configuration option=Default value
\n\
-
Description
\n\
-
\n\
- \n\
- ')
- return section_file
-
-
-def create_new_tables(repo, verbose):
- """
- writes a set of docbook-formatted tables, one per section in swift
- configuration files. Uses existing tables and swift devref as a source
- of truth in that order to determine helptext for options found in
- sample config files
- """
- existing_tables = glob.glob('../../doc/common/tables/swift*xml')
- options = {}
- #use the existing tables to get a list of option names
- options = get_existing_options(existing_tables)
- option_descs = extract_descriptions_from_devref(repo, options)
- conf_samples = glob.glob(repo + '/etc/*conf-sample')
- for sample in conf_samples:
- current_section = None
- section_file = None
- sample_file = open(sample, 'r')
- for line in sample_file:
- if '[' in line and ']\n' in line and '=' not in line:
- """
- it's a header line in the conf file, open a new table file
- for this section and close any existing one
- """
- if current_section != line.strip('#').strip():
- if section_file is not None:
- section_file.write('\n \n\
-
\n\
- ')
- section_file.close()
- current_section = line.strip('#').strip()
- section_file = new_section_file(sample, current_section)
- elif section_file is not None:
- """
- it's a config option line in the conf file, find out the
- help text and write to the table file.
- """
- parsed_line = parse_line(line)
- if parsed_line is not None:
- if (parsed_line[0] in options.keys()
- and 'No help text' not in options[parsed_line[0]]):
- # use the help text from existing tables
- option_desc = options[parsed_line[0]].replace(u'\xa0', u' ')
- elif parsed_line[0] in option_descs:
- # use the help text from the devref
- option_desc = option_descs[parsed_line[0]].replace(u'\xa0', u' ')
- else:
- option_desc = 'No help text available for this option'
- if verbose > 0:
- print parsed_line[0] + "has no help text"
- section_file.write('\n