From efdd9c8c80b442c9cc4890935b3fdd087d10f1f4 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 5 Jul 2012 16:27:12 -0500 Subject: [PATCH] Update common setup code to latest. This gets us up to date with common/setup.py and replaces custom nova autodoc generation with the port of that code found in common. Change-Id: I2a1c5d2c0fdcf40dbea50cc123b537adb068cdc2 --- .gitignore | 4 + .mailmap | 2 +- doc/ext/nova_autodoc.py | 12 -- doc/find_autodoc_modules.sh | 20 --- doc/generate_autodoc_index.sh | 46 ------- doc/source/conf.py | 5 - nova/openstack/common/setup.py | 244 +++++++++++++++++++++++++++++---- setup.py | 27 +--- 8 files changed, 227 insertions(+), 133 deletions(-) delete mode 100644 doc/ext/nova_autodoc.py delete mode 100755 doc/find_autodoc_modules.sh delete mode 100755 doc/generate_autodoc_index.sh diff --git a/.gitignore b/.gitignore index 94661beec..9d4779509 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,7 @@ tools/conf/nova.conf* *.sqlite *.DS_Store *.egg* +.project +.pydevproject +doc/source/api/* +doc/build/* diff --git a/.mailmap b/.mailmap index 5fcd106ed..d372b8cc2 100644 --- a/.mailmap +++ b/.mailmap @@ -43,7 +43,7 @@ - Masumoto + diff --git a/doc/ext/nova_autodoc.py b/doc/ext/nova_autodoc.py deleted file mode 100644 index 6c5376fe1..000000000 --- a/doc/ext/nova_autodoc.py +++ /dev/null @@ -1,12 +0,0 @@ -import gettext -import os - -gettext.install('nova') - -from nova import utils - - -def setup(app): - print "**Autodocumenting from %s" % os.path.abspath(os.curdir) - rv = utils.execute('./doc/generate_autodoc_index.sh') - print rv[0] diff --git a/doc/find_autodoc_modules.sh b/doc/find_autodoc_modules.sh deleted file mode 100755 index 0aff36f7d..000000000 --- a/doc/find_autodoc_modules.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -NOVA_DIR='nova/' # include trailing slash -DOCS_DIR='source' - -modules='' -for x in `find ${NOVA_DIR} -name '*.py' | grep -v nova/tests`; do - if [ `basename ${x} .py` == "__init__" ] ; then - continue - fi - relative=nova.`echo ${x} | sed -e 's$^'${NOVA_DIR}'$$' -e 's/.py$//' -e 's$/$.$g'` - modules="${modules} ${relative}" -done - -for mod in ${modules} ; do - if [ ! -f "${DOCS_DIR}/${mod}.rst" ]; - then - echo ${mod} - fi -done diff --git a/doc/generate_autodoc_index.sh b/doc/generate_autodoc_index.sh deleted file mode 100755 index bdfa73a49..000000000 --- a/doc/generate_autodoc_index.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh - -SOURCEDIR=doc/source/api - -if [ ! -d ${SOURCEDIR} ] ; then - mkdir -p ${SOURCEDIR} -fi - -for x in `./doc/find_autodoc_modules.sh`; -do - echo "Generating ${SOURCEDIR}/${x}.rst" - echo "${SOURCEDIR}/${x}.rst" >> .autogenerated - heading="The :mod:\`${x}\` Module" - # Figure out how long the heading is - # and make sure to emit that many '=' under - # it to avoid heading format errors - # in Sphinx. - heading_len=$(echo "$heading" | wc -c) - underline=$(head -c $heading_len < /dev/zero | tr '\0' '=') - ( cat < ${SOURCEDIR}/${x}.rst - -done - -if [ ! -f ${SOURCEDIR}/autoindex.rst ] ; then - - cat > ${SOURCEDIR}/autoindex.rst <> ${SOURCEDIR}/autoindex.rst - done - - echo ${SOURCEDIR}/autoindex.rst >> .autogenerated -fi diff --git a/doc/source/conf.py b/doc/source/conf.py index 8ced294d2..78feba48e 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -27,11 +27,6 @@ sys.path.insert(0, os.path.abspath('./')) extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'ext.nova_todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig','sphinx.ext.graphviz'] -# autodoc generation is a bit aggressive and a nuisance when doing heavy text edit cycles. -# execute "export SPHINX_DEBUG=1" in your terminal to disable -if not os.getenv('SPHINX_DEBUG'): - extensions += ['ext.nova_autodoc'] - todo_include_todos = True # Add any paths that contain templates here, relative to this directory. diff --git a/nova/openstack/common/setup.py b/nova/openstack/common/setup.py index 60c731a9a..140d37b2d 100644 --- a/nova/openstack/common/setup.py +++ b/nova/openstack/common/setup.py @@ -19,9 +19,13 @@ Utilities with minimum-depends for use in setup.py """ +import datetime import os import re import subprocess +import sys + +from setuptools.command import sdist def parse_mailmap(mailmap='.mailmap'): @@ -31,7 +35,8 @@ def parse_mailmap(mailmap='.mailmap'): for l in fp: l = l.strip() if not l.startswith('#') and ' ' in l: - canonical_email, alias = l.split(' ') + canonical_email, alias = [x for x in l.split(' ') + if x.startswith('<')] mapping[alias] = canonical_email return mapping @@ -58,11 +63,25 @@ def parse_requirements(requirements_files=['requirements.txt', 'tools/pip-requires']): requirements = [] for line in get_reqs_from_files(requirements_files): + # For the requirements list, we need to inject only the portion + # after egg= so that distutils knows the package it's looking for + # such as: + # -e git://github.com/openstack/nova/master#egg=nova if re.match(r'\s*-e\s+', line): requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', line)) + # such as: + # http://github.com/openstack/nova/zipball/master#egg=nova + elif re.match(r'\s*https?:', line): + requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1', + line)) + # -f lines are for index locations, and don't get used here elif re.match(r'\s*-f\s+', line): pass + # argparse is part of the standard library starting with 2.7 + # adding it to the requirements list screws distro installs + elif line == 'argparse' and sys.version_info >= (2, 7): + pass else: requirements.append(line) @@ -72,11 +91,18 @@ def parse_requirements(requirements_files=['requirements.txt', def parse_dependency_links(requirements_files=['requirements.txt', 'tools/pip-requires']): dependency_links = [] + # dependency_links inject alternate locations to find packages listed + # in requirements for line in get_reqs_from_files(requirements_files): + # skip comments and blank lines if re.match(r'(\s*#)|(\s*$)', line): continue + # lines with -e or -f need the whole line, minus the flag if re.match(r'\s*-[ef]\s+', line): dependency_links.append(re.sub(r'\s*-[ef]\s+', '', line)) + # lines that are only urls can go in unmolested + elif re.match(r'\s*https?:', line): + dependency_links.append(line) return dependency_links @@ -93,28 +119,54 @@ def write_requirements(): def _run_shell_command(cmd): output = subprocess.Popen(["/bin/sh", "-c", cmd], stdout=subprocess.PIPE) - return output.communicate()[0].strip() + out = output.communicate() + if len(out) == 0: + return None + if len(out[0].strip()) == 0: + return None + return out[0].strip() -def write_vcsversion(location): - """Produce a vcsversion dict that mimics the old one produced by bzr. - """ - if os.path.isdir('.git'): - branch_nick_cmd = 'git branch | grep -Ei "\* (.*)" | cut -f2 -d" "' - branch_nick = _run_shell_command(branch_nick_cmd) - revid_cmd = "git rev-parse HEAD" - revid = _run_shell_command(revid_cmd).split()[0] - revno_cmd = "git log --oneline | wc -l" - revno = _run_shell_command(revno_cmd) - with open(location, 'w') as version_file: - version_file.write(""" -# This file is automatically generated by setup.py, So don't edit it. :) -version_info = { - 'branch_nick': '%s', - 'revision_id': '%s', - 'revno': %s -} -""" % (branch_nick, revid, revno)) +def _get_git_next_version_suffix(branch_name): + datestamp = datetime.datetime.now().strftime('%Y%m%d') + if branch_name == 'milestone-proposed': + revno_prefix = "r" + else: + revno_prefix = "" + _run_shell_command("git fetch origin +refs/meta/*:refs/remotes/meta/*") + milestone_cmd = "git show meta/openstack/release:%s" % branch_name + milestonever = _run_shell_command(milestone_cmd) + if not milestonever: + milestonever = "" + post_version = _get_git_post_version() + revno = post_version.split(".")[-1] + return "%s~%s.%s%s" % (milestonever, datestamp, revno_prefix, revno) + + +def _get_git_current_tag(): + return _run_shell_command("git tag --contains HEAD") + + +def _get_git_tag_info(): + return _run_shell_command("git describe --tags") + + +def _get_git_post_version(): + current_tag = _get_git_current_tag() + if current_tag is not None: + return current_tag + else: + tag_info = _get_git_tag_info() + if tag_info is None: + base_version = "0.0" + cmd = "git --no-pager log --oneline" + out = _run_shell_command(cmd) + revno = len(out.split("\n")) + else: + tag_infos = tag_info.split("-") + base_version = "-".join(tag_infos[:-2]) + revno = tag_infos[-2] + return "%s.%s" % (base_version, revno) def write_git_changelog(): @@ -134,8 +186,8 @@ def generate_authors(): new_authors = 'AUTHORS' if os.path.isdir('.git'): # don't include jenkins email address in AUTHORS file - git_log_cmd = "git log --format='%aN <%aE>' | sort -u | " \ - "grep -v " + jenkins_email + git_log_cmd = ("git log --format='%aN <%aE>' | sort -u | " + "grep -v " + jenkins_email) changelog = _run_shell_command(git_log_cmd) mailmap = parse_mailmap() with open(new_authors, 'w') as new_authors_fh: @@ -143,3 +195,149 @@ def generate_authors(): if os.path.exists(old_authors): with open(old_authors, "r") as old_authors_fh: new_authors_fh.write('\n' + old_authors_fh.read()) + +_rst_template = """%(heading)s +%(underline)s + +.. automodule:: %(module)s + :members: + :undoc-members: + :show-inheritance: +""" + + +def read_versioninfo(project): + """Read the versioninfo file. If it doesn't exist, we're in a github + zipball, and there's really no way to know what version we really + are, but that should be ok, because the utility of that should be + just about nil if this code path is in use in the first place.""" + versioninfo_path = os.path.join(project, 'versioninfo') + if os.path.exists(versioninfo_path): + with open(versioninfo_path, 'r') as vinfo: + version = vinfo.read().strip() + else: + version = "0.0.0" + return version + + +def write_versioninfo(project, version): + """Write a simple file containing the version of the package.""" + open(os.path.join(project, 'versioninfo'), 'w').write("%s\n" % version) + + +def get_cmdclass(): + """Return dict of commands to run from setup.py.""" + + cmdclass = dict() + + def _find_modules(arg, dirname, files): + for filename in files: + if filename.endswith('.py') and filename != '__init__.py': + arg["%s.%s" % (dirname.replace('/', '.'), + filename[:-3])] = True + + class LocalSDist(sdist.sdist): + """Builds the ChangeLog and Authors files from VC first.""" + + def run(self): + write_git_changelog() + generate_authors() + # sdist.sdist is an old style class, can't use super() + sdist.sdist.run(self) + + cmdclass['sdist'] = LocalSDist + + # If Sphinx is installed on the box running setup.py, + # enable setup.py to build the documentation, otherwise, + # just ignore it + try: + from sphinx.setup_command import BuildDoc + + class LocalBuildDoc(BuildDoc): + def generate_autoindex(self): + print "**Autodocumenting from %s" % os.path.abspath(os.curdir) + modules = {} + option_dict = self.distribution.get_option_dict('build_sphinx') + source_dir = os.path.join(option_dict['source_dir'][1], 'api') + if not os.path.exists(source_dir): + os.makedirs(source_dir) + for pkg in self.distribution.packages: + if '.' not in pkg: + os.path.walk(pkg, _find_modules, modules) + module_list = modules.keys() + module_list.sort() + autoindex_filename = os.path.join(source_dir, 'autoindex.rst') + with open(autoindex_filename, 'w') as autoindex: + autoindex.write(""".. toctree:: + :maxdepth: 1 + +""") + for module in module_list: + output_filename = os.path.join(source_dir, + "%s.rst" % module) + heading = "The :mod:`%s` Module" % module + underline = "=" * len(heading) + values = dict(module=module, heading=heading, + underline=underline) + + print "Generating %s" % output_filename + with open(output_filename, 'w') as output_file: + output_file.write(_rst_template % values) + autoindex.write(" %s.rst\n" % module) + + def run(self): + if not os.getenv('SPHINX_DEBUG'): + self.generate_autoindex() + + for builder in ['html', 'man']: + self.builder = builder + self.finalize_options() + self.project = self.distribution.get_name() + self.version = self.distribution.get_version() + self.release = self.distribution.get_version() + BuildDoc.run(self) + cmdclass['build_sphinx'] = LocalBuildDoc + except ImportError: + pass + + return cmdclass + + +def get_git_branchname(): + for branch in _run_shell_command("git branch --color=never").split("\n"): + if branch.startswith('*'): + _branch_name = branch.split()[1].strip() + if _branch_name == "(no": + _branch_name = "no-branch" + return _branch_name + + +def get_pre_version(projectname, base_version): + """Return a version which is based""" + if os.path.isdir('.git'): + current_tag = _get_git_current_tag() + if current_tag is not None: + version = current_tag + else: + branch_name = os.getenv('BRANCHNAME', + os.getenv('GERRIT_REFNAME', + get_git_branchname())) + version_suffix = _get_git_next_version_suffix(branch_name) + version = "%s~%s" % (base_version, version_suffix) + write_versioninfo(projectname, version) + return version.split('~')[0] + else: + version = read_versioninfo(projectname) + return version.split('~')[0] + + +def get_post_version(projectname): + """Return a version which is equal to the tag that's on the current + revision if there is one, or tag plus number of additional revisions + if the current revision has no tag.""" + + if os.path.isdir('.git'): + version = _get_git_post_version() + write_versioninfo(projectname, version) + return version + return read_versioninfo(projectname) diff --git a/setup.py b/setup.py index 0f71ec2e8..1e4082923 100644 --- a/setup.py +++ b/setup.py @@ -19,36 +19,11 @@ import glob import os import setuptools -from setuptools.command import sdist from nova.openstack.common import setup as common_setup from nova import version -class local_sdist(sdist.sdist): - """Customized sdist hook - builds the ChangeLog file from VC first.""" - def run(self): - common_setup.write_git_changelog() - # sdist.sdist is an old style class, can't user super() - sdist.sdist.run(self) - -nova_cmdclass = {'sdist': local_sdist} - -try: - from sphinx import setup_command - - class local_BuildDoc(setup_command.BuildDoc): - def run(self): - for builder in ['html', 'man']: - self.builder = builder - self.finalize_options() - setup_command.BuildDoc.run(self) - nova_cmdclass['build_sphinx'] = local_BuildDoc - -except Exception: - pass - - def find_data_files(destdir, srcdir): package_data = [] files = [] @@ -68,7 +43,7 @@ setuptools.setup(name='nova', author='OpenStack', author_email='nova@lists.launchpad.net', url='http://www.openstack.org/', - cmdclass=nova_cmdclass, + cmdclass=common_setup.get_cmdclass(), packages=setuptools.find_packages(exclude=['bin', 'smoketests']), include_package_data=True, test_suite='nose.collector',