openstack
/
pbr
Code Issues Proposed changes

Only import sphinx during hook processing 

When pbr is imported to handle writing the egg_info file because of
the entry point, it's causing sphinx to get imported. This has a
cascading effect once docutils is trying to be installed on a
system with pbr installed. If some of the imports fail along the way,
allow pbr to continue usefully but without the Sphinx extensions
available. Eventually, when everything is installed, those extensions
will work again when the commands for build_sphinx, etc. are run
separately.

Also slip in a change to reorder the default list of environments run by
tox so the testr database is created using a dbm format available to all
python versions.

Change-Id: I79d67bf41a09d7e5aad8ed32eaf107f139167eb8
Closes-bug: #1403510
This commit is contained in:
Doug Hellmann 2014-12-17 14:26:03 -05:00
parent e73e67acdd
commit 65f4fafd90
7 changed files with 267 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

189
pbr/builddoc.py Normal file
View File

@ -0,0 +1,189 @@
# Copyright 2011 OpenStack LLC.
# Copyright 2012-2013 Hewlett-Packard Development Company, L.P.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from distutils import log
import os
import sys
try:
import cStringIO
except ImportError:
import io as cStringIO
try:
from sphinx import apidoc
from sphinx import application
from sphinx import config
from sphinx import setup_command
except Exception as e:
# NOTE(dhellmann): During the installation of docutils, setuptools
# tries to import pbr code to find the egg_info.writer hooks. That
# imports this module, which imports sphinx, which imports
# docutils, which is being installed. Because docutils uses 2to3
# to convert its code during installation under python 3, the
# import fails, but it fails with an error other than ImportError
# (today it's a NameError on StandardError, an exception base
# class). Convert the exception type here so it can be caught in
# packaging.py where we try to determine if we can import and use
# sphinx by importing this module. See bug #1403510 for details.
raise ImportError(str(e))
from pbr import options
_rst_template = """%(heading)s
%(underline)s
.. automodule:: %(module)s
:members:
:undoc-members:
:show-inheritance:
"""
def _find_modules(arg, dirname, files):
for filename in files:
if filename.endswith('.py') and filename != '__init__.py':
arg["%s.%s" % (dirname.replace('/', '.'),
filename[:-3])] = True
class LocalBuildDoc(setup_command.BuildDoc):
command_name = 'build_sphinx'
builders = ['html', 'man']
def _get_source_dir(self):
option_dict = self.distribution.get_option_dict('build_sphinx')
if 'source_dir' in option_dict:
source_dir = os.path.join(option_dict['source_dir'][1], 'api')
else:
source_dir = 'doc/source/api'
if not os.path.exists(source_dir):
os.makedirs(source_dir)
return source_dir
def generate_autoindex(self):
log.info("[pbr] Autodocumenting from %s"
% os.path.abspath(os.curdir))
modules = {}
source_dir = self._get_source_dir()
for pkg in self.distribution.packages:
if '.' not in pkg:
for dirpath, dirnames, files in os.walk(pkg):
_find_modules(modules, dirpath, files)
module_list = list(modules.keys())
module_list.sort()
autoindex_filename = os.path.join(source_dir, 'autoindex.rst')
with open(autoindex_filename, 'w') as autoindex:
autoindex.write(""".. toctree::
:maxdepth: 1
""")
for module in module_list:
output_filename = os.path.join(source_dir,
"%s.rst" % module)
heading = "The :mod:`%s` Module" % module
underline = "=" * len(heading)
values = dict(module=module, heading=heading,
underline=underline)
log.info("[pbr] Generating %s"
% output_filename)
with open(output_filename, 'w') as output_file:
output_file.write(_rst_template % values)
autoindex.write(" %s.rst\n" % module)
def _sphinx_tree(self):
source_dir = self._get_source_dir()
apidoc.main(['apidoc', '.', '-H', 'Modules', '-o', source_dir])
def _sphinx_run(self):
if not self.verbose:
status_stream = cStringIO.StringIO()
else:
status_stream = sys.stdout
confoverrides = {}
if self.version:
confoverrides['version'] = self.version
if self.release:
confoverrides['release'] = self.release
if self.today:
confoverrides['today'] = self.today
sphinx_config = config.Config(self.config_dir, 'conf.py', {}, [])
sphinx_config.init_values()
if self.builder == 'man' and len(sphinx_config.man_pages) == 0:
return
app = application.Sphinx(
self.source_dir, self.config_dir,
self.builder_target_dir, self.doctree_dir,
self.builder, confoverrides, status_stream,
freshenv=self.fresh_env, warningiserror=True)
try:
app.build(force_all=self.all_files)
except Exception as err:
from docutils import utils
if isinstance(err, utils.SystemMessage):
sys.stder.write('reST markup error:\n')
sys.stderr.write(err.args[0].encode('ascii',
'backslashreplace'))
sys.stderr.write('\n')
else:
raise
if self.link_index:
src = app.config.master_doc + app.builder.out_suffix
dst = app.builder.get_outfilename('index')
os.symlink(src, dst)
def run(self):
option_dict = self.distribution.get_option_dict('pbr')
tree_index = options.get_boolean_option(option_dict,
'autodoc_tree_index_modules',
'AUTODOC_TREE_INDEX_MODULES')
auto_index = options.get_boolean_option(option_dict,
'autodoc_index_modules',
'AUTODOC_INDEX_MODULES')
if not os.getenv('SPHINX_DEBUG'):
#NOTE(afazekas): These options can be used together,
# but they do a very similar thing in a difffernet way
if tree_index:
self._sphinx_tree()
if auto_index:
self.generate_autoindex()
for builder in self.builders:
self.builder = builder
self.finalize_options()
self.project = self.distribution.get_name()
self.version = self.distribution.get_version()
self.release = self.distribution.get_version()
if 'warnerrors' in option_dict:
self._sphinx_run()
else:
setup_command.BuildDoc.run(self)
def finalize_options(self):
# Not a new style class, super keyword does not work.
setup_command.BuildDoc.finalize_options(self)
# Allow builders to be configurable - as a comma separated list.
if not isinstance(self.builders, list) and self.builders:
self.builders = self.builders.split(',')
class LocalBuildLatex(LocalBuildDoc):
builders = ['latex']
command_name = 'build_sphinx_latex'

7
pbr/hooks/commands.py
View File

@ -20,6 +20,7 @@ import os
from setuptools.command import easy_install
from pbr.hooks import base
from pbr import options
from pbr import packaging
@ -46,8 +47,8 @@ class CommandsConfig(base.BaseConfig):
easy_install.get_script_args = packaging.override_get_script_args
if packaging.have_sphinx():
self.add_command('pbr.packaging.LocalBuildDoc')
self.add_command('pbr.packaging.LocalBuildLatex')
self.add_command('pbr.builddoc.LocalBuildDoc')
self.add_command('pbr.builddoc.LocalBuildLatex')
if os.path.exists('.testr.conf') and packaging.have_testr():
# There is a .testr.conf file. We want to use it.
@ -56,7 +57,7 @@ class CommandsConfig(base.BaseConfig):
# We seem to still have nose configured
self.add_command('pbr.packaging.NoseTest')
use_egg = packaging.get_boolean_option(
use_egg = options.get_boolean_option(
self.pbr_config, 'use-egg', 'PBR_USE_EGG')
# We always want non-egg install unless explicitly requested
if 'manpages' in self.pbr_config or not use_egg:

48
pbr/options.py Normal file
View File

@ -0,0 +1,48 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Copyright (C) 2013 Association of Universities for Research in Astronomy
# (AURA)
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials provided
# with the distribution.
#
# 3. The name of AURA and its representatives may not be used to
# endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY AURA ``AS IS'' AND ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL AURA BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
import os
TRUE_VALUES = ('true', '1', 'yes')
def get_boolean_option(option_dict, option_name, env_name):
return ((option_name in option_dict
and option_dict[option_name][1].lower() in TRUE_VALUES) or
str(os.getenv(env_name)).lower() in TRUE_VALUES)

184
pbr/packaging.py
View File

@ -40,14 +40,9 @@ from setuptools.command import install
from setuptools.command import install_scripts
from setuptools.command import sdist
try:
import cStringIO
except ImportError:
import io as cStringIO
from pbr import extra_files
from pbr import options
TRUE_VALUES = ('true', '1', 'yes')
REQUIREMENTS_FILES = ('requirements.txt', 'tools/pip-requires')
TEST_REQUIREMENTS_FILES = ('test-requirements.txt', 'tools/test-requires')
@ -76,7 +71,7 @@ def append_text_list(config, key, text_list):
def _pip_install(links, requires, root=None, option_dict=dict()):
if get_boolean_option(
if options.get_boolean_option(
option_dict, 'skip_pip_install', 'SKIP_PIP_INSTALL'):
return
cmd = [sys.executable, '-m', 'pip.__init__', 'install']
@ -236,17 +231,11 @@ def _get_highest_tag(tags):
return max(tags, key=pkg_resources.parse_version)
def get_boolean_option(option_dict, option_name, env_name):
return ((option_name in option_dict
and option_dict[option_name][1].lower() in TRUE_VALUES) or
str(os.getenv(env_name)).lower() in TRUE_VALUES)
def write_git_changelog(git_dir=None, dest_dir=os.path.curdir,
option_dict=dict()):
"""Write a changelog based on the git changelog."""
should_skip = get_boolean_option(option_dict, 'skip_changelog',
'SKIP_WRITE_GIT_CHANGELOG')
should_skip = options.get_boolean_option(option_dict, 'skip_changelog',
'SKIP_WRITE_GIT_CHANGELOG')
if should_skip:
return
@ -303,8 +292,8 @@ def write_git_changelog(git_dir=None, dest_dir=os.path.curdir,
def generate_authors(git_dir=None, dest_dir='.', option_dict=dict()):
"""Create AUTHORS file using git commits."""
should_skip = get_boolean_option(option_dict, 'skip_authors',
'SKIP_GENERATE_AUTHORS')
should_skip = options.get_boolean_option(option_dict, 'skip_authors',
'SKIP_GENERATE_AUTHORS')
if should_skip:
return
@ -360,23 +349,6 @@ def _find_git_files(dirname='', git_dir=None):
return [f for f in file_list if f]
_rst_template = """%(heading)s
%(underline)s
.. automodule:: %(module)s
:members:
:undoc-members:
:show-inheritance:
"""
def _find_modules(arg, dirname, files):
for filename in files:
if filename.endswith('.py') and filename != '__init__.py':
arg["%s.%s" % (dirname.replace('/', '.'),
filename[:-3])] = True
class LocalInstall(install.install):
"""Runs python setup.py install in a sensible manner.
@ -583,8 +555,8 @@ class LocalManifestMaker(egg_info.manifest_maker):
self.filelist.append(self.template)
self.filelist.append(self.manifest)
self.filelist.extend(extra_files.get_extra_files())
should_skip = get_boolean_option(option_dict, 'skip_git_sdist',
'SKIP_GIT_SDIST')
should_skip = options.get_boolean_option(option_dict, 'skip_git_sdist',
'SKIP_GIT_SDIST')
if not should_skip:
rcfiles = _find_git_files()
if rcfiles:
@ -635,142 +607,16 @@ class LocalSDist(sdist.sdist):
sdist.sdist.run(self)
try:
from sphinx import apidoc
from sphinx import application
from sphinx import config
from sphinx import setup_command
class LocalBuildDoc(setup_command.BuildDoc):
command_name = 'build_sphinx'
builders = ['html', 'man']
def _get_source_dir(self):
option_dict = self.distribution.get_option_dict('build_sphinx')
if 'source_dir' in option_dict:
source_dir = os.path.join(option_dict['source_dir'][1], 'api')
else:
source_dir = 'doc/source/api'
if not os.path.exists(source_dir):
os.makedirs(source_dir)
return source_dir
def generate_autoindex(self):
log.info("[pbr] Autodocumenting from %s"
% os.path.abspath(os.curdir))
modules = {}
source_dir = self._get_source_dir()
for pkg in self.distribution.packages:
if '.' not in pkg:
for dirpath, dirnames, files in os.walk(pkg):
_find_modules(modules, dirpath, files)
module_list = list(modules.keys())
module_list.sort()
autoindex_filename = os.path.join(source_dir, 'autoindex.rst')
with open(autoindex_filename, 'w') as autoindex:
autoindex.write(""".. toctree::
:maxdepth: 1
""")
for module in module_list:
output_filename = os.path.join(source_dir,
"%s.rst" % module)
heading = "The :mod:`%s` Module" % module
underline = "=" * len(heading)
values = dict(module=module, heading=heading,
underline=underline)
log.info("[pbr] Generating %s"
% output_filename)
with open(output_filename, 'w') as output_file:
output_file.write(_rst_template % values)
autoindex.write(" %s.rst\n" % module)
def _sphinx_tree(self):
source_dir = self._get_source_dir()
apidoc.main(['apidoc', '.', '-H', 'Modules', '-o', source_dir])
def _sphinx_run(self):
if not self.verbose:
status_stream = cStringIO.StringIO()
else:
status_stream = sys.stdout
confoverrides = {}
if self.version:
confoverrides['version'] = self.version
if self.release:
confoverrides['release'] = self.release
if self.today:
confoverrides['today'] = self.today
sphinx_config = config.Config(self.config_dir, 'conf.py', {}, [])
sphinx_config.init_values()
if self.builder == 'man' and len(sphinx_config.man_pages) == 0:
return
app = application.Sphinx(
self.source_dir, self.config_dir,
self.builder_target_dir, self.doctree_dir,
self.builder, confoverrides, status_stream,
freshenv=self.fresh_env, warningiserror=True)
try:
app.build(force_all=self.all_files)
except Exception as err:
from docutils import utils
if isinstance(err, utils.SystemMessage):
sys.stder.write('reST markup error:\n')
sys.stderr.write(err.args[0].encode('ascii',
'backslashreplace'))
sys.stderr.write('\n')
else:
raise
if self.link_index:
src = app.config.master_doc + app.builder.out_suffix
dst = app.builder.get_outfilename('index')
os.symlink(src, dst)
def run(self):
option_dict = self.distribution.get_option_dict('pbr')
tree_index = get_boolean_option(option_dict,
'autodoc_tree_index_modules',
'AUTODOC_TREE_INDEX_MODULES')
auto_index = get_boolean_option(option_dict,
'autodoc_index_modules',
'AUTODOC_INDEX_MODULES')
if not os.getenv('SPHINX_DEBUG'):
#NOTE(afazekas): These options can be used together,
# but they do a very similar thing in a difffernet way
if tree_index:
self._sphinx_tree()
if auto_index:
self.generate_autoindex()
for builder in self.builders:
self.builder = builder
self.finalize_options()
self.project = self.distribution.get_name()
self.version = self.distribution.get_version()
self.release = self.distribution.get_version()
if 'warnerrors' in option_dict:
self._sphinx_run()
else:
setup_command.BuildDoc.run(self)
def finalize_options(self):
# Not a new style class, super keyword does not work.
setup_command.BuildDoc.finalize_options(self)
# Allow builders to be configurable - as a comma separated list.
if not isinstance(self.builders, list) and self.builders:
self.builders = self.builders.split(',')
class LocalBuildLatex(LocalBuildDoc):
builders = ['latex']
command_name = 'build_sphinx_latex'
from pbr import builddoc
_have_sphinx = True
# Import the symbols from their new home so the package API stays
# compatible.
LocalBuildDoc = builddoc.LocalBuildDoc
LocalBuildLatex = builddoc.LocalBuildLatex
except ImportError:
_have_sphinx = False
LocalBuildDoc = None
LocalBuildLatex = None
def have_sphinx():

6
pbr/tests/base.py
View File

@ -50,7 +50,7 @@ import fixtures
import testresources
import testtools
from pbr import packaging
from pbr import options
class DiveDir(fixtures.Fixture):
@ -83,10 +83,10 @@ class BaseTestCase(testtools.TestCase, testresources.ResourcedTestCase):
if test_timeout > 0:
self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
if os.environ.get('OS_STDOUT_CAPTURE') in packaging.TRUE_VALUES:
if os.environ.get('OS_STDOUT_CAPTURE') in options.TRUE_VALUES:
stdout = self.useFixture(fixtures.StringStream('stdout')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
if os.environ.get('OS_STDERR_CAPTURE') in packaging.TRUE_VALUES:
if os.environ.get('OS_STDERR_CAPTURE') in options.TRUE_VALUES:
stderr = self.useFixture(fixtures.StringStream('stderr')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
self.log_fixture = self.useFixture(

4
pbr/util.py
View File

@ -354,8 +354,8 @@ def setup_cfg_to_setup_kwargs(config):
elif arg == 'cmdclass':
cmdclass = {}
dist = Distribution()
for cls in in_cfg_value:
cls = resolve_name(cls)
for cls_name in in_cfg_value:
cls = resolve_name(cls_name)
cmd = cls(dist)
cmdclass[cmd.get_command_name()] = cls
in_cfg_value = cmdclass

8
tox.ini
View File

@ -1,13 +1,17 @@
[tox]
minversion = 1.6
skipsdist = True
envlist = py26,py27,py33,pypy,pep8
envlist = py33,py34,py26,py27,pypy,pep8
[testenv]
usedevelop = True
install_command = pip install {opts} {packages}
setenv = VIRTUAL_ENV={envdir}
deps = -r{toxinidir}/requirements.txt
# NOTE(dhellmann): List ourself as a dependency first to ensure that
# the source being tested is used to install all of the other
# dependencies that want to use pbr for installation.
deps = .
-r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
commands =
python setup.py testr --testr-args='{posargs}'