diff --git a/pbr/core.py b/pbr/core.py index fee603a5..71d1e569 100644 --- a/pbr/core.py +++ b/pbr/core.py @@ -108,7 +108,7 @@ def pbr(dist, attr, value): # Converts the setup.cfg file to setup() arguments try: - attrs = util.cfg_to_args(path) + attrs = util.cfg_to_args(path, dist.script_args) except Exception: e = sys.exc_info()[1] # NB: This will output to the console if no explicit logging has diff --git a/pbr/tests/test_integration.py b/pbr/tests/test_integration.py index a357e911..d4be9cd7 100644 --- a/pbr/tests/test_integration.py +++ b/pbr/tests/test_integration.py @@ -18,10 +18,10 @@ import sys import fixtures import testtools import textwrap -import virtualenv from pbr.tests import base -from pbr.tests.test_packaging import TestRepo +from pbr.tests.test_packaging import CreatePackages +from pbr.tests.test_packaging import Venv PIPFLAGS = shlex.split(os.environ.get('PIPFLAGS', '')) PIPVERSION = os.environ.get('PIPVERSION', 'pip') @@ -53,41 +53,6 @@ def all_projects(): yield (short_name, dict(name=name, short_name=short_name)) -class Venv(fixtures.Fixture): - """Create a virtual environment for testing with. - - :attr path: The path to the environment root. - :attr python: The path to the python binary in the environment. - """ - - def __init__(self, reason, install_pbr=True): - """Create a Venv fixture. - - :param reason: A human readable string to bake into the venv - file path to aid diagnostics in the case of failures. - :param install_pbr: By default pbr is installed inside the - venv. Setting this to false will disable that. - """ - self._reason = reason - self._install_pbr = install_pbr - - def _setUp(self): - path = self.useFixture(fixtures.TempDir()).path - virtualenv.create_environment(path, clear=True) - python = os.path.join(path, 'bin', 'python') - command = [python] + PIP_CMD + [ - '-U', PIPVERSION, 'wheel'] - if self._install_pbr: - command.append(PBRVERSION) - self.useFixture(base.CapturedSubprocess( - 'mkvenv-' + self._reason, command)) - self.addCleanup(delattr, self, 'path') - self.addCleanup(delattr, self, 'python') - self.path = path - self.python = python - return path, python - - class TestIntegration(base.BaseTestCase): scenarios = list(all_projects()) @@ -126,17 +91,23 @@ class TestIntegration(base.BaseTestCase): self.useFixture(base.CapturedSubprocess( 'clone', ['git', 'clone', os.path.join(REPODIR, self.short_name), path])) - venv = self.useFixture(Venv('sdist')) + venv = self.useFixture(Venv('sdist', + modules=['pip', 'wheel', PBRVERSION], + pip_cmd=PIP_CMD)) python = venv.python self.useFixture(base.CapturedSubprocess( 'sdist', [python, 'setup.py', 'sdist'], cwd=path)) - venv = self.useFixture(Venv('tarball')) + venv = self.useFixture(Venv('tarball', + modules=['pip', 'wheel', PBRVERSION], + pip_cmd=PIP_CMD)) python = venv.python filename = os.path.join( path, 'dist', os.listdir(os.path.join(path, 'dist'))[0]) self.useFixture(base.CapturedSubprocess( 'tarball', [python] + PIP_CMD + [filename])) - venv = self.useFixture(Venv('install-git')) + venv = self.useFixture(Venv('install-git', + modules=['pip', 'wheel', PBRVERSION], + pip_cmd=PIP_CMD)) root = venv.path python = venv.python self.useFixture(base.CapturedSubprocess( @@ -147,7 +118,9 @@ class TestIntegration(base.BaseTestCase): if 'migrate.cfg' in filenames: found = True self.assertTrue(found) - venv = self.useFixture(Venv('install-e')) + venv = self.useFixture(Venv('install-e', + modules=['pip', 'wheel', PBRVERSION], + pip_cmd=PIP_CMD)) root = venv.path python = venv.python self.useFixture(base.CapturedSubprocess( @@ -171,56 +144,83 @@ class TestInstallWithoutPbr(base.BaseTestCase): # testpkg - this requires a pbr-using package test_pkg_dir = os.path.join(tempdir, 'testpkg') os.mkdir(test_pkg_dir) - with open(os.path.join(test_pkg_dir, 'setup.py'), 'wt') as f: - f.write(textwrap.dedent("""\ - #!/usr/bin/env python - import setuptools - setuptools.setup( - name = 'pkgTest', - tests_require = ['pkgReq'], - test_suite='pkgReq' - ) - """)) - with open(os.path.join(test_pkg_dir, 'setup.cfg'), 'wt') as f: - f.write(textwrap.dedent("""\ - [easy_install] - find_links = %s - """ % dist_dir)) - repoTest = self.useFixture(TestRepo(test_pkg_dir)) - repoTest.commit() - # reqpkg - this is a package that requires pbr - req_pkg_dir = os.path.join(tempdir, 'reqpkg') - pkg_req_module = os.path.join(req_pkg_dir, 'pkgReq/') - os.makedirs(pkg_req_module) - with open(os.path.join(req_pkg_dir, 'setup.py'), 'wt') as f: - f.write(textwrap.dedent("""\ - #!/usr/bin/env python - import setuptools - setuptools.setup( - setup_requires=['pbr'], - pbr=True - ) - """)) - with open(os.path.join(req_pkg_dir, 'setup.cfg'), 'wt') as f: - f.write(textwrap.dedent("""\ - [metadata] - name = pkgReq - """)) - with open(os.path.join(req_pkg_dir, 'requirements.txt'), 'wt') as f: - f.write(textwrap.dedent("""\ - pbr - """)) - with open(os.path.join(req_pkg_dir, 'pkgReq/__init__.py'), 'wt') as f: - f.write(textwrap.dedent("""\ - print("FakeTest loaded and ran") - """)) - repoReq = self.useFixture(TestRepo(req_pkg_dir)) - repoReq.commit() + pkgs = { + 'pkgTest': { + 'setup.py': textwrap.dedent("""\ + #!/usr/bin/env python + import setuptools + setuptools.setup( + name = 'pkgTest', + tests_require = ['pkgReq'], + test_suite='pkgReq' + ) + """), + 'setup.cfg': textwrap.dedent("""\ + [easy_install] + find_links = %s + """ % dist_dir)}, + 'pkgReq': { + 'requirements.txt': textwrap.dedent("""\ + pbr + """), + 'pkgReq/__init__.py': textwrap.dedent("""\ + print("FakeTest loaded and ran") + """)}, + } + pkg_dirs = self.useFixture(CreatePackages(pkgs)).package_dirs + test_pkg_dir = pkg_dirs['pkgTest'] + req_pkg_dir = pkg_dirs['pkgReq'] + self._run_cmd(sys.executable, ('setup.py', 'sdist', '-d', dist_dir), allow_fail=False, cwd=req_pkg_dir) # A venv to test within - venv = self.useFixture(Venv('nopbr', install_pbr=False)) + venv = self.useFixture(Venv('nopbr', ['pip', 'wheel'])) python = venv.python # Run the depending script self.useFixture(base.CapturedSubprocess( 'nopbr', [python] + ['setup.py', 'test'], cwd=test_pkg_dir)) + + +class TestMarkersPip(base.BaseTestCase): + + scenarios = [ + ('pip-1.5', {'version': 'pip>=1.5,<1.6'}), + ('pip-6.0', {'version': 'pip>=6.0,<6.1'}), + ('pip-latest', {'version': 'pip'}), + ] + + @testtools.skipUnless( + os.environ.get('PBR_INTEGRATION', None) == '1', + 'integration tests not enabled') + def test_pip_versions(self): + pkgs = { + 'test_markers': + {'requirements.txt': textwrap.dedent("""\ + pkg_a; python_version=='1.2' + pkg_b; python_version!='1.2' + """)}, + 'pkg_a': {}, + 'pkg_b': {}, + } + pkg_dirs = self.useFixture(CreatePackages(pkgs)).package_dirs + temp_dir = self.useFixture(fixtures.TempDir()).path + repo_dir = os.path.join(temp_dir, 'repo') + venv = self.useFixture(Venv('markers')) + bin_python = venv.python + os.mkdir(repo_dir) + for pkg in pkg_dirs: + self._run_cmd( + bin_python, ['setup.py', 'sdist', '-d', repo_dir], + cwd=pkg_dirs[pkg], allow_fail=False) + self._run_cmd( + bin_python, + ['-m', 'pip', 'install', '--upgrade', self.version], + cwd=venv.path, allow_fail=False) + self._run_cmd( + bin_python, + ['-m', 'pip', 'install', '--no-index', '-f', repo_dir, + 'test_markers'], + cwd=venv.path, allow_fail=False) + self.assertIn('pkg-b', self._run_cmd( + bin_python, ['-m', 'pip', 'freeze'], cwd=venv.path, + allow_fail=False)[0]) diff --git a/pbr/tests/test_packaging.py b/pbr/tests/test_packaging.py index 5b7651f0..fbddffde 100644 --- a/pbr/tests/test_packaging.py +++ b/pbr/tests/test_packaging.py @@ -40,7 +40,6 @@ import os import re -import sys import tempfile import textwrap @@ -49,12 +48,16 @@ import mock import pkg_resources import six from testtools import matchers +import virtualenv from pbr import git from pbr import packaging from pbr.tests import base +PBR_ROOT = os.path.abspath(os.path.join(__file__, '..', '..', '..')) + + class TestRepo(fixtures.Fixture): """A git repo for testing with. @@ -144,6 +147,114 @@ class GPGKeyFixture(fixtures.Fixture): tempdir.path) +class Venv(fixtures.Fixture): + """Create a virtual environment for testing with. + + :attr path: The path to the environment root. + :attr python: The path to the python binary in the environment. + """ + + def __init__(self, reason, modules=(), pip_cmd=None): + """Create a Venv fixture. + + :param reason: A human readable string to bake into the venv + file path to aid diagnostics in the case of failures. + :param modules: A list of modules to install, defaults to latest + pip, wheel, and the working copy of PBR. + :attr pip_cmd: A list to override the default pip_cmd passed to + python for installing base packages. + """ + self._reason = reason + if modules == (): + pbr = 'file://%s#egg=pbr' % PBR_ROOT + modules = ['pip', 'wheel', pbr] + self.modules = modules + if pip_cmd is None: + self.pip_cmd = ['-m', 'pip', 'install'] + else: + self.pip_cmd = pip_cmd + + def _setUp(self): + path = self.useFixture(fixtures.TempDir()).path + virtualenv.create_environment(path, clear=True) + python = os.path.join(path, 'bin', 'python') + command = [python] + self.pip_cmd + ['-U'] + if self.modules and len(self.modules) > 0: + command.extend(self.modules) + self.useFixture(base.CapturedSubprocess( + 'mkvenv-' + self._reason, command)) + self.addCleanup(delattr, self, 'path') + self.addCleanup(delattr, self, 'python') + self.path = path + self.python = python + return path, python + + +class CreatePackages(fixtures.Fixture): + """Creates packages from dict with defaults + + :param package_dirs: A dict of package name to directory strings + {'pkg_a': '/tmp/path/to/tmp/pkg_a', 'pkg_b': '/tmp/path/to/tmp/pkg_b'} + """ + + defaults = { + 'setup.py': textwrap.dedent(six.u("""\ + #!/usr/bin/env python + import setuptools + setuptools.setup( + setup_requires=['pbr'], + pbr=True, + ) + """)), + 'setup.cfg': textwrap.dedent(six.u("""\ + [metadata] + name = {pkg_name} + """)) + } + + def __init__(self, packages): + """Creates packages from dict with defaults + + :param packages: a dict where the keys are the package name and a + value that is a second dict that may be empty, containing keys of + filenames and a string value of the contents. + {'package-a': {'requirements.txt': 'string', 'setup.cfg': 'string'} + """ + self.packages = packages + + def _writeFile(self, directory, file_name, contents): + path = os.path.abspath(os.path.join(directory, file_name)) + path_dir = os.path.dirname(path) + if not os.path.exists(path_dir): + if path_dir.startswith(directory): + os.makedirs(path_dir) + else: + raise ValueError + with open(path, 'wt') as f: + f.write(contents) + + def _setUp(self): + tmpdir = self.useFixture(fixtures.TempDir()).path + package_dirs = {} + for pkg_name in self.packages: + pkg_path = os.path.join(tmpdir, pkg_name) + package_dirs[pkg_name] = pkg_path + os.mkdir(pkg_path) + for cf in ['setup.py', 'setup.cfg']: + if cf in self.packages[pkg_name]: + contents = self.packages[pkg_name].pop(cf) + else: + contents = self.defaults[cf].format(pkg_name=pkg_name) + self._writeFile(pkg_path, cf, contents) + + for cf in self.packages[pkg_name]: + self._writeFile(pkg_path, cf, self.packages[pkg_name][cf]) + self.useFixture(TestRepo(pkg_path)).commit() + self.addCleanup(delattr, self, 'package_dirs') + self.package_dirs = package_dirs + return package_dirs + + class TestPackagingInGitRepoWithCommit(base.BaseTestCase): scenarios = [ @@ -459,28 +570,29 @@ class TestVersions(base.BaseTestCase): class TestRequirementParsing(base.BaseTestCase): def test_requirement_parsing(self): - tempdir = self.useFixture(fixtures.TempDir()).path - requirements = os.path.join(tempdir, 'requirements.txt') - with open(requirements, 'wt') as f: - f.write(textwrap.dedent(six.u("""\ - bar - quux<1.0; python_version=='2.6' - requests-aws>=0.1.4 # BSD License (3 clause) - Routes>=1.12.3,!=2.0,!=2.1;python_version=='2.7' - requests-kerberos>=0.6;python_version=='2.7' # MIT - """))) - setup_cfg = os.path.join(tempdir, 'setup.cfg') - with open(setup_cfg, 'wt') as f: - f.write(textwrap.dedent(six.u("""\ - [metadata] - name = test_reqparse + pkgs = { + 'test_reqparse': + { + 'requirements.txt': textwrap.dedent("""\ + bar + quux<1.0; python_version=='2.6' + requests-aws>=0.1.4 # BSD License (3 clause) + Routes>=1.12.3,!=2.0,!=2.1;python_version=='2.7' + requests-kerberos>=0.6;python_version=='2.7' # MIT + """), + 'setup.cfg': textwrap.dedent("""\ + [metadata] + name = test_reqparse - [extras] - test = - foo - baz>3.2 :python_version=='2.7' # MIT - bar>3.3 :python_version=='2.7' # MIT # Apache - """))) + [extras] + test = + foo + baz>3.2 :python_version=='2.7' # MIT + bar>3.3 :python_version=='2.7' # MIT # Apache + """)}, + } + pkg_dirs = self.useFixture(CreatePackages(pkgs)).package_dirs + pkg_dir = pkg_dirs['test_reqparse'] # pkg_resources.split_sections uses None as the title of an # anonymous section instead of the empty string. Weird. expected_requirements = { @@ -491,21 +603,14 @@ class TestRequirementParsing(base.BaseTestCase): 'test': ['foo'], "test:(python_version=='2.7')": ['baz>3.2', 'bar>3.3'] } - - setup_py = os.path.join(tempdir, 'setup.py') - with open(setup_py, 'wt') as f: - f.write(textwrap.dedent(six.u("""\ - #!/usr/bin/env python - import setuptools - setuptools.setup( - setup_requires=['pbr'], - pbr=True, - ) - """))) - - self._run_cmd(sys.executable, (setup_py, 'egg_info'), - allow_fail=False, cwd=tempdir) - egg_info = os.path.join(tempdir, 'test_reqparse.egg-info') + venv = self.useFixture(Venv('reqParse')) + bin_python = venv.python + # Two things are tested by this + # 1) pbr properly parses markers from requiremnts.txt and setup.cfg + # 2) bdist_wheel causes pbr to not evaluate markers + self._run_cmd(bin_python, ('setup.py', 'bdist_wheel'), + allow_fail=False, cwd=pkg_dir) + egg_info = os.path.join(pkg_dir, 'test_reqparse.egg-info') requires_txt = os.path.join(egg_info, 'requires.txt') with open(requires_txt, 'rt') as requires: diff --git a/pbr/util.py b/pbr/util.py index e1989b5e..ce429b96 100644 --- a/pbr/util.py +++ b/pbr/util.py @@ -188,7 +188,7 @@ def resolve_name(name): return ret -def cfg_to_args(path='setup.cfg'): +def cfg_to_args(path='setup.cfg', script_args=()): """ Distutils2 to distutils1 compatibility util. This method uses an existing setup.cfg to generate a dictionary of @@ -196,6 +196,8 @@ def cfg_to_args(path='setup.cfg'): :param file: The setup.cfg path. + :parm script_args: + List of commands setup.py was called with. :raises DistutilsFileError: When the setup.cfg file is not found. @@ -243,7 +245,7 @@ def cfg_to_args(path='setup.cfg'): # Run the pbr hook pbr.hooks.setup_hook(config) - kwargs = setup_cfg_to_setup_kwargs(config) + kwargs = setup_cfg_to_setup_kwargs(config, script_args) # Set default config overrides kwargs['include_package_data'] = True @@ -274,14 +276,14 @@ def cfg_to_args(path='setup.cfg'): return kwargs -def setup_cfg_to_setup_kwargs(config): +def setup_cfg_to_setup_kwargs(config, script_args=()): """Processes the setup.cfg options and converts them to arguments accepted by setuptools' setup() function. """ kwargs = {} - # Temporarily holds install_reqires and extra_requires while we + # Temporarily holds install_requires and extra_requires while we # parse env_markers. all_requirements = {} @@ -420,6 +422,16 @@ def setup_cfg_to_setup_kwargs(config): for requirement, env_marker in all_requirements[req_group]: if env_marker: extras_key = '%s:(%s)' % (req_group, env_marker) + # We do not want to poison wheel creation with locally + # evaluated markers. sdists always re-create the egg_info + # and as such do not need guarded, and pip will never call + # multiple setup.py commands at once. + if 'bdist_wheel' not in script_args: + try: + if pkg_resources.evaluate_marker('(%s)' % env_marker): + extras_key = req_group + except SyntaxError: + pass else: extras_key = req_group extras_require.setdefault(extras_key, []).append(requirement)