Merge "Handle markers to support sdist on pip < 6"

pbr/core.py

@@ -108,7 +108,7 @@ def pbr(dist, attr, value):
 
         # Converts the setup.cfg file to setup() arguments
         try:
-            attrs = util.cfg_to_args(path)
+            attrs = util.cfg_to_args(path, dist.script_args)
         except Exception:
             e = sys.exc_info()[1]
             # NB: This will output to the console if no explicit logging has

pbr/tests/test_integration.py

@@ -18,10 +18,10 @@ import sys
 import fixtures
 import testtools
 import textwrap
-import virtualenv
 
 from pbr.tests import base
-from pbr.tests.test_packaging import TestRepo
+from pbr.tests.test_packaging import CreatePackages
+from pbr.tests.test_packaging import Venv
 
 PIPFLAGS = shlex.split(os.environ.get('PIPFLAGS', ''))
 PIPVERSION = os.environ.get('PIPVERSION', 'pip')
@@ -53,41 +53,6 @@ def all_projects():
         yield (short_name, dict(name=name, short_name=short_name))
 
 
-class Venv(fixtures.Fixture):
-    """Create a virtual environment for testing with.
-
-    :attr path: The path to the environment root.
-    :attr python: The path to the python binary in the environment.
-    """
-
-    def __init__(self, reason, install_pbr=True):
-        """Create a Venv fixture.
-
-        :param reason: A human readable string to bake into the venv
-            file path to aid diagnostics in the case of failures.
-        :param install_pbr: By default pbr is installed inside the
-            venv. Setting this to false will disable that.
-        """
-        self._reason = reason
-        self._install_pbr = install_pbr
-
-    def _setUp(self):
-        path = self.useFixture(fixtures.TempDir()).path
-        virtualenv.create_environment(path, clear=True)
-        python = os.path.join(path, 'bin', 'python')
-        command = [python] + PIP_CMD + [
-            '-U', PIPVERSION, 'wheel']
-        if self._install_pbr:
-            command.append(PBRVERSION)
-        self.useFixture(base.CapturedSubprocess(
-            'mkvenv-' + self._reason, command))
-        self.addCleanup(delattr, self, 'path')
-        self.addCleanup(delattr, self, 'python')
-        self.path = path
-        self.python = python
-        return path, python
-
-
 class TestIntegration(base.BaseTestCase):
 
     scenarios = list(all_projects())
@@ -126,17 +91,23 @@ class TestIntegration(base.BaseTestCase):
         self.useFixture(base.CapturedSubprocess(
             'clone',
             ['git', 'clone', os.path.join(REPODIR, self.short_name), path]))
-        venv = self.useFixture(Venv('sdist'))
+        venv = self.useFixture(Venv('sdist',
+                                    modules=['pip', 'wheel', PBRVERSION],
+                                    pip_cmd=PIP_CMD))
         python = venv.python
         self.useFixture(base.CapturedSubprocess(
             'sdist', [python, 'setup.py', 'sdist'], cwd=path))
-        venv = self.useFixture(Venv('tarball'))
+        venv = self.useFixture(Venv('tarball',
+                                    modules=['pip', 'wheel', PBRVERSION],
+                                    pip_cmd=PIP_CMD))
         python = venv.python
         filename = os.path.join(
             path, 'dist', os.listdir(os.path.join(path, 'dist'))[0])
         self.useFixture(base.CapturedSubprocess(
             'tarball', [python] + PIP_CMD + [filename]))
-        venv = self.useFixture(Venv('install-git'))
+        venv = self.useFixture(Venv('install-git',
+                                    modules=['pip', 'wheel', PBRVERSION],
+                                    pip_cmd=PIP_CMD))
         root = venv.path
         python = venv.python
         self.useFixture(base.CapturedSubprocess(
@@ -147,7 +118,9 @@ class TestIntegration(base.BaseTestCase):
                 if 'migrate.cfg' in filenames:
                     found = True
             self.assertTrue(found)
-        venv = self.useFixture(Venv('install-e'))
+        venv = self.useFixture(Venv('install-e',
+                                    modules=['pip', 'wheel', PBRVERSION],
+                                    pip_cmd=PIP_CMD))
         root = venv.path
         python = venv.python
         self.useFixture(base.CapturedSubprocess(
@@ -171,56 +144,83 @@ class TestInstallWithoutPbr(base.BaseTestCase):
         # testpkg - this requires a pbr-using package
         test_pkg_dir = os.path.join(tempdir, 'testpkg')
         os.mkdir(test_pkg_dir)
-        with open(os.path.join(test_pkg_dir, 'setup.py'), 'wt') as f:
+        pkgs = {
-            f.write(textwrap.dedent("""\
+            'pkgTest': {
-                #!/usr/bin/env python
+                'setup.py': textwrap.dedent("""\
-                import setuptools
+                    #!/usr/bin/env python
-                setuptools.setup(
+                    import setuptools
-                    name = 'pkgTest',
+                    setuptools.setup(
-                    tests_require = ['pkgReq'],
+                        name = 'pkgTest',
-                    test_suite='pkgReq'
+                        tests_require = ['pkgReq'],
-                )
+                        test_suite='pkgReq'
-                """))
+                    )
-        with open(os.path.join(test_pkg_dir, 'setup.cfg'), 'wt') as f:
+                """),
-            f.write(textwrap.dedent("""\
+                'setup.cfg': textwrap.dedent("""\
-                [easy_install]
+                    [easy_install]
-                find_links = %s
+                    find_links = %s
-                """ % dist_dir))
+                """ % dist_dir)},
-        repoTest = self.useFixture(TestRepo(test_pkg_dir))
+            'pkgReq': {
-        repoTest.commit()
+                'requirements.txt': textwrap.dedent("""\
-        # reqpkg - this is a package that requires pbr
+                    pbr
-        req_pkg_dir = os.path.join(tempdir, 'reqpkg')
+                """),
-        pkg_req_module = os.path.join(req_pkg_dir, 'pkgReq/')
+                'pkgReq/__init__.py': textwrap.dedent("""\
-        os.makedirs(pkg_req_module)
+                    print("FakeTest loaded and ran")
-        with open(os.path.join(req_pkg_dir, 'setup.py'), 'wt') as f:
+                """)},
-            f.write(textwrap.dedent("""\
+        }
-                #!/usr/bin/env python
+        pkg_dirs = self.useFixture(CreatePackages(pkgs)).package_dirs
-                import setuptools
+        test_pkg_dir = pkg_dirs['pkgTest']
-                setuptools.setup(
+        req_pkg_dir = pkg_dirs['pkgReq']
-                    setup_requires=['pbr'],
+
-                    pbr=True
-                )
-                """))
-        with open(os.path.join(req_pkg_dir, 'setup.cfg'), 'wt') as f:
-            f.write(textwrap.dedent("""\
-                [metadata]
-                name = pkgReq
-                """))
-        with open(os.path.join(req_pkg_dir, 'requirements.txt'), 'wt') as f:
-            f.write(textwrap.dedent("""\
-                pbr
-                """))
-        with open(os.path.join(req_pkg_dir, 'pkgReq/__init__.py'), 'wt') as f:
-            f.write(textwrap.dedent("""\
-                print("FakeTest loaded and ran")
-                """))
-        repoReq = self.useFixture(TestRepo(req_pkg_dir))
-        repoReq.commit()
         self._run_cmd(sys.executable, ('setup.py', 'sdist', '-d', dist_dir),
                       allow_fail=False, cwd=req_pkg_dir)
         # A venv to test within
-        venv = self.useFixture(Venv('nopbr', install_pbr=False))
+        venv = self.useFixture(Venv('nopbr', ['pip', 'wheel']))
         python = venv.python
         # Run the depending script
         self.useFixture(base.CapturedSubprocess(
             'nopbr', [python] + ['setup.py', 'test'], cwd=test_pkg_dir))
+
+
+class TestMarkersPip(base.BaseTestCase):
+
+    scenarios = [
+        ('pip-1.5', {'version': 'pip>=1.5,<1.6'}),
+        ('pip-6.0', {'version': 'pip>=6.0,<6.1'}),
+        ('pip-latest', {'version': 'pip'}),
+    ]
+
+    @testtools.skipUnless(
+        os.environ.get('PBR_INTEGRATION', None) == '1',
+        'integration tests not enabled')
+    def test_pip_versions(self):
+        pkgs = {
+            'test_markers':
+                {'requirements.txt': textwrap.dedent("""\
+                    pkg_a; python_version=='1.2'
+                    pkg_b; python_version!='1.2'
+                """)},
+            'pkg_a': {},
+            'pkg_b': {},
+        }
+        pkg_dirs = self.useFixture(CreatePackages(pkgs)).package_dirs
+        temp_dir = self.useFixture(fixtures.TempDir()).path
+        repo_dir = os.path.join(temp_dir, 'repo')
+        venv = self.useFixture(Venv('markers'))
+        bin_python = venv.python
+        os.mkdir(repo_dir)
+        for pkg in pkg_dirs:
+            self._run_cmd(
+                bin_python, ['setup.py', 'sdist', '-d', repo_dir],
+                cwd=pkg_dirs[pkg], allow_fail=False)
+        self._run_cmd(
+            bin_python,
+            ['-m', 'pip', 'install', '--upgrade', self.version],
+            cwd=venv.path, allow_fail=False)
+        self._run_cmd(
+            bin_python,
+            ['-m', 'pip', 'install', '--no-index', '-f', repo_dir,
+             'test_markers'],
+            cwd=venv.path, allow_fail=False)
+        self.assertIn('pkg-b', self._run_cmd(
+            bin_python, ['-m', 'pip', 'freeze'], cwd=venv.path,
+            allow_fail=False)[0])

pbr/tests/test_packaging.py

@@ -40,7 +40,6 @@
 
 import os
 import re
-import sys
 import tempfile
 import textwrap
 
@@ -49,12 +48,16 @@ import mock
 import pkg_resources
 import six
 from testtools import matchers
+import virtualenv
 
 from pbr import git
 from pbr import packaging
 from pbr.tests import base
 
 
+PBR_ROOT = os.path.abspath(os.path.join(__file__, '..', '..', '..'))
+
+
 class TestRepo(fixtures.Fixture):
     """A git repo for testing with.
 
@@ -144,6 +147,114 @@ class GPGKeyFixture(fixtures.Fixture):
             tempdir.path)
 
 
+class Venv(fixtures.Fixture):
+    """Create a virtual environment for testing with.
+
+    :attr path: The path to the environment root.
+    :attr python: The path to the python binary in the environment.
+    """
+
+    def __init__(self, reason, modules=(), pip_cmd=None):
+        """Create a Venv fixture.
+
+        :param reason: A human readable string to bake into the venv
+            file path to aid diagnostics in the case of failures.
+        :param modules: A list of modules to install, defaults to latest
+            pip, wheel, and the working copy of PBR.
+        :attr pip_cmd: A list to override the default pip_cmd passed to
+            python for installing base packages.
+        """
+        self._reason = reason
+        if modules == ():
+            pbr = 'file://%s#egg=pbr' % PBR_ROOT
+            modules = ['pip', 'wheel', pbr]
+        self.modules = modules
+        if pip_cmd is None:
+            self.pip_cmd = ['-m', 'pip', 'install']
+        else:
+            self.pip_cmd = pip_cmd
+
+    def _setUp(self):
+        path = self.useFixture(fixtures.TempDir()).path
+        virtualenv.create_environment(path, clear=True)
+        python = os.path.join(path, 'bin', 'python')
+        command = [python] + self.pip_cmd + ['-U']
+        if self.modules and len(self.modules) > 0:
+            command.extend(self.modules)
+            self.useFixture(base.CapturedSubprocess(
+                'mkvenv-' + self._reason, command))
+        self.addCleanup(delattr, self, 'path')
+        self.addCleanup(delattr, self, 'python')
+        self.path = path
+        self.python = python
+        return path, python
+
+
+class CreatePackages(fixtures.Fixture):
+    """Creates packages from dict with defaults
+
+        :param package_dirs: A dict of package name to directory strings
+        {'pkg_a': '/tmp/path/to/tmp/pkg_a', 'pkg_b': '/tmp/path/to/tmp/pkg_b'}
+    """
+
+    defaults = {
+        'setup.py': textwrap.dedent(six.u("""\
+            #!/usr/bin/env python
+            import setuptools
+            setuptools.setup(
+                setup_requires=['pbr'],
+                pbr=True,
+            )
+        """)),
+        'setup.cfg': textwrap.dedent(six.u("""\
+            [metadata]
+            name = {pkg_name}
+        """))
+    }
+
+    def __init__(self, packages):
+        """Creates packages from dict with defaults
+
+            :param packages: a dict where the keys are the package name and a
+            value that is a second dict that may be empty, containing keys of
+            filenames and a string value of the contents.
+            {'package-a': {'requirements.txt': 'string', 'setup.cfg': 'string'}
+        """
+        self.packages = packages
+
+    def _writeFile(self, directory, file_name, contents):
+        path = os.path.abspath(os.path.join(directory, file_name))
+        path_dir = os.path.dirname(path)
+        if not os.path.exists(path_dir):
+            if path_dir.startswith(directory):
+                os.makedirs(path_dir)
+            else:
+                raise ValueError
+        with open(path, 'wt') as f:
+            f.write(contents)
+
+    def _setUp(self):
+        tmpdir = self.useFixture(fixtures.TempDir()).path
+        package_dirs = {}
+        for pkg_name in self.packages:
+            pkg_path = os.path.join(tmpdir, pkg_name)
+            package_dirs[pkg_name] = pkg_path
+            os.mkdir(pkg_path)
+            for cf in ['setup.py', 'setup.cfg']:
+                if cf in self.packages[pkg_name]:
+                    contents = self.packages[pkg_name].pop(cf)
+                else:
+                    contents = self.defaults[cf].format(pkg_name=pkg_name)
+                self._writeFile(pkg_path, cf, contents)
+
+            for cf in self.packages[pkg_name]:
+                self._writeFile(pkg_path, cf, self.packages[pkg_name][cf])
+            self.useFixture(TestRepo(pkg_path)).commit()
+        self.addCleanup(delattr, self, 'package_dirs')
+        self.package_dirs = package_dirs
+        return package_dirs
+
+
 class TestPackagingInGitRepoWithCommit(base.BaseTestCase):
 
     scenarios = [
@@ -459,28 +570,29 @@ class TestVersions(base.BaseTestCase):
 class TestRequirementParsing(base.BaseTestCase):
 
     def test_requirement_parsing(self):
-        tempdir = self.useFixture(fixtures.TempDir()).path
+        pkgs = {
-        requirements = os.path.join(tempdir, 'requirements.txt')
+            'test_reqparse':
-        with open(requirements, 'wt') as f:
+                {
-            f.write(textwrap.dedent(six.u("""\
+                    'requirements.txt': textwrap.dedent("""\
-                bar
+                        bar
-                quux<1.0; python_version=='2.6'
+                        quux<1.0; python_version=='2.6'
-                requests-aws>=0.1.4    # BSD License (3 clause)
+                        requests-aws>=0.1.4    # BSD License (3 clause)
-                Routes>=1.12.3,!=2.0,!=2.1;python_version=='2.7'
+                        Routes>=1.12.3,!=2.0,!=2.1;python_version=='2.7'
-                requests-kerberos>=0.6;python_version=='2.7' # MIT
+                        requests-kerberos>=0.6;python_version=='2.7' # MIT
-            """)))
+                    """),
-        setup_cfg = os.path.join(tempdir, 'setup.cfg')
+                    'setup.cfg': textwrap.dedent("""\
-        with open(setup_cfg, 'wt') as f:
+                        [metadata]
-            f.write(textwrap.dedent(six.u("""\
+                        name = test_reqparse
-                [metadata]
-                name = test_reqparse
 
-                [extras]
+                        [extras]
-                test =
+                        test =
-                    foo
+                            foo
-                    baz>3.2 :python_version=='2.7' # MIT
+                            baz>3.2 :python_version=='2.7' # MIT
-                    bar>3.3 :python_version=='2.7' # MIT # Apache
+                            bar>3.3 :python_version=='2.7' # MIT # Apache
-            """)))
+                    """)},
+        }
+        pkg_dirs = self.useFixture(CreatePackages(pkgs)).package_dirs
+        pkg_dir = pkg_dirs['test_reqparse']
         # pkg_resources.split_sections uses None as the title of an
         # anonymous section instead of the empty string. Weird.
         expected_requirements = {
@@ -491,21 +603,14 @@ class TestRequirementParsing(base.BaseTestCase):
             'test': ['foo'],
             "test:(python_version=='2.7')": ['baz>3.2', 'bar>3.3']
         }
-
+        venv = self.useFixture(Venv('reqParse'))
-        setup_py = os.path.join(tempdir, 'setup.py')
+        bin_python = venv.python
-        with open(setup_py, 'wt') as f:
+        # Two things are tested by this
-            f.write(textwrap.dedent(six.u("""\
+        # 1) pbr properly parses markers from requiremnts.txt and setup.cfg
-                #!/usr/bin/env python
+        # 2) bdist_wheel causes pbr to not evaluate markers
-                import setuptools
+        self._run_cmd(bin_python, ('setup.py', 'bdist_wheel'),
-                setuptools.setup(
+                      allow_fail=False, cwd=pkg_dir)
-                    setup_requires=['pbr'],
+        egg_info = os.path.join(pkg_dir, 'test_reqparse.egg-info')
-                    pbr=True,
-                )
-            """)))
-
-        self._run_cmd(sys.executable, (setup_py, 'egg_info'),
-                      allow_fail=False, cwd=tempdir)
-        egg_info = os.path.join(tempdir, 'test_reqparse.egg-info')
 
         requires_txt = os.path.join(egg_info, 'requires.txt')
         with open(requires_txt, 'rt') as requires:

pbr/util.py

@@ -188,7 +188,7 @@ def resolve_name(name):
     return ret
 
 
-def cfg_to_args(path='setup.cfg'):
+def cfg_to_args(path='setup.cfg', script_args=()):
     """ Distutils2 to distutils1 compatibility util.
 
         This method uses an existing setup.cfg to generate a dictionary of
@@ -196,6 +196,8 @@ def cfg_to_args(path='setup.cfg'):
 
         :param file:
             The setup.cfg path.
+        :parm script_args:
+            List of commands setup.py was called with.
         :raises DistutilsFileError:
             When the setup.cfg file is not found.
 
@@ -243,7 +245,7 @@ def cfg_to_args(path='setup.cfg'):
         # Run the pbr hook
         pbr.hooks.setup_hook(config)
 
-        kwargs = setup_cfg_to_setup_kwargs(config)
+        kwargs = setup_cfg_to_setup_kwargs(config, script_args)
 
         # Set default config overrides
         kwargs['include_package_data'] = True
@@ -274,14 +276,14 @@ def cfg_to_args(path='setup.cfg'):
     return kwargs
 
 
-def setup_cfg_to_setup_kwargs(config):
+def setup_cfg_to_setup_kwargs(config, script_args=()):
     """Processes the setup.cfg options and converts them to arguments accepted
     by setuptools' setup() function.
     """
 
     kwargs = {}
 
-    # Temporarily holds install_reqires and extra_requires while we
+    # Temporarily holds install_requires and extra_requires while we
     # parse env_markers.
     all_requirements = {}
 
@@ -420,6 +422,16 @@ def setup_cfg_to_setup_kwargs(config):
         for requirement, env_marker in all_requirements[req_group]:
             if env_marker:
                 extras_key = '%s:(%s)' % (req_group, env_marker)
+                # We do not want to poison wheel creation with locally
+                # evaluated markers.  sdists always re-create the egg_info
+                # and as such do not need guarded, and pip will never call
+                # multiple setup.py commands at once.
+                if 'bdist_wheel' not in script_args:
+                    try:
+                        if pkg_resources.evaluate_marker('(%s)' % env_marker):
+                            extras_key = req_group
+                    except SyntaxError:
+                        pass
             else:
                 extras_key = req_group
             extras_require.setdefault(extras_key, []).append(requirement)