[Tempest]: Try to find py27 interpreter

Since Rally team wants to support all Python 2.6, Python 2.7 and Python 3
envs and based on way of launching tempest(Rally install Tempest in virtual
environment), we can try to find and select a suitable python interpreter
for Tempest(Tempest designed only for Python 2.7). It can allow to support
`rally verify` stuff in all python environments.

Change-Id: If8c94c28119a34e60c00f7ef2cb9b94bc2e62004

rally/common/costilius.py

@@ -18,8 +18,14 @@
 This module is a storage for different types of workarounds.
 """
 
+from distutils import spawn
+import os
+import subprocess
 import sys
 
+from rally.common.i18n import _LE
+from rally import exceptions
+
 
 try:
     from collections import OrderedDict  # noqa
@@ -49,3 +55,64 @@ def json_loads(*args, **kwargs):
     """
 
     return json.loads(*args, **kwargs)
+
+
+def sp_check_output(*popenargs, **kwargs):
+    """Run command with arguments and return its output as a byte string.
+
+    If the exit code was non-zero it raises a CalledProcessError.  The
+    CalledProcessError object will have the return code in the returncode
+    attribute and output in the output attribute.
+
+    The arguments are the same as for the Popen constructor.
+    """
+
+    if is_py26():
+        # NOTE(andreykurilin): as I said before, support python 26 env is hard
+        # task. Subprocess supports check_output function from Python 2.7, so
+        # let's copy-paste code of this function from it.
+        if "stdout" in kwargs:
+            raise ValueError("stdout argument not allowed, "
+                             "it will be overridden.")
+        process = subprocess.Popen(stdout=subprocess.PIPE,
+                                   *popenargs, **kwargs)
+        output, unused_err = process.communicate()
+        retcode = process.poll()
+        if retcode:
+            cmd = kwargs.get("args")
+            if cmd is None:
+                cmd = popenargs[0]
+            raise subprocess.CalledProcessError(retcode, cmd, output=output)
+        return output
+    return subprocess.check_output(*popenargs, **kwargs)
+
+
+def get_interpreter(python_version):
+    """Discovers PATH to find proper python interpreter
+
+    :param python_version: (major, minor) version numbers
+    :type python_version: tuple
+    """
+
+    if not isinstance(python_version, tuple):
+        msg = (_LE("given format of python version `%s` is invalid") %
+               python_version)
+        raise exceptions.InvalidArgumentsException(msg)
+
+    interpreter_name = "python%s.%s" % python_version
+    interpreter = spawn.find_executable(interpreter_name)
+    if interpreter:
+        return interpreter
+    else:
+        interpreters = filter(
+            os.path.isfile, [os.path.join(p, interpreter_name)
+                             for p in os.environ.get("PATH", "").split(":")])
+        cmd = "%s -c 'import sys; print(sys.version_info[:2])'"
+        for interpreter in interpreters:
+            try:
+                out = sp_check_output(cmd % interpreter, shell=True)
+            except subprocess.CalledProcessError:
+                pass
+            else:
+                if out.strip() == str(python_version):
+                    return interpreter

rally/exceptions.py

@@ -250,7 +250,7 @@ class ImageCleanUpException(CleanUpException):
 
 class IncompatiblePythonVersion(RallyException):
     msg_fmt = _("Incompatible python version found '%(version)s', "
-                "required at least python>=2.7.x")
+                "required '%(required_version)s'")
 
 
 class WorkerNotFound(NotFoundException):

rally/verification/tempest/tempest.py

@@ -20,7 +20,9 @@ import subprocess
 import sys
 
 from oslo_serialization import jsonutils
+from oslo_utils import encodeutils
 
+from rally.common import costilius
 from rally.common.i18n import _
 from rally.common import log as logging
 from rally.common import utils
@@ -41,14 +43,14 @@ class TempestSetupFailure(exceptions.RallyException):
 def check_output(*args, **kwargs):
     kwargs["stderr"] = subprocess.STDOUT
     try:
-        output = subprocess.check_output(*args, **kwargs)
+        output = costilius.sp_check_output(*args, **kwargs)
     except subprocess.CalledProcessError as e:
         LOG.debug("failed cmd: '%s'" % e.cmd)
-        LOG.debug("error output: '%s'" % e.output)
+        LOG.debug("error output: '%s'" % encodeutils.safe_decode(e.output))
         raise
 
-    if logging.is_debug():
+    LOG.debug("subprocess output: '%s'" % encodeutils.safe_decode(output))
-        print(output)
+    return output
 
 
 class Tempest(object):
@@ -118,7 +120,7 @@ class Tempest(object):
             |_for-deployment-<UUID1> -> copy from relevant tempest base
             |_for-deployment-<UUID2> -> copy from relevant tempest base
 
-       new:
+        new:
             _ rally/tempest
             |_base
             ||_ tempest_base-<rand suffix specific for source> -> clone
@@ -157,21 +159,34 @@ class Tempest(object):
 
     @staticmethod
     def _get_remote_origin(directory):
-        out = subprocess.check_output("git config --get remote.origin.url",
+        out = check_output("git config --get remote.origin.url",
-                                      shell=True,
+                           shell=True, cwd=os.path.abspath(directory))
-                                      cwd=os.path.abspath(directory))
         return out.strip()
 
     def _install_venv(self):
         path_to_venv = self.path(".venv")
 
         if not os.path.isdir(path_to_venv):
-            self.validate_env()
             print("No virtual environment found...Install the virtualenv.")
             LOG.debug("Virtual environment directory: %s" % path_to_venv)
+            required_vers = (2, 7)
+            if sys.version_info[:2] != required_vers:
+                # NOTE(andreykurilin): let's try to find a suitable python
+                # interpreter for Tempest
+                python_interpreter = costilius.get_interpreter(required_vers)
+                if not python_interpreter:
+                    raise exceptions.IncompatiblePythonVersion(
+                        version=sys.version, required_version=required_vers)
+                LOG.info(
+                    _("Tempest requires Python %(required)s, '%(found)s' was "
+                      "found in your system and it will be used for installing"
+                      " virtual environment.") % {"required": required_vers,
+                                                  "found": python_interpreter})
+            else:
+                python_interpreter = sys.executable
             try:
-                check_output("python ./tools/install_venv.py", shell=True,
+                check_output("%s ./tools/install_venv.py" % python_interpreter,
-                             cwd=self.path())
+                             shell=True, cwd=self.path())
                 check_output("%s python setup.py install" % self.venv_wrapper,
                              shell=True, cwd=self.path())
             except subprocess.CalledProcessError:
@@ -342,16 +357,6 @@ class Tempest(object):
         else:
             self.verification.set_failed()
 
-    def validate_env(self):
-        """Validate environment parameters required for running tempest.
-
-           eg: python>2.7
-        """
-
-        if sys.version_info < (2, 7):
-            raise exceptions.IncompatiblePythonVersion(
-                version=sys.version_info)
-
     def verify(self, set_name, regex):
         self._prepare_and_run(set_name, regex)
         self._save_results()

tests/unit/common/test_costilius.py

@@ -0,0 +1,100 @@
+#
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+import mock
+
+from rally.common import costilius
+from rally import exceptions
+from tests.unit import test
+
+
+PATH = "rally.common.costilius"
+
+
+class SPCheckCallTestCase(test.TestCase):
+
+    @mock.patch("%s.subprocess" % PATH)
+    @mock.patch("%s.is_py26" % PATH, return_value=True)
+    def test_simulation_of_py26_env(self, mock_is_py26, mock_sp):
+        output = "output"
+        process = mock.MagicMock()
+        process.communicate.return_value = (output, "unused_err")
+        process.poll.return_value = None
+
+        mock_sp.Popen.return_value = process
+        some_args = (1, 2)
+        some_kwargs = {"a": 2}
+
+        self.assertEqual(output, costilius.sp_check_output(*some_args,
+                                                           **some_kwargs))
+
+        mock_sp.Popen.assert_called_once_with(
+            stdout=mock_sp.PIPE, *some_args, **some_kwargs)
+        self.assertFalse(mock_sp.check_output.called)
+
+    @mock.patch("%s.subprocess" % PATH)
+    @mock.patch("%s.is_py26" % PATH, return_value=False)
+    def test_simulation_of_any_not_py26_env(self, mock_is_py26, mock_sp):
+        output = "output"
+        mock_sp.check_output.return_value = output
+
+        some_args = (1, 2)
+        some_kwargs = {"a": 2}
+
+        self.assertEqual(output, costilius.sp_check_output(*some_args,
+                                                           **some_kwargs))
+
+        mock_sp.check_output.assert_called_once_with(*some_args, **some_kwargs)
+        self.assertFalse(mock_sp.Popen.called)
+
+
+class GetInterpreterTestCase(test.TestCase):
+    def test_wrong_format(self):
+        self.assertRaises(exceptions.InvalidArgumentsException,
+                          costilius.get_interpreter, "something_bad")
+
+    @mock.patch("%s.spawn" % PATH)
+    @mock.patch("%s.sp_check_output" % PATH)
+    @mock.patch("%s.os.path.isfile" % PATH)
+    @mock.patch("%s.os.environ" % PATH)
+    def test_found_correct_python_interpreter_with_distutils(
+            self, mock_env, mock_isfile, mock_check_output, mock_spawn):
+        vers = (2, 7)
+        interpreter = "something"
+        mock_spawn.find_executable.return_value = interpreter
+
+        self.assertEqual(interpreter, costilius.get_interpreter(vers))
+        self.assertFalse(mock_env.called)
+        self.assertFalse(mock_isfile.called)
+        self.assertFalse(mock_check_output.called)
+
+    @mock.patch("%s.spawn" % PATH)
+    @mock.patch("%s.sp_check_output" % PATH)
+    @mock.patch("%s.os.path.isfile" % PATH, return_value=True)
+    @mock.patch("%s.os.environ" % PATH)
+    def test_found_correct_python_interpreter_without_distutils(
+            self, mock_env, mock_isfile, mock_check_output, mock_spawn):
+        vers = (2, 7)
+        paths = ["one_path", "second_path"]
+        mock_env.get.return_value = ":".join(paths)
+        mock_check_output.return_value = "%s\n" % str(vers)
+        mock_spawn.find_executable.return_value = None
+
+        found_interpreter = costilius.get_interpreter(vers)
+
+        self.assertEqual(1, mock_check_output.call_count)
+        self.assertIn(
+            found_interpreter, ["%s/%s" % (f, "python2.7") for f in paths])

tests/unit/verification/test_tempest.py

@@ -16,11 +16,9 @@
 import copy
 import os
 import subprocess
-import sys
 
 import mock
 from oslo_serialization import jsonutils
-import testtools
 
 from rally import exceptions
 from rally.verification.tempest import subunit2json
@@ -88,33 +86,41 @@ class TempestUtilsTestCase(BaseTestCase):
         self.assertFalse(mock_env.copy.called)
 
     @mock.patch("os.path.isdir", return_value=True)
-    @mock.patch(TEMPEST_PATH + ".tempest.subprocess")
+    @mock.patch(TEMPEST_PATH + ".tempest.check_output")
-    @testtools.skipIf(sys.version_info < (2, 7), "Incompatible Python Version")
+    def test__venv_install_when_venv_exists(self, mock_co, mock_isdir):
-    def test__venv_install_when_venv_exists(self, mock_sp, mock_isdir):
         self.verifier._install_venv()
 
         mock_isdir.assert_called_once_with(self.verifier.path(".venv"))
-        self.assertFalse(mock_sp.check_output.called)
+        self.assertFalse(mock_co.called)
 
+    @mock.patch("%s.tempest.sys" % TEMPEST_PATH)
+    @mock.patch("%s.tempest.costilius.get_interpreter" % TEMPEST_PATH,
+                return_value="python")
     @mock.patch("os.path.isdir", return_value=False)
-    @mock.patch("%s.tempest.subprocess.check_output" % TEMPEST_PATH,
+    @mock.patch("%s.tempest.check_output" % TEMPEST_PATH,
                 return_value="some_output")
-    @testtools.skipIf(sys.version_info < (2, 7), "Incompatible Python Version")
+    def test__venv_install_when_venv_not_exist(self, mock_co, mock_isdir,
-    def test__venv_install_when_venv_not_exist(self, mock_sp, mock_isdir):
+                                               mock_get_interpreter, mock_sys):
+        mock_sys.version_info = "not_py27_env"
+
         self.verifier._install_venv()
 
         mock_isdir.assert_called_once_with(self.verifier.path(".venv"))
-        mock_sp.assert_has_calls([
+        mock_co.assert_has_calls([
             mock.call("python ./tools/install_venv.py", shell=True,
-                      cwd=self.verifier.path(), stderr=subprocess.STDOUT),
+                      cwd=self.verifier.path()),
             mock.call("%s python setup.py install" %
                       self.verifier.venv_wrapper, shell=True,
-                      cwd=self.verifier.path(), stderr=subprocess.STDOUT)])
+                      cwd=self.verifier.path())])
 
+    @mock.patch("%s.tempest.sys" % TEMPEST_PATH)
+    @mock.patch("%s.tempest.costilius.get_interpreter" % TEMPEST_PATH,
+                return_value=None)
     @mock.patch("os.path.isdir", return_value=False)
-    @testtools.skipIf(sys.version_info >= (2, 7),
+    def test__venv_install_fails__when_py27_is_not_present(
-                      "Incompatible Python Version")
+            self, mock_isdir, mock_get_interpreter, mock_sys):
-    def test__venv_install_for_py26_fails(self, mock_isdir):
+        mock_sys.version_info = "not_py27_env"
+
         self.assertRaises(exceptions.IncompatiblePythonVersion,
                           self.verifier._install_venv)
 
@@ -130,18 +136,17 @@ class TempestUtilsTestCase(BaseTestCase):
             self.verifier.path(".testrepository"))
         self.assertFalse(mock_sp.called)
 
-    @testtools.skipIf(sys.version_info < (2, 7), "Incompatible Python Version")
     @mock.patch("os.path.isdir", return_value=False)
-    @mock.patch(TEMPEST_PATH + ".tempest.subprocess.check_output")
+    @mock.patch(TEMPEST_PATH + ".tempest.check_output")
     def test__initialize_testr_when_testr_not_initialized(
-            self, mock_sp, mock_isdir):
+            self, mock_co, mock_isdir):
         self.verifier._initialize_testr()
 
         mock_isdir.assert_called_once_with(
             self.verifier.path(".testrepository"))
-        mock_sp.assert_called_once_with(
+        mock_co.assert_called_once_with(
             "%s testr init" % self.verifier.venv_wrapper, shell=True,
-            cwd=self.verifier.path(), stderr=subprocess.STDOUT)
+            cwd=self.verifier.path())
 
     @mock.patch.object(subunit2json, "main")
     @mock.patch("os.path.isfile", return_value=False)
@@ -191,12 +196,11 @@ class TempestInstallAndUninstallTestCase(BaseTestCase):
     def test__is_git_repo(self, mock_isdir, mock_git_status):
         self.assertTrue(self.verifier._is_git_repo("fake_dir"))
 
-    @testtools.skipIf(sys.version_info < (2, 7), "Incompatible Python Version")
+    @mock.patch("%s.tempest.check_output" % TEMPEST_PATH,
-    @mock.patch("subprocess.check_output", return_value="fake_url")
+                return_value="fake_url")
-    def test__get_remote_origin(self, mock_sp):
+    def test__get_remote_origin(self, mock_co):
-        with mock_sp:
+        self.assertEqual("fake_url",
-            self.assertEqual("fake_url",
+                         self.verifier._get_remote_origin("fake_dir"))
-                             self.verifier._get_remote_origin("fake_dir"))
 
     @mock.patch("shutil.rmtree")
     @mock.patch(TEMPEST_PATH + ".tempest.os.path.exists", return_value=True)