diff --git a/rally/common/costilius.py b/rally/common/costilius.py index 4a19eb16fa..18e8857e76 100644 --- a/rally/common/costilius.py +++ b/rally/common/costilius.py @@ -18,8 +18,14 @@ This module is a storage for different types of workarounds. """ +from distutils import spawn +import os +import subprocess import sys +from rally.common.i18n import _LE +from rally import exceptions + try: from collections import OrderedDict # noqa @@ -49,3 +55,64 @@ def json_loads(*args, **kwargs): """ return json.loads(*args, **kwargs) + + +def sp_check_output(*popenargs, **kwargs): + """Run command with arguments and return its output as a byte string. + + If the exit code was non-zero it raises a CalledProcessError. The + CalledProcessError object will have the return code in the returncode + attribute and output in the output attribute. + + The arguments are the same as for the Popen constructor. + """ + + if is_py26(): + # NOTE(andreykurilin): as I said before, support python 26 env is hard + # task. Subprocess supports check_output function from Python 2.7, so + # let's copy-paste code of this function from it. + if "stdout" in kwargs: + raise ValueError("stdout argument not allowed, " + "it will be overridden.") + process = subprocess.Popen(stdout=subprocess.PIPE, + *popenargs, **kwargs) + output, unused_err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise subprocess.CalledProcessError(retcode, cmd, output=output) + return output + return subprocess.check_output(*popenargs, **kwargs) + + +def get_interpreter(python_version): + """Discovers PATH to find proper python interpreter + + :param python_version: (major, minor) version numbers + :type python_version: tuple + """ + + if not isinstance(python_version, tuple): + msg = (_LE("given format of python version `%s` is invalid") % + python_version) + raise exceptions.InvalidArgumentsException(msg) + + interpreter_name = "python%s.%s" % python_version + interpreter = spawn.find_executable(interpreter_name) + if interpreter: + return interpreter + else: + interpreters = filter( + os.path.isfile, [os.path.join(p, interpreter_name) + for p in os.environ.get("PATH", "").split(":")]) + cmd = "%s -c 'import sys; print(sys.version_info[:2])'" + for interpreter in interpreters: + try: + out = sp_check_output(cmd % interpreter, shell=True) + except subprocess.CalledProcessError: + pass + else: + if out.strip() == str(python_version): + return interpreter diff --git a/rally/exceptions.py b/rally/exceptions.py index 0641b669e0..818288c524 100644 --- a/rally/exceptions.py +++ b/rally/exceptions.py @@ -250,7 +250,7 @@ class ImageCleanUpException(CleanUpException): class IncompatiblePythonVersion(RallyException): msg_fmt = _("Incompatible python version found '%(version)s', " - "required at least python>=2.7.x") + "required '%(required_version)s'") class WorkerNotFound(NotFoundException): diff --git a/rally/verification/tempest/tempest.py b/rally/verification/tempest/tempest.py index 81b9c7160a..12cf2b9df5 100644 --- a/rally/verification/tempest/tempest.py +++ b/rally/verification/tempest/tempest.py @@ -20,7 +20,9 @@ import subprocess import sys from oslo_serialization import jsonutils +from oslo_utils import encodeutils +from rally.common import costilius from rally.common.i18n import _ from rally.common import log as logging from rally.common import utils @@ -41,14 +43,14 @@ class TempestSetupFailure(exceptions.RallyException): def check_output(*args, **kwargs): kwargs["stderr"] = subprocess.STDOUT try: - output = subprocess.check_output(*args, **kwargs) + output = costilius.sp_check_output(*args, **kwargs) except subprocess.CalledProcessError as e: LOG.debug("failed cmd: '%s'" % e.cmd) - LOG.debug("error output: '%s'" % e.output) + LOG.debug("error output: '%s'" % encodeutils.safe_decode(e.output)) raise - if logging.is_debug(): - print(output) + LOG.debug("subprocess output: '%s'" % encodeutils.safe_decode(output)) + return output class Tempest(object): @@ -118,7 +120,7 @@ class Tempest(object): |_for-deployment- -> copy from relevant tempest base |_for-deployment- -> copy from relevant tempest base - new: + new: _ rally/tempest |_base ||_ tempest_base- -> clone @@ -157,21 +159,34 @@ class Tempest(object): @staticmethod def _get_remote_origin(directory): - out = subprocess.check_output("git config --get remote.origin.url", - shell=True, - cwd=os.path.abspath(directory)) + out = check_output("git config --get remote.origin.url", + shell=True, cwd=os.path.abspath(directory)) return out.strip() def _install_venv(self): path_to_venv = self.path(".venv") if not os.path.isdir(path_to_venv): - self.validate_env() print("No virtual environment found...Install the virtualenv.") LOG.debug("Virtual environment directory: %s" % path_to_venv) + required_vers = (2, 7) + if sys.version_info[:2] != required_vers: + # NOTE(andreykurilin): let's try to find a suitable python + # interpreter for Tempest + python_interpreter = costilius.get_interpreter(required_vers) + if not python_interpreter: + raise exceptions.IncompatiblePythonVersion( + version=sys.version, required_version=required_vers) + LOG.info( + _("Tempest requires Python %(required)s, '%(found)s' was " + "found in your system and it will be used for installing" + " virtual environment.") % {"required": required_vers, + "found": python_interpreter}) + else: + python_interpreter = sys.executable try: - check_output("python ./tools/install_venv.py", shell=True, - cwd=self.path()) + check_output("%s ./tools/install_venv.py" % python_interpreter, + shell=True, cwd=self.path()) check_output("%s python setup.py install" % self.venv_wrapper, shell=True, cwd=self.path()) except subprocess.CalledProcessError: @@ -342,16 +357,6 @@ class Tempest(object): else: self.verification.set_failed() - def validate_env(self): - """Validate environment parameters required for running tempest. - - eg: python>2.7 - """ - - if sys.version_info < (2, 7): - raise exceptions.IncompatiblePythonVersion( - version=sys.version_info) - def verify(self, set_name, regex): self._prepare_and_run(set_name, regex) self._save_results() diff --git a/tests/unit/common/test_costilius.py b/tests/unit/common/test_costilius.py new file mode 100644 index 0000000000..10895621aa --- /dev/null +++ b/tests/unit/common/test_costilius.py @@ -0,0 +1,100 @@ +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +import mock + +from rally.common import costilius +from rally import exceptions +from tests.unit import test + + +PATH = "rally.common.costilius" + + +class SPCheckCallTestCase(test.TestCase): + + @mock.patch("%s.subprocess" % PATH) + @mock.patch("%s.is_py26" % PATH, return_value=True) + def test_simulation_of_py26_env(self, mock_is_py26, mock_sp): + output = "output" + process = mock.MagicMock() + process.communicate.return_value = (output, "unused_err") + process.poll.return_value = None + + mock_sp.Popen.return_value = process + some_args = (1, 2) + some_kwargs = {"a": 2} + + self.assertEqual(output, costilius.sp_check_output(*some_args, + **some_kwargs)) + + mock_sp.Popen.assert_called_once_with( + stdout=mock_sp.PIPE, *some_args, **some_kwargs) + self.assertFalse(mock_sp.check_output.called) + + @mock.patch("%s.subprocess" % PATH) + @mock.patch("%s.is_py26" % PATH, return_value=False) + def test_simulation_of_any_not_py26_env(self, mock_is_py26, mock_sp): + output = "output" + mock_sp.check_output.return_value = output + + some_args = (1, 2) + some_kwargs = {"a": 2} + + self.assertEqual(output, costilius.sp_check_output(*some_args, + **some_kwargs)) + + mock_sp.check_output.assert_called_once_with(*some_args, **some_kwargs) + self.assertFalse(mock_sp.Popen.called) + + +class GetInterpreterTestCase(test.TestCase): + def test_wrong_format(self): + self.assertRaises(exceptions.InvalidArgumentsException, + costilius.get_interpreter, "something_bad") + + @mock.patch("%s.spawn" % PATH) + @mock.patch("%s.sp_check_output" % PATH) + @mock.patch("%s.os.path.isfile" % PATH) + @mock.patch("%s.os.environ" % PATH) + def test_found_correct_python_interpreter_with_distutils( + self, mock_env, mock_isfile, mock_check_output, mock_spawn): + vers = (2, 7) + interpreter = "something" + mock_spawn.find_executable.return_value = interpreter + + self.assertEqual(interpreter, costilius.get_interpreter(vers)) + self.assertFalse(mock_env.called) + self.assertFalse(mock_isfile.called) + self.assertFalse(mock_check_output.called) + + @mock.patch("%s.spawn" % PATH) + @mock.patch("%s.sp_check_output" % PATH) + @mock.patch("%s.os.path.isfile" % PATH, return_value=True) + @mock.patch("%s.os.environ" % PATH) + def test_found_correct_python_interpreter_without_distutils( + self, mock_env, mock_isfile, mock_check_output, mock_spawn): + vers = (2, 7) + paths = ["one_path", "second_path"] + mock_env.get.return_value = ":".join(paths) + mock_check_output.return_value = "%s\n" % str(vers) + mock_spawn.find_executable.return_value = None + + found_interpreter = costilius.get_interpreter(vers) + + self.assertEqual(1, mock_check_output.call_count) + self.assertIn( + found_interpreter, ["%s/%s" % (f, "python2.7") for f in paths]) diff --git a/tests/unit/verification/test_tempest.py b/tests/unit/verification/test_tempest.py index 33585bceb8..031380fc41 100644 --- a/tests/unit/verification/test_tempest.py +++ b/tests/unit/verification/test_tempest.py @@ -16,11 +16,9 @@ import copy import os import subprocess -import sys import mock from oslo_serialization import jsonutils -import testtools from rally import exceptions from rally.verification.tempest import subunit2json @@ -88,33 +86,41 @@ class TempestUtilsTestCase(BaseTestCase): self.assertFalse(mock_env.copy.called) @mock.patch("os.path.isdir", return_value=True) - @mock.patch(TEMPEST_PATH + ".tempest.subprocess") - @testtools.skipIf(sys.version_info < (2, 7), "Incompatible Python Version") - def test__venv_install_when_venv_exists(self, mock_sp, mock_isdir): + @mock.patch(TEMPEST_PATH + ".tempest.check_output") + def test__venv_install_when_venv_exists(self, mock_co, mock_isdir): self.verifier._install_venv() mock_isdir.assert_called_once_with(self.verifier.path(".venv")) - self.assertFalse(mock_sp.check_output.called) + self.assertFalse(mock_co.called) + @mock.patch("%s.tempest.sys" % TEMPEST_PATH) + @mock.patch("%s.tempest.costilius.get_interpreter" % TEMPEST_PATH, + return_value="python") @mock.patch("os.path.isdir", return_value=False) - @mock.patch("%s.tempest.subprocess.check_output" % TEMPEST_PATH, + @mock.patch("%s.tempest.check_output" % TEMPEST_PATH, return_value="some_output") - @testtools.skipIf(sys.version_info < (2, 7), "Incompatible Python Version") - def test__venv_install_when_venv_not_exist(self, mock_sp, mock_isdir): + def test__venv_install_when_venv_not_exist(self, mock_co, mock_isdir, + mock_get_interpreter, mock_sys): + mock_sys.version_info = "not_py27_env" + self.verifier._install_venv() mock_isdir.assert_called_once_with(self.verifier.path(".venv")) - mock_sp.assert_has_calls([ + mock_co.assert_has_calls([ mock.call("python ./tools/install_venv.py", shell=True, - cwd=self.verifier.path(), stderr=subprocess.STDOUT), + cwd=self.verifier.path()), mock.call("%s python setup.py install" % self.verifier.venv_wrapper, shell=True, - cwd=self.verifier.path(), stderr=subprocess.STDOUT)]) + cwd=self.verifier.path())]) + @mock.patch("%s.tempest.sys" % TEMPEST_PATH) + @mock.patch("%s.tempest.costilius.get_interpreter" % TEMPEST_PATH, + return_value=None) @mock.patch("os.path.isdir", return_value=False) - @testtools.skipIf(sys.version_info >= (2, 7), - "Incompatible Python Version") - def test__venv_install_for_py26_fails(self, mock_isdir): + def test__venv_install_fails__when_py27_is_not_present( + self, mock_isdir, mock_get_interpreter, mock_sys): + mock_sys.version_info = "not_py27_env" + self.assertRaises(exceptions.IncompatiblePythonVersion, self.verifier._install_venv) @@ -130,18 +136,17 @@ class TempestUtilsTestCase(BaseTestCase): self.verifier.path(".testrepository")) self.assertFalse(mock_sp.called) - @testtools.skipIf(sys.version_info < (2, 7), "Incompatible Python Version") @mock.patch("os.path.isdir", return_value=False) - @mock.patch(TEMPEST_PATH + ".tempest.subprocess.check_output") + @mock.patch(TEMPEST_PATH + ".tempest.check_output") def test__initialize_testr_when_testr_not_initialized( - self, mock_sp, mock_isdir): + self, mock_co, mock_isdir): self.verifier._initialize_testr() mock_isdir.assert_called_once_with( self.verifier.path(".testrepository")) - mock_sp.assert_called_once_with( + mock_co.assert_called_once_with( "%s testr init" % self.verifier.venv_wrapper, shell=True, - cwd=self.verifier.path(), stderr=subprocess.STDOUT) + cwd=self.verifier.path()) @mock.patch.object(subunit2json, "main") @mock.patch("os.path.isfile", return_value=False) @@ -191,12 +196,11 @@ class TempestInstallAndUninstallTestCase(BaseTestCase): def test__is_git_repo(self, mock_isdir, mock_git_status): self.assertTrue(self.verifier._is_git_repo("fake_dir")) - @testtools.skipIf(sys.version_info < (2, 7), "Incompatible Python Version") - @mock.patch("subprocess.check_output", return_value="fake_url") - def test__get_remote_origin(self, mock_sp): - with mock_sp: - self.assertEqual("fake_url", - self.verifier._get_remote_origin("fake_dir")) + @mock.patch("%s.tempest.check_output" % TEMPEST_PATH, + return_value="fake_url") + def test__get_remote_origin(self, mock_co): + self.assertEqual("fake_url", + self.verifier._get_remote_origin("fake_dir")) @mock.patch("shutil.rmtree") @mock.patch(TEMPEST_PATH + ".tempest.os.path.exists", return_value=True)