openstack/rally
Code Issues Proposed changes

[Tempest]: Try to find py27 interpreter

Browse Source 
Since Rally team wants to support all Python 2.6, Python 2.7 and Python 3
envs and based on way of launching tempest(Rally install Tempest in virtual
environment), we can try to find and select a suitable python interpreter
for Tempest(Tempest designed only for Python 2.7). It can allow to support
`rally verify` stuff in all python environments.

Change-Id: If8c94c28119a34e60c00f7ef2cb9b94bc2e62004
This commit is contained in:
Andrey Kurilin 2015-02-24 01:27:43 +02:00
parent 16f8ab6f20
commit 11cdb55471
5 changed files with 224 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
rally/common/costilius.py
View File

@ -18,8 +18,14 @@
This module is a storage for different types of workarounds.
"""
from distutils import spawn
import os
import subprocess
import sys
from rally.common.i18n import _LE
from rally import exceptions
try:
from collections import OrderedDict # noqa
@ -49,3 +55,64 @@ def json_loads(*args, **kwargs):
"""
return json.loads(*args, **kwargs)
def sp_check_output(*popenargs, **kwargs):
"""Run command with arguments and return its output as a byte string.
If the exit code was non-zero it raises a CalledProcessError. The
CalledProcessError object will have the return code in the returncode
attribute and output in the output attribute.
The arguments are the same as for the Popen constructor.
"""
if is_py26():
# NOTE(andreykurilin): as I said before, support python 26 env is hard
# task. Subprocess supports check_output function from Python 2.7, so
# let's copy-paste code of this function from it.
if "stdout" in kwargs:
raise ValueError("stdout argument not allowed, "
"it will be overridden.")
process = subprocess.Popen(stdout=subprocess.PIPE,
*popenargs, **kwargs)
output, unused_err = process.communicate()
retcode = process.poll()
if retcode:
cmd = kwargs.get("args")
if cmd is None:
cmd = popenargs[0]
raise subprocess.CalledProcessError(retcode, cmd, output=output)
return output
return subprocess.check_output(*popenargs, **kwargs)
def get_interpreter(python_version):
"""Discovers PATH to find proper python interpreter
:param python_version: (major, minor) version numbers
:type python_version: tuple
"""
if not isinstance(python_version, tuple):
msg = (_LE("given format of python version `%s` is invalid") %
python_version)
raise exceptions.InvalidArgumentsException(msg)
interpreter_name = "python%s.%s" % python_version
interpreter = spawn.find_executable(interpreter_name)
if interpreter:
return interpreter
else:
interpreters = filter(
os.path.isfile, [os.path.join(p, interpreter_name)
for p in os.environ.get("PATH", "").split(":")])
cmd = "%s -c 'import sys; print(sys.version_info[:2])'"
for interpreter in interpreters:
try:
out = sp_check_output(cmd % interpreter, shell=True)
except subprocess.CalledProcessError:
pass
else:
if out.strip() == str(python_version):
return interpreter

2
rally/exceptions.py
View File

@ -250,7 +250,7 @@ class ImageCleanUpException(CleanUpException):
class IncompatiblePythonVersion(RallyException):
msg_fmt = _("Incompatible python version found '%(version)s', "
"required at least python>=2.7.x")
"required '%(required_version)s'")
class WorkerNotFound(NotFoundException):

47
rally/verification/tempest/tempest.py
View File

@ -20,7 +20,9 @@ import subprocess
import sys
from oslo_serialization import jsonutils
from oslo_utils import encodeutils
from rally.common import costilius
from rally.common.i18n import _
from rally.common import log as logging
from rally.common import utils
@ -41,14 +43,14 @@ class TempestSetupFailure(exceptions.RallyException):
def check_output(*args, **kwargs):
kwargs["stderr"] = subprocess.STDOUT
try:
output = subprocess.check_output(*args, **kwargs)
output = costilius.sp_check_output(*args, **kwargs)
except subprocess.CalledProcessError as e:
LOG.debug("failed cmd: '%s'" % e.cmd)
LOG.debug("error output: '%s'" % e.output)
LOG.debug("error output: '%s'" % encodeutils.safe_decode(e.output))
raise
if logging.is_debug():
print(output)
LOG.debug("subprocess output: '%s'" % encodeutils.safe_decode(output))
return output
class Tempest(object):
@ -118,7 +120,7 @@ class Tempest(object):
|_for-deployment-<UUID1> -> copy from relevant tempest base
|_for-deployment-<UUID2> -> copy from relevant tempest base
new:
new:
_ rally/tempest
|_base
||_ tempest_base-<rand suffix specific for source> -> clone
@ -157,21 +159,34 @@ class Tempest(object):
@staticmethod
def _get_remote_origin(directory):
out = subprocess.check_output("git config --get remote.origin.url",
shell=True,
cwd=os.path.abspath(directory))
out = check_output("git config --get remote.origin.url",
shell=True, cwd=os.path.abspath(directory))
return out.strip()
def _install_venv(self):
path_to_venv = self.path(".venv")
if not os.path.isdir(path_to_venv):
self.validate_env()
print("No virtual environment found...Install the virtualenv.")
LOG.debug("Virtual environment directory: %s" % path_to_venv)
required_vers = (2, 7)
if sys.version_info[:2] != required_vers:
# NOTE(andreykurilin): let's try to find a suitable python
# interpreter for Tempest
python_interpreter = costilius.get_interpreter(required_vers)
if not python_interpreter:
raise exceptions.IncompatiblePythonVersion(
version=sys.version, required_version=required_vers)
LOG.info(
_("Tempest requires Python %(required)s, '%(found)s' was "
"found in your system and it will be used for installing"
" virtual environment.") % {"required": required_vers,
"found": python_interpreter})
else:
python_interpreter = sys.executable
try:
check_output("python ./tools/install_venv.py", shell=True,
cwd=self.path())
check_output("%s ./tools/install_venv.py" % python_interpreter,
shell=True, cwd=self.path())
check_output("%s python setup.py install" % self.venv_wrapper,
shell=True, cwd=self.path())
except subprocess.CalledProcessError:
@ -342,16 +357,6 @@ class Tempest(object):
else:
self.verification.set_failed()
def validate_env(self):
"""Validate environment parameters required for running tempest.
eg: python>2.7
"""
if sys.version_info < (2, 7):
raise exceptions.IncompatiblePythonVersion(
version=sys.version_info)
def verify(self, set_name, regex):
self._prepare_and_run(set_name, regex)
self._save_results()

100
tests/unit/common/test_costilius.py Normal file
View File

@ -0,0 +1,100 @@
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import mock
from rally.common import costilius
from rally import exceptions
from tests.unit import test
PATH = "rally.common.costilius"
class SPCheckCallTestCase(test.TestCase):
@mock.patch("%s.subprocess" % PATH)
@mock.patch("%s.is_py26" % PATH, return_value=True)
def test_simulation_of_py26_env(self, mock_is_py26, mock_sp):
output = "output"
process = mock.MagicMock()
process.communicate.return_value = (output, "unused_err")
process.poll.return_value = None
mock_sp.Popen.return_value = process
some_args = (1, 2)
some_kwargs = {"a": 2}
self.assertEqual(output, costilius.sp_check_output(*some_args,
**some_kwargs))
mock_sp.Popen.assert_called_once_with(
stdout=mock_sp.PIPE, *some_args, **some_kwargs)
self.assertFalse(mock_sp.check_output.called)
@mock.patch("%s.subprocess" % PATH)
@mock.patch("%s.is_py26" % PATH, return_value=False)
def test_simulation_of_any_not_py26_env(self, mock_is_py26, mock_sp):
output = "output"
mock_sp.check_output.return_value = output
some_args = (1, 2)
some_kwargs = {"a": 2}
self.assertEqual(output, costilius.sp_check_output(*some_args,
**some_kwargs))
mock_sp.check_output.assert_called_once_with(*some_args, **some_kwargs)
self.assertFalse(mock_sp.Popen.called)
class GetInterpreterTestCase(test.TestCase):
def test_wrong_format(self):
self.assertRaises(exceptions.InvalidArgumentsException,
costilius.get_interpreter, "something_bad")
@mock.patch("%s.spawn" % PATH)
@mock.patch("%s.sp_check_output" % PATH)
@mock.patch("%s.os.path.isfile" % PATH)
@mock.patch("%s.os.environ" % PATH)
def test_found_correct_python_interpreter_with_distutils(
self, mock_env, mock_isfile, mock_check_output, mock_spawn):
vers = (2, 7)
interpreter = "something"
mock_spawn.find_executable.return_value = interpreter
self.assertEqual(interpreter, costilius.get_interpreter(vers))
self.assertFalse(mock_env.called)
self.assertFalse(mock_isfile.called)
self.assertFalse(mock_check_output.called)
@mock.patch("%s.spawn" % PATH)
@mock.patch("%s.sp_check_output" % PATH)
@mock.patch("%s.os.path.isfile" % PATH, return_value=True)
@mock.patch("%s.os.environ" % PATH)
def test_found_correct_python_interpreter_without_distutils(
self, mock_env, mock_isfile, mock_check_output, mock_spawn):
vers = (2, 7)
paths = ["one_path", "second_path"]
mock_env.get.return_value = ":".join(paths)
mock_check_output.return_value = "%s\n" % str(vers)
mock_spawn.find_executable.return_value = None
found_interpreter = costilius.get_interpreter(vers)
self.assertEqual(1, mock_check_output.call_count)
self.assertIn(
found_interpreter, ["%s/%s" % (f, "python2.7") for f in paths])

56
tests/unit/verification/test_tempest.py
View File

@ -16,11 +16,9 @@
import copy
import os
import subprocess
import sys
import mock
from oslo_serialization import jsonutils
import testtools
from rally import exceptions
from rally.verification.tempest import subunit2json
@ -88,33 +86,41 @@ class TempestUtilsTestCase(BaseTestCase):
self.assertFalse(mock_env.copy.called)
@mock.patch("os.path.isdir", return_value=True)
@mock.patch(TEMPEST_PATH + ".tempest.subprocess")
@testtools.skipIf(sys.version_info < (2, 7), "Incompatible Python Version")
def test__venv_install_when_venv_exists(self, mock_sp, mock_isdir):
@mock.patch(TEMPEST_PATH + ".tempest.check_output")
def test__venv_install_when_venv_exists(self, mock_co, mock_isdir):
self.verifier._install_venv()
mock_isdir.assert_called_once_with(self.verifier.path(".venv"))
self.assertFalse(mock_sp.check_output.called)
self.assertFalse(mock_co.called)
@mock.patch("%s.tempest.sys" % TEMPEST_PATH)
@mock.patch("%s.tempest.costilius.get_interpreter" % TEMPEST_PATH,
return_value="python")
@mock.patch("os.path.isdir", return_value=False)
@mock.patch("%s.tempest.subprocess.check_output" % TEMPEST_PATH,
@mock.patch("%s.tempest.check_output" % TEMPEST_PATH,
return_value="some_output")
@testtools.skipIf(sys.version_info < (2, 7), "Incompatible Python Version")
def test__venv_install_when_venv_not_exist(self, mock_sp, mock_isdir):
def test__venv_install_when_venv_not_exist(self, mock_co, mock_isdir,
mock_get_interpreter, mock_sys):
mock_sys.version_info = "not_py27_env"
self.verifier._install_venv()
mock_isdir.assert_called_once_with(self.verifier.path(".venv"))
mock_sp.assert_has_calls([
mock_co.assert_has_calls([
mock.call("python ./tools/install_venv.py", shell=True,
cwd=self.verifier.path(), stderr=subprocess.STDOUT),
cwd=self.verifier.path()),
mock.call("%s python setup.py install" %
self.verifier.venv_wrapper, shell=True,
cwd=self.verifier.path(), stderr=subprocess.STDOUT)])
cwd=self.verifier.path())])
@mock.patch("%s.tempest.sys" % TEMPEST_PATH)
@mock.patch("%s.tempest.costilius.get_interpreter" % TEMPEST_PATH,
return_value=None)
@mock.patch("os.path.isdir", return_value=False)
@testtools.skipIf(sys.version_info >= (2, 7),
"Incompatible Python Version")
def test__venv_install_for_py26_fails(self, mock_isdir):
def test__venv_install_fails__when_py27_is_not_present(
self, mock_isdir, mock_get_interpreter, mock_sys):
mock_sys.version_info = "not_py27_env"
self.assertRaises(exceptions.IncompatiblePythonVersion,
self.verifier._install_venv)
@ -130,18 +136,17 @@ class TempestUtilsTestCase(BaseTestCase):
self.verifier.path(".testrepository"))
self.assertFalse(mock_sp.called)
@testtools.skipIf(sys.version_info < (2, 7), "Incompatible Python Version")
@mock.patch("os.path.isdir", return_value=False)
@mock.patch(TEMPEST_PATH + ".tempest.subprocess.check_output")
@mock.patch(TEMPEST_PATH + ".tempest.check_output")
def test__initialize_testr_when_testr_not_initialized(
self, mock_sp, mock_isdir):
self, mock_co, mock_isdir):
self.verifier._initialize_testr()
mock_isdir.assert_called_once_with(
self.verifier.path(".testrepository"))
mock_sp.assert_called_once_with(
mock_co.assert_called_once_with(
"%s testr init" % self.verifier.venv_wrapper, shell=True,
cwd=self.verifier.path(), stderr=subprocess.STDOUT)
cwd=self.verifier.path())
@mock.patch.object(subunit2json, "main")
@mock.patch("os.path.isfile", return_value=False)
@ -191,12 +196,11 @@ class TempestInstallAndUninstallTestCase(BaseTestCase):
def test__is_git_repo(self, mock_isdir, mock_git_status):
self.assertTrue(self.verifier._is_git_repo("fake_dir"))
@testtools.skipIf(sys.version_info < (2, 7), "Incompatible Python Version")
@mock.patch("subprocess.check_output", return_value="fake_url")
def test__get_remote_origin(self, mock_sp):
with mock_sp:
self.assertEqual("fake_url",
self.verifier._get_remote_origin("fake_dir"))
@mock.patch("%s.tempest.check_output" % TEMPEST_PATH,
return_value="fake_url")
def test__get_remote_origin(self, mock_co):
self.assertEqual("fake_url",
self.verifier._get_remote_origin("fake_dir"))
@mock.patch("shutil.rmtree")
@mock.patch(TEMPEST_PATH + ".tempest.os.path.exists", return_value=True)