diff --git a/openstack-common.conf b/openstack-common.conf index 74f13100a..33f18d3a8 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,6 +1,7 @@ [DEFAULT] # The list of modules to copy from openstack-common +module=install_venv_common module=strutils module=timeutils module=uuidutils diff --git a/tools/install_venv.py b/tools/install_venv.py index 59d0b30a3..0011a8be1 100644 --- a/tools/install_venv.py +++ b/tools/install_venv.py @@ -5,242 +5,73 @@ # All Rights Reserved. # # Copyright 2010 OpenStack Foundation +# Copyright 2013 IBM Corp. +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. -""" -Installation script for Nova's development virtualenv -""" - -from __future__ import print_function -import optparse +import ConfigParser import os -import subprocess import sys - -ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -VENV = os.path.join(ROOT, '.venv') -PIP_REQUIRES = os.path.join(ROOT, 'requirements.txt') -TEST_REQUIRES = os.path.join(ROOT, 'test-requirements.txt') -PY_VERSION = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) +import install_venv_common as install_venv # flake8: noqa -def die(message, *args): - print(message % args, file=sys.stderr) - sys.exit(1) - - -def check_python_version(): - if sys.version_info < (2, 6): - die("Need Python Version >= 2.6") - - -def run_command_with_code(cmd, redirect_output=True, check_exit_code=True): - """ - Runs a command in an out-of-process shell, returning the - output of that command. Working directory is ROOT. - """ - if redirect_output: - stdout = subprocess.PIPE - else: - stdout = None - - proc = subprocess.Popen(cmd, cwd=ROOT, stdout=stdout) - output = proc.communicate()[0] - if check_exit_code and proc.returncode != 0: - die('Command "%s" failed.\n%s', ' '.join(cmd), output) - return (output, proc.returncode) - - -def run_command(cmd, redirect_output=True, check_exit_code=True): - return run_command_with_code(cmd, redirect_output, check_exit_code)[0] - - -class Distro(object): - - def check_cmd(self, cmd): - return bool(run_command(['which', cmd], check_exit_code=False).strip()) - - def install_virtualenv(self): - if self.check_cmd('virtualenv'): - return - - if self.check_cmd('easy_install'): - print('Installing virtualenv via easy_install...') - if run_command(['easy_install', 'virtualenv']): - print('Succeeded') - return - else: - print('Failed') - - die('ERROR: virtualenv not found.\n\nDevelopment' - ' requires virtualenv, please install it using your' - ' favorite package management tool') - - def post_process(self): - """Any distribution-specific post-processing gets done here. - - In particular, this is useful for applying patches to code inside - the venv.""" - pass - - -class Debian(Distro): - """This covers all Debian-based distributions.""" - - def check_pkg(self, pkg): - return run_command_with_code(['dpkg', '-l', pkg], - check_exit_code=False)[1] == 0 - - def apt_install(self, pkg, **kwargs): - run_command(['sudo', 'apt-get', 'install', '-y', pkg], **kwargs) - - def apply_patch(self, originalfile, patchfile): - run_command(['patch', originalfile, patchfile]) - - def install_virtualenv(self): - if self.check_cmd('virtualenv'): - return - - if not self.check_pkg('python-virtualenv'): - self.apt_install('python-virtualenv', check_exit_code=False) - - super(Debian, self).install_virtualenv() - - -class Fedora(Distro): - """This covers all Fedora-based distributions. - - Includes: Fedora, RHEL, CentOS, Scientific Linux""" - - def check_pkg(self, pkg): - return run_command_with_code(['rpm', '-q', pkg], - check_exit_code=False)[1] == 0 - - def yum_install(self, pkg, **kwargs): - run_command(['sudo', 'yum', 'install', '-y', pkg], **kwargs) - - def apply_patch(self, originalfile, patchfile): - run_command(['patch', originalfile, patchfile]) - - def install_virtualenv(self): - if self.check_cmd('virtualenv'): - return - - if not self.check_pkg('python-virtualenv'): - self.yum_install('python-virtualenv', check_exit_code=False) - - super(Fedora, self).install_virtualenv() - - -def get_distro(): - if os.path.exists('/etc/fedora-release') or \ - os.path.exists('/etc/redhat-release'): - return Fedora() - elif os.path.exists('/etc/debian_version'): - return Debian() - else: - return Distro() - - -def check_dependencies(): - get_distro().install_virtualenv() - - -def create_virtualenv(venv=VENV, no_site_packages=True): - """Creates the virtual environment and installs PIP only into the - virtual environment - """ - print('Creating venv...') - if no_site_packages: - run_command(['virtualenv', '-q', '--no-site-packages', VENV]) - else: - run_command(['virtualenv', '-q', VENV]) - print('done.') - print('Installing pip in virtualenv...') - if not run_command(['tools/with_venv.sh', 'easy_install', - 'pip>1.0']).strip(): - die("Failed to install pip.") - print('done.') - - -def pip_install(*args): - run_command(['tools/with_venv.sh', - 'pip', 'install', '--upgrade'] + list(args), - redirect_output=False) - - -def install_dependencies(venv=VENV): - print('Installing dependencies with pip (this can take a while)...') - - # First things first, make sure our venv has the latest pip and distribute. - pip_install('pip') - pip_install('distribute') - - pip_install('-r', PIP_REQUIRES) - pip_install('-r', TEST_REQUIRES) - - # Tell the virtual env how to "import nova" - pthfile = os.path.join(venv, "lib", PY_VERSION, "site-packages", - "novaclient.pth") - f = open(pthfile, 'w') - f.write("%s\n" % ROOT) - - -def post_process(): - get_distro().post_process() - - -def print_help(): +def print_help(project, venv, root): help = """ - python-novaclient development environment setup is complete. + %(project)s development environment setup is complete. - python-novaclient development uses virtualenv to track and manage Python + %(project)s development uses virtualenv to track and manage Python dependencies while in development and testing. - To activate the python-novaclient virtualenv for the extent of your current + To activate the %(project)s virtualenv for the extent of your current shell session you can run: - $ source .venv/bin/activate + $ source %(venv)s/bin/activate - Or, if you prefer, you can run commands in the virtualenv on a case by case - basis by running: + Or, if you prefer, you can run commands in the virtualenv on a case by + case basis by running: - $ tools/with_venv.sh - - Also, make test will automatically use the virtualenv. + $ %(root)s/tools/with_venv.sh """ - print(help) - - -def parse_args(): - """Parse command-line arguments""" - parser = optparse.OptionParser() - parser.add_option("-n", "--no-site-packages", dest="no_site_packages", - default=False, action="store_true", - help="Do not inherit packages from global Python install") - return parser.parse_args() + print help % dict(project=project, venv=venv, root=root) def main(argv): - (options, args) = parse_args() - check_python_version() - check_dependencies() - create_virtualenv(no_site_packages=options.no_site_packages) - install_dependencies() - post_process() - print_help() + root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + + if os.environ.get('tools_path'): + root = os.environ['tools_path'] + venv = os.path.join(root, '.venv') + if os.environ.get('venv'): + venv = os.environ['venv'] + + pip_requires = os.path.join(root, 'requirements.txt') + test_requires = os.path.join(root, 'test-requirements.txt') + py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) + setup_cfg = ConfigParser.ConfigParser() + setup_cfg.read('setup.cfg') + project = setup_cfg.get('metadata', 'name') + + install = install_venv.InstallVenv( + root, venv, pip_requires, test_requires, py_version, project) + options = install.parse_args(argv) + install.check_python_version() + install.check_dependencies() + install.create_virtualenv(no_site_packages=options.no_site_packages) + install.install_dependencies() + install.post_process() + print_help(project, venv, root) if __name__ == '__main__': main(sys.argv) diff --git a/tools/install_venv_common.py b/tools/install_venv_common.py new file mode 100644 index 000000000..f428c1e02 --- /dev/null +++ b/tools/install_venv_common.py @@ -0,0 +1,212 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 OpenStack Foundation +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides methods needed by installation script for OpenStack development +virtual environments. + +Since this script is used to bootstrap a virtualenv from the system's Python +environment, it should be kept strictly compatible with Python 2.6. + +Synced in from openstack-common +""" + +from __future__ import print_function + +import optparse +import os +import subprocess +import sys + + +class InstallVenv(object): + + def __init__(self, root, venv, requirements, + test_requirements, py_version, + project): + self.root = root + self.venv = venv + self.requirements = requirements + self.test_requirements = test_requirements + self.py_version = py_version + self.project = project + + def die(self, message, *args): + print(message % args, file=sys.stderr) + sys.exit(1) + + def check_python_version(self): + if sys.version_info < (2, 6): + self.die("Need Python Version >= 2.6") + + def run_command_with_code(self, cmd, redirect_output=True, + check_exit_code=True): + """Runs a command in an out-of-process shell. + + Returns the output of that command. Working directory is self.root. + """ + if redirect_output: + stdout = subprocess.PIPE + else: + stdout = None + + proc = subprocess.Popen(cmd, cwd=self.root, stdout=stdout) + output = proc.communicate()[0] + if check_exit_code and proc.returncode != 0: + self.die('Command "%s" failed.\n%s', ' '.join(cmd), output) + return (output, proc.returncode) + + def run_command(self, cmd, redirect_output=True, check_exit_code=True): + return self.run_command_with_code(cmd, redirect_output, + check_exit_code)[0] + + def get_distro(self): + if (os.path.exists('/etc/fedora-release') or + os.path.exists('/etc/redhat-release')): + return Fedora( + self.root, self.venv, self.requirements, + self.test_requirements, self.py_version, self.project) + else: + return Distro( + self.root, self.venv, self.requirements, + self.test_requirements, self.py_version, self.project) + + def check_dependencies(self): + self.get_distro().install_virtualenv() + + def create_virtualenv(self, no_site_packages=True): + """Creates the virtual environment and installs PIP. + + Creates the virtual environment and installs PIP only into the + virtual environment. + """ + if not os.path.isdir(self.venv): + print('Creating venv...', end=' ') + if no_site_packages: + self.run_command(['virtualenv', '-q', '--no-site-packages', + self.venv]) + else: + self.run_command(['virtualenv', '-q', self.venv]) + print('done.') + else: + print("venv already exists...") + pass + + def pip_install(self, *args): + self.run_command(['tools/with_venv.sh', + 'pip', 'install', '--upgrade'] + list(args), + redirect_output=False) + + def install_dependencies(self): + print('Installing dependencies with pip (this can take a while)...') + + # First things first, make sure our venv has the latest pip and + # setuptools. + self.pip_install('pip>=1.3') + self.pip_install('setuptools') + + self.pip_install('-r', self.requirements) + self.pip_install('-r', self.test_requirements) + + def post_process(self): + self.get_distro().post_process() + + def parse_args(self, argv): + """Parses command-line arguments.""" + parser = optparse.OptionParser() + parser.add_option('-n', '--no-site-packages', + action='store_true', + help="Do not inherit packages from global Python " + "install") + return parser.parse_args(argv[1:])[0] + + +class Distro(InstallVenv): + + def check_cmd(self, cmd): + return bool(self.run_command(['which', cmd], + check_exit_code=False).strip()) + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if self.check_cmd('easy_install'): + print('Installing virtualenv via easy_install...', end=' ') + if self.run_command(['easy_install', 'virtualenv']): + print('Succeeded') + return + else: + print('Failed') + + self.die('ERROR: virtualenv not found.\n\n%s development' + ' requires virtualenv, please install it using your' + ' favorite package management tool' % self.project) + + def post_process(self): + """Any distribution-specific post-processing gets done here. + + In particular, this is useful for applying patches to code inside + the venv. + """ + pass + + +class Fedora(Distro): + """This covers all Fedora-based distributions. + + Includes: Fedora, RHEL, CentOS, Scientific Linux + """ + + def check_pkg(self, pkg): + return self.run_command_with_code(['rpm', '-q', pkg], + check_exit_code=False)[1] == 0 + + def apply_patch(self, originalfile, patchfile): + self.run_command(['patch', '-N', originalfile, patchfile], + check_exit_code=False) + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if not self.check_pkg('python-virtualenv'): + self.die("Please install 'python-virtualenv'.") + + super(Fedora, self).install_virtualenv() + + def post_process(self): + """Workaround for a bug in eventlet. + + This currently affects RHEL6.1, but the fix can safely be + applied to all RHEL and Fedora distributions. + + This can be removed when the fix is applied upstream. + + Nova: https://bugs.launchpad.net/nova/+bug/884915 + Upstream: https://bitbucket.org/eventlet/eventlet/issue/89 + RHEL: https://bugzilla.redhat.com/958868 + """ + + # Install "patch" program if it's not there + if not self.check_pkg('patch'): + self.die("Please install 'patch'.") + + # Apply the eventlet patch + self.apply_patch(os.path.join(self.venv, 'lib', self.py_version, + 'site-packages', + 'eventlet/green/subprocess.py'), + 'contrib/redhat-eventlet.patch')