From 0c7535e4b9a10a21d131ba1a46c91183e624201d Mon Sep 17 00:00:00 2001 From: Endre Karlson Date: Thu, 23 May 2013 22:21:59 +0200 Subject: [PATCH] Helper utilities Change-Id: Ia689302fc1bcc3b11dceb51f131cdd4ac1ad30ba --- tools/install_venv.py | 76 ++++++++++++ tools/install_venv_common.py | 224 +++++++++++++++++++++++++++++++++++ tools/patch_tox_venv.py | 39 ++++++ tools/with_venv.sh | 21 ++++ 4 files changed, 360 insertions(+) create mode 100644 tools/install_venv.py create mode 100644 tools/install_venv_common.py create mode 100644 tools/patch_tox_venv.py create mode 100755 tools/with_venv.sh diff --git a/tools/install_venv.py b/tools/install_venv.py new file mode 100644 index 000000000..096a95b02 --- /dev/null +++ b/tools/install_venv.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Copyright 2010 OpenStack Foundation. +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Installation script for BillingStack's development virtualenv +""" + +import os +import subprocess +import sys + +import install_venv_common as install_venv + + +def print_help(): + help = """ + BillingStack development environment setup is complete. + + BillingStack development uses virtualenv to track and manage Python dependencies + while in development and testing. + + To activate the BillingStack virtualenv for the extent of your current shell + session you can run: + + $ source .venv/bin/activate + + Or, if you prefer, you can run commands in the virtualenv on a case by case + basis by running: + + $ tools/with_venv.sh + + Also, make test will automatically use the virtualenv. + """ + print help + + +def main(argv): + root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + venv = os.path.join(root, '.venv') + pip_requires = os.path.join(root, 'tools', 'pip-requires') + pip_options = os.path.join(root, 'tools', 'pip-options') + test_requires = os.path.join(root, 'tools', 'test-requires') + py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) + project = 'quantum' + install = install_venv.InstallVenv(root, venv, pip_requires, pip_options, test_requires, + py_version, project) + options = install.parse_args(argv) + install.check_python_version() + install.check_dependencies() + install.create_virtualenv(no_site_packages=options.no_site_packages) + install.install_dependencies() + install.post_process() + print_help() + + +if __name__ == '__main__': + main(sys.argv) diff --git a/tools/install_venv_common.py b/tools/install_venv_common.py new file mode 100644 index 000000000..8123f8915 --- /dev/null +++ b/tools/install_venv_common.py @@ -0,0 +1,224 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 OpenStack Foundation +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides methods needed by installation script for OpenStack development +virtual environments. + +Synced in from openstack-common +""" + +import argparse +import os +import subprocess +import sys + + +class InstallVenv(object): + + def __init__(self, root, venv, pip_requires, pip_options, test_requires, + py_version, project): + self.root = root + self.venv = venv + self.pip_requires = pip_requires + self.pip_options = pip_options + self.test_requires = test_requires + self.py_version = py_version + self.project = project + + def die(self, message, *args): + print >> sys.stderr, message % args + sys.exit(1) + + def check_python_version(self): + if sys.version_info < (2, 6): + self.die("Need Python Version >= 2.6") + + def run_command_with_code(self, cmd, redirect_output=True, + check_exit_code=True): + """Runs a command in an out-of-process shell. + + Returns the output of that command. Working directory is self.root. + """ + if redirect_output: + stdout = subprocess.PIPE + else: + stdout = None + + proc = subprocess.Popen(cmd, cwd=self.root, stdout=stdout) + output = proc.communicate()[0] + if check_exit_code and proc.returncode != 0: + self.die('Command "%s" failed.\n%s', ' '.join(cmd), output) + return (output, proc.returncode) + + def run_command(self, cmd, redirect_output=True, check_exit_code=True): + return self.run_command_with_code(cmd, redirect_output, + check_exit_code)[0] + + def get_distro(self): + if (os.path.exists('/etc/fedora-release') or + os.path.exists('/etc/redhat-release')): + return Fedora(self.root, self.venv, self.pip_requires, + self.pip_options, self.test_requires, + self.py_version, self.project) + else: + return Distro(self.root, self.venv, self.pip_requires, + self.pip_options, self.test_requires, + self.py_version, self.project) + + def check_dependencies(self): + self.get_distro().install_virtualenv() + + def create_virtualenv(self, no_site_packages=True): + """Creates the virtual environment and installs PIP. + + Creates the virtual environment and installs PIP only into the + virtual environment. + """ + if not os.path.isdir(self.venv): + print 'Creating venv...', + if no_site_packages: + self.run_command(['virtualenv', '-q', '--no-site-packages', + self.venv]) + else: + self.run_command(['virtualenv', '-q', self.venv]) + print 'done.' + print 'Installing pip in venv...', + if not self.run_command(['tools/with_venv.sh', 'easy_install', + 'pip>1.0']).strip(): + self.die("Failed to install pip.") + print 'done.' + else: + print "venv already exists..." + pass + + def pip_install(self, *args): + self.run_command(['tools/with_venv.sh', + 'pip', 'install', '--upgrade'] + list(args), + redirect_output=False) + + def install_dependencies(self): + print 'Installing dependencies with pip (this can take a while)...' + + # First things first, make sure our venv has the latest pip and + # distribute. + # NOTE: we keep pip at version 1.1 since the most recent version causes + # the .venv creation to fail. See: + # https://bugs.launchpad.net/nova/+bug/1047120 + self.pip_install('pip==1.1') + self.pip_install('distribute') + + # Install greenlet by hand - just listing it in the requires file does + # not + # get it installed in the right order + self.pip_install('greenlet') + + self.pip_install('-r', self.pip_requires) + self.pip_install('-r', self.pip_options) + self.pip_install('-r', self.test_requires) + + def post_process(self): + self.get_distro().post_process() + + def parse_args(self, argv): + """Parses command-line arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument('-n', '--no-site-packages', + action='store_true', + help="Do not inherit packages from global Python " + "install") + return parser.parse_args(argv[1:]) + + +class Distro(InstallVenv): + + def check_cmd(self, cmd): + return bool(self.run_command(['which', cmd], + check_exit_code=False).strip()) + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if self.check_cmd('easy_install'): + print 'Installing virtualenv via easy_install...', + if self.run_command(['easy_install', 'virtualenv']): + print 'Succeeded' + return + else: + print 'Failed' + + self.die('ERROR: virtualenv not found.\n\n%s development' + ' requires virtualenv, please install it using your' + ' favorite package management tool' % self.project) + + def post_process(self): + """Any distribution-specific post-processing gets done here. + + In particular, this is useful for applying patches to code inside + the venv. + """ + pass + + +class Fedora(Distro): + """This covers all Fedora-based distributions. + + Includes: Fedora, RHEL, CentOS, Scientific Linux + """ + + def check_pkg(self, pkg): + return self.run_command_with_code(['rpm', '-q', pkg], + check_exit_code=False)[1] == 0 + + def yum_install(self, pkg, **kwargs): + print "Attempting to install '%s' via yum" % pkg + self.run_command(['sudo', 'yum', 'install', '-y', pkg], **kwargs) + + def apply_patch(self, originalfile, patchfile): + self.run_command(['patch', '-N', originalfile, patchfile], + check_exit_code=False) + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if not self.check_pkg('python-virtualenv'): + self.yum_install('python-virtualenv', check_exit_code=False) + + super(Fedora, self).install_virtualenv() + + def post_process(self): + """Workaround for a bug in eventlet. + + This currently affects RHEL6.1, but the fix can safely be + applied to all RHEL and Fedora distributions. + + This can be removed when the fix is applied upstream. + + Nova: https://bugs.launchpad.net/nova/+bug/884915 + Upstream: https://bitbucket.org/which_linden/eventlet/issue/89 + """ + + # Install "patch" program if it's not there + if not self.check_pkg('patch'): + self.yum_install('patch') + + # Apply the eventlet patch + self.apply_patch(os.path.join(self.venv, 'lib', self.py_version, + 'site-packages', + 'eventlet/green/subprocess.py'), + 'contrib/redhat-eventlet.patch') diff --git a/tools/patch_tox_venv.py b/tools/patch_tox_venv.py new file mode 100644 index 000000000..7a8f8fbbe --- /dev/null +++ b/tools/patch_tox_venv.py @@ -0,0 +1,39 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys + +import install_venv_common as install_venv + + +def main(argv): + root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + + venv = os.environ['VIRTUAL_ENV'] + + pip_requires = os.path.join(root, 'tools', 'pip-requires') + pip_options = os.path.join(root, 'tools', 'pip-options') + test_requires = os.path.join(root, 'tools', 'test-requires') + py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) + project = 'Quantum' + install = install_venv.InstallVenv(root, venv, pip_requires, pip_options, + test_requires, py_version, project) + #NOTE(dprince): For Tox we only run post_process (which patches files, etc) + install.post_process() + +if __name__ == '__main__': + main(sys.argv) diff --git a/tools/with_venv.sh b/tools/with_venv.sh new file mode 100755 index 000000000..5c4a271e4 --- /dev/null +++ b/tools/with_venv.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +TOOLS=`dirname $0` +VENV=$TOOLS/../.venv +source $VENV/bin/activate && $@