openstack/nova
Code Issues Proposed changes

Use install_venv_common.py from oslo.

Browse Source 
This syncs install_venv_common.py from oslo and reworks the
tools/install_venv.py script to use the new library.

Change-Id: I020eae1ec9e3a6aae0ecc7825c0ef723ab247f39
This commit is contained in:
Matthew Treinish 2013-01-23 13:27:00 -05:00
parent a8a0826ea6
commit c69ce047ad
3 changed files with 242 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
openstack-common.conf
View File

@ -1,7 +1,7 @@
[DEFAULT] [DEFAULT]
# The list of modules to copy from openstack-common # The list of modules to copy from openstack-common
modules=cfg,cliutils,context,excutils,eventlet_backdoor,fileutils,gettextutils,importutils,iniparser,jsonutils,local,lockutils,log,network_utils,notifier,plugin,policy,rootwrap,setup,timeutils,rpc,uuidutils modules=cfg,cliutils,context,excutils,eventlet_backdoor,fileutils,gettextutils,importutils,iniparser,jsonutils,local,lockutils,log,network_utils,notifier,plugin,policy,rootwrap,setup,timeutils,rpc,uuidutils,install_venv_common
# The base module to hold the copy of openstack.common # The base module to hold the copy of openstack.common
base=nova base=nova

209
tools/install_venv.py
View File

@ -1,4 +1,3 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4 # vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010 United States Government as represented by the # Copyright 2010 United States Government as represented by the
@ -6,6 +5,7 @@
# All Rights Reserved. # All Rights Reserved.
# #
# Copyright 2010 OpenStack, LLC # Copyright 2010 OpenStack, LLC
# Copyright 2013 IBM Corp.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may # Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain # not use this file except in compliance with the License. You may obtain
@ -19,186 +19,10 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
"""Installation script for Nova's development virtualenv."""
import optparse
import os import os
import subprocess
import sys import sys
import install_venv_common as install_venv
ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
VENV = os.path.join(ROOT, '.venv')
PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires')
TEST_REQUIRES = os.path.join(ROOT, 'tools', 'test-requires')
PY_VERSION = "python%s.%s" % (sys.version_info[0], sys.version_info[1])
def die(message, *args):
print >> sys.stderr, message % args
sys.exit(1)
def check_python_version():
if sys.version_info < (2, 6):
die("Need Python Version >= 2.6")
def run_command_with_code(cmd, redirect_output=True, check_exit_code=True):
"""Runs a command in an out-of-process shell.
Returns the output of that command. Working directory is ROOT.
"""
if redirect_output:
stdout = subprocess.PIPE
else:
stdout = None
proc = subprocess.Popen(cmd, cwd=ROOT, stdout=stdout)
output = proc.communicate()[0]
if check_exit_code and proc.returncode != 0:
die('Command "%s" failed.\n%s', ' '.join(cmd), output)
return (output, proc.returncode)
def run_command(cmd, redirect_output=True, check_exit_code=True):
return run_command_with_code(cmd, redirect_output, check_exit_code)[0]
class Distro(object):
def check_cmd(self, cmd):
return bool(run_command(['which', cmd], check_exit_code=False).strip())
def install_virtualenv(self):
if self.check_cmd('virtualenv'):
return
if self.check_cmd('easy_install'):
print 'Installing virtualenv via easy_install...',
if run_command(['easy_install', 'virtualenv']):
print 'Succeeded'
return
else:
print 'Failed'
die('ERROR: virtualenv not found.\n\nNova development'
' requires virtualenv, please install it using your'
' favorite package management tool')
def post_process(self):
"""Any distribution-specific post-processing gets done here.
In particular, this is useful for applying patches to code inside
the venv.
"""
pass
class Fedora(Distro):
"""This covers all Fedora-based distributions.
Includes: Fedora, RHEL, CentOS, Scientific Linux"""
def check_pkg(self, pkg):
return run_command_with_code(['rpm', '-q', pkg],
check_exit_code=False)[1] == 0
def yum_install(self, pkg, **kwargs):
print "Attempting to install '%s' via yum" % pkg
run_command(['sudo', 'yum', 'install', '-y', pkg], **kwargs)
def apply_patch(self, originalfile, patchfile):
run_command(['patch', originalfile, patchfile])
def install_virtualenv(self):
if self.check_cmd('virtualenv'):
return
if not self.check_pkg('python-virtualenv'):
self.yum_install('python-virtualenv', check_exit_code=False)
super(Fedora, self).install_virtualenv()
def post_process(self):
"""Workaround for a bug in eventlet.
This currently affects RHEL6.1, but the fix can safely be
applied to all RHEL and Fedora distributions.
This can be removed when the fix is applied upstream.
Nova: https://bugs.launchpad.net/nova/+bug/884915
Upstream: https://bitbucket.org/which_linden/eventlet/issue/89
"""
# Install "patch" program if it's not there
if not self.check_pkg('patch'):
self.yum_install('patch')
# Apply the eventlet patch
self.apply_patch(os.path.join(VENV, 'lib', PY_VERSION, 'site-packages',
'eventlet/green/subprocess.py'),
'contrib/redhat-eventlet.patch')
def get_distro():
if (os.path.exists('/etc/fedora-release') or
os.path.exists('/etc/redhat-release')):
return Fedora()
else:
return Distro()
def check_dependencies():
get_distro().install_virtualenv()
def create_virtualenv(venv=VENV, no_site_packages=True):
"""Creates the virtual environment and installs PIP.
Creates the virtual environment and installs PIP only into the
virtual environment.
"""
print 'Creating venv...',
if no_site_packages:
run_command(['virtualenv', '-q', '--no-site-packages', VENV])
else:
run_command(['virtualenv', '-q', VENV])
print 'done.'
print 'Installing pip in virtualenv...',
if not run_command(['tools/with_venv.sh', 'easy_install',
'pip>1.0']).strip():
die("Failed to install pip.")
print 'done.'
def pip_install(*args):
run_command(['tools/with_venv.sh',
'pip', 'install', '--upgrade'] + list(args),
redirect_output=False)
def install_dependencies(venv=VENV):
print 'Installing dependencies with pip (this can take a while)...'
# First things first, make sure our venv has the latest pip and distribute.
# NOTE: we keep pip at version 1.1 since the most recent version causes
# the .venv creation to fail. See:
# https://bugs.launchpad.net/nova/+bug/1047120
pip_install('pip==1.1')
pip_install('distribute')
# Install greenlet by hand - just listing it in the requires file does not
# get it in stalled in the right order
pip_install('greenlet')
pip_install('-r', PIP_REQUIRES)
pip_install('-r', TEST_REQUIRES)
def post_process():
get_distro().post_process()
def print_help(): def print_help():
@ -223,22 +47,21 @@ def print_help():
print help print help
def parse_args():
"""Parses command-line arguments."""
parser = optparse.OptionParser()
parser.add_option("-n", "--no-site-packages", dest="no_site_packages",
default=False, action="store_true",
help="Do not inherit packages from global Python install")
return parser.parse_args()
def main(argv): def main(argv):
(options, args) = parse_args() root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
check_python_version() venv = os.path.join(root, '.venv')
check_dependencies() pip_requires = os.path.join(root, 'tools', 'pip-requires')
create_virtualenv(no_site_packages=options.no_site_packages) test_requires = os.path.join(root, 'tools', 'test-requires')
install_dependencies() py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1])
post_process() project = 'Nova'
install = install_venv.InstallVenv(root, venv, pip_requires, test_requires,
py_version, project)
options = install.parse_args(argv)
install.check_python_version()
install.check_dependencies()
install.create_virtualenv(no_site_packages=options.no_site_packages)
install.install_dependencies()
install.post_process()
print_help() print_help()
if __name__ == '__main__': if __name__ == '__main__':

225
tools/install_venv_common.py Normal file
View File

@ -0,0 +1,225 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 OpenStack, LLC
# Copyright 2013 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Provides methods needed by installation script for OpenStack development
virtual environments.
Synced in from openstack-common
"""
import os
import subprocess
import sys
from nova.openstack.common import cfg
class InstallVenv(object):
def __init__(self, root, venv, pip_requires, test_requires, py_version,
project):
self.root = root
self.venv = venv
self.pip_requires = pip_requires
self.test_requires = test_requires
self.py_version = py_version
self.project = project
def die(self, message, *args):
print >> sys.stderr, message % args
sys.exit(1)
def check_python_version(self):
if sys.version_info < (2, 6):
self.die("Need Python Version >= 2.6")
def run_command_with_code(self, cmd, redirect_output=True,
check_exit_code=True):
"""Runs a command in an out-of-process shell.
Returns the output of that command. Working directory is ROOT.
"""
if redirect_output:
stdout = subprocess.PIPE
else:
stdout = None
proc = subprocess.Popen(cmd, cwd=self.root, stdout=stdout)
output = proc.communicate()[0]
if check_exit_code and proc.returncode != 0:
self.die('Command "%s" failed.\n%s', ' '.join(cmd), output)
return (output, proc.returncode)
def run_command(self, cmd, redirect_output=True, check_exit_code=True):
return self.run_command_with_code(cmd, redirect_output,
check_exit_code)[0]
def get_distro(self):
if (os.path.exists('/etc/fedora-release') or
os.path.exists('/etc/redhat-release')):
return Fedora(self.root, self.venv, self.pip_requires,
self.test_requires, self.py_version, self.project)
else:
return Distro(self.root, self.venv, self.pip_requires,
self.test_requires, self.py_version, self.project)
def check_dependencies(self):
self.get_distro().install_virtualenv()
def create_virtualenv(self, no_site_packages=True):
"""Creates the virtual environment and installs PIP.
Creates the virtual environment and installs PIP only into the
virtual environment.
"""
if not os.path.isdir(self.venv):
print 'Creating venv...',
if no_site_packages:
self.run_command(['virtualenv', '-q', '--no-site-packages',
self.venv])
else:
self.run_command(['virtualenv', '-q', self.venv])
print 'done.'
print 'Installing pip in virtualenv...',
if not self.run_command(['tools/with_venv.sh', 'easy_install',
'pip>1.0']).strip():
self.die("Failed to install pip.")
print 'done.'
else:
print "venv already exists..."
pass
def pip_install(self, *args):
self.run_command(['tools/with_venv.sh',
'pip', 'install', '--upgrade'] + list(args),
redirect_output=False)
def install_dependencies(self):
print 'Installing dependencies with pip (this can take a while)...'
# First things first, make sure our venv has the latest pip and
# distribute.
# NOTE: we keep pip at version 1.1 since the most recent version causes
# the .venv creation to fail. See:
# https://bugs.launchpad.net/nova/+bug/1047120
self.pip_install('pip==1.1')
self.pip_install('distribute')
# Install greenlet by hand - just listing it in the requires file does
# not
# get it installed in the right order
self.pip_install('greenlet')
self.pip_install('-r', self.pip_requires)
self.pip_install('-r', self.test_requires)
def post_process(self):
self.get_distro().post_process()
def parse_args(self, argv):
"""Parses command-line arguments."""
cli_opts = [
cfg.BoolOpt('no-site-packages',
default=False,
short='n',
help="Do not inherit packages from global Python"
"install"),
]
CLI = cfg.ConfigOpts()
CLI.register_cli_opts(cli_opts)
CLI(argv[1:])
return CLI
class Distro(InstallVenv):
def check_cmd(self, cmd):
return bool(self.run_command(['which', cmd],
check_exit_code=False).strip())
def install_virtualenv(self):
if self.check_cmd('virtualenv'):
return
if self.check_cmd('easy_install'):
print 'Installing virtualenv via easy_install...',
if self.run_command(['easy_install', 'virtualenv']):
print 'Succeeded'
return
else:
print 'Failed'
self.die('ERROR: virtualenv not found.\n\n%s development'
' requires virtualenv, please install it using your'
' favorite package management tool' % self.project)
def post_process(self):
"""Any distribution-specific post-processing gets done here.
In particular, this is useful for applying patches to code inside
the venv.
"""
pass
class Fedora(Distro):
"""This covers all Fedora-based distributions.
Includes: Fedora, RHEL, CentOS, Scientific Linux
"""
def check_pkg(self, pkg):
return self.run_command_with_code(['rpm', '-q', pkg],
check_exit_code=False)[1] == 0
def yum_install(self, pkg, **kwargs):
print "Attempting to install '%s' via yum" % pkg
self.run_command(['sudo', 'yum', 'install', '-y', pkg], **kwargs)
def apply_patch(self, originalfile, patchfile):
self.run_command(['patch', originalfile, patchfile])
def install_virtualenv(self):
if self.check_cmd('virtualenv'):
return
if not self.check_pkg('python-virtualenv'):
self.yum_install('python-virtualenv', check_exit_code=False)
super(Fedora, self).install_virtualenv()
def post_process(self):
"""Workaround for a bug in eventlet.
This currently affects RHEL6.1, but the fix can safely be
applied to all RHEL and Fedora distributions.
This can be removed when the fix is applied upstream.
Nova: https://bugs.launchpad.net/nova/+bug/884915
Upstream: https://bitbucket.org/which_linden/eventlet/issue/89
"""
# Install "patch" program if it's not there
if not self.check_pkg('patch'):
self.yum_install('patch')
# Apply the eventlet patch
self.apply_patch(os.path.join(self.venv, 'lib', self.py_version,
'site-packages',
'eventlet/green/subprocess.py'),
'contrib/redhat-eventlet.patch')