openstack/charm-cinder
Code Issues Proposed changes

Deploy from source

Browse Source 
Adds support to:
* Use a Python virtualenv when deploying from source
* Default to shallow clone for git (depth=1)
* Pass http_proxy to pip installs if specified
This commit is contained in:
James Page 2015-05-27 14:57:34 +01:00
commit 15f52309b9
6 changed files with 166 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
hooks/charmhelpers/contrib/openstack/utils.py
View File

@ -53,9 +53,13 @@ from charmhelpers.contrib.network.ip import (
get_ipv6_addr
)
from charmhelpers.contrib.python.packages import (
pip_create_virtualenv,
pip_install,
)
from charmhelpers.core.host import lsb_release, mounts, umount
from charmhelpers.fetch import apt_install, apt_cache, install_remote
from charmhelpers.contrib.python.packages import pip_install
from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
@ -497,7 +501,17 @@ def git_install_requested():
requirements_dir = None
def git_clone_and_install(projects_yaml, core_project):
def _git_yaml_load(projects_yaml):
"""
Load the specified yaml into a dictionary.
"""
if not projects_yaml:
return None
return yaml.load(projects_yaml)
def git_clone_and_install(projects_yaml, core_project, depth=1):
"""
Clone/install all specified OpenStack repositories.
@ -510,23 +524,22 @@ def git_clone_and_install(projects_yaml, core_project):
repository: 'git://git.openstack.org/openstack/requirements.git',
branch: 'stable/icehouse'}
directory: /mnt/openstack-git
http_proxy: http://squid.internal:3128
https_proxy: https://squid.internal:3128
http_proxy: squid-proxy-url
https_proxy: squid-proxy-url
The directory, http_proxy, and https_proxy keys are optional.
"""
global requirements_dir
parent_dir = '/mnt/openstack-git'
http_proxy = None
if not projects_yaml:
return
projects = yaml.load(projects_yaml)
projects = _git_yaml_load(projects_yaml)
_git_validate_projects_yaml(projects, core_project)
old_environ = dict(os.environ)
if 'http_proxy' in projects.keys():
http_proxy = projects['http_proxy']
os.environ['http_proxy'] = projects['http_proxy']
if 'https_proxy' in projects.keys():
os.environ['https_proxy'] = projects['https_proxy']
@ -534,15 +547,19 @@ def git_clone_and_install(projects_yaml, core_project):
if 'directory' in projects.keys():
parent_dir = projects['directory']
pip_create_virtualenv(os.path.join(parent_dir, 'venv'))
for p in projects['repositories']:
repo = p['repository']
branch = p['branch']
if p['name'] == 'requirements':
repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
repo_dir = _git_clone_and_install_single(repo, branch, depth,
parent_dir, http_proxy,
update_requirements=False)
requirements_dir = repo_dir
else:
repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
repo_dir = _git_clone_and_install_single(repo, branch, depth,
parent_dir, http_proxy,
update_requirements=True)
os.environ = old_environ
@ -574,7 +591,8 @@ def _git_ensure_key_exists(key, keys):
error_out('openstack-origin-git key \'{}\' is missing'.format(key))
def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements):
def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
update_requirements):
"""
Clone and install a single git repository.
"""
@ -587,7 +605,8 @@ def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements)
if not os.path.exists(dest_dir):
juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
repo_dir = install_remote(repo, dest=parent_dir, branch=branch)
repo_dir = install_remote(repo, dest=parent_dir, branch=branch,
depth=depth)
else:
repo_dir = dest_dir
@ -598,7 +617,12 @@ def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements)
_git_update_requirements(repo_dir, requirements_dir)
juju_log('Installing git repo from dir: {}'.format(repo_dir))
pip_install(repo_dir)
if http_proxy:
pip_install(repo_dir, proxy=http_proxy,
venv=os.path.join(parent_dir, 'venv'))
else:
pip_install(repo_dir,
venv=os.path.join(parent_dir, 'venv'))
return repo_dir
@ -621,16 +645,27 @@ def _git_update_requirements(package_dir, reqs_dir):
os.chdir(orig_dir)
def git_pip_venv_dir(projects_yaml):
"""
Return the pip virtualenv path.
"""
parent_dir = '/mnt/openstack-git'
projects = _git_yaml_load(projects_yaml)
if 'directory' in projects.keys():
parent_dir = projects['directory']
return os.path.join(parent_dir, 'venv')
def git_src_dir(projects_yaml, project):
"""
Return the directory where the specified project's source is located.
"""
parent_dir = '/mnt/openstack-git'
if not projects_yaml:
return
projects = yaml.load(projects_yaml)
projects = _git_yaml_load(projects_yaml)
if 'directory' in projects.keys():
parent_dir = projects['directory']
@ -640,3 +675,15 @@ def git_src_dir(projects_yaml, project):
return os.path.join(parent_dir, os.path.basename(p['repository']))
return None
def git_yaml_value(projects_yaml, key):
"""
Return the value in projects_yaml for the specified key.
"""
projects = _git_yaml_load(projects_yaml)
if key in projects.keys():
return projects[key]
return None

29
hooks/charmhelpers/contrib/python/packages.py
View File

@ -17,8 +17,11 @@
# You should have received a copy of the GNU Lesser General Public License
# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
import os
import subprocess
from charmhelpers.fetch import apt_install, apt_update
from charmhelpers.core.hookenv import log
from charmhelpers.core.hookenv import charm_dir, log
try:
from pip import main as pip_execute
@ -51,11 +54,15 @@ def pip_install_requirements(requirements, **options):
pip_execute(command)
def pip_install(package, fatal=False, upgrade=False, **options):
def pip_install(package, fatal=False, upgrade=False, venv=None, **options):
"""Install a python package"""
if venv:
venv_python = os.path.join(venv, 'bin/pip')
command = [venv_python, "install"]
else:
command = ["install"]
available_options = ('proxy', 'src', 'log', "index-url", )
available_options = ('proxy', 'src', 'log', 'index-url', )
for option in parse_options(options, available_options):
command.append(option)
@ -69,6 +76,9 @@ def pip_install(package, fatal=False, upgrade=False, **options):
log("Installing {} package with options: {}".format(package,
command))
if venv:
subprocess.check_call(command)
else:
pip_execute(command)
@ -94,3 +104,16 @@ def pip_list():
"""Returns the list of current python installed packages
"""
return pip_execute(["list"])
def pip_create_virtualenv(path=None):
"""Create an isolated Python environment."""
apt_install('python-virtualenv')
if path:
venv_path = path
else:
venv_path = os.path.join(charm_dir(), 'venv')
if not os.path.exists(venv_path):
subprocess.check_call(['virtualenv', venv_path])

12
hooks/charmhelpers/fetch/giturl.py
View File

@ -45,14 +45,16 @@ class GitUrlFetchHandler(BaseFetchHandler):
else:
return True
def clone(self, source, dest, branch):
def clone(self, source, dest, branch, depth=None):
if not self.can_handle(source):
raise UnhandledSource("Cannot handle {}".format(source))
repo = Repo.clone_from(source, dest)
repo.git.checkout(branch)
if depth:
Repo.clone_from(source, dest, branch=branch, depth=depth)
else:
Repo.clone_from(source, dest, branch=branch)
def install(self, source, branch="master", dest=None):
def install(self, source, branch="master", dest=None, depth=None):
url_parts = self.parse_url(source)
branch_name = url_parts.path.strip("/").split("/")[-1]
if dest:
@ -63,7 +65,7 @@ class GitUrlFetchHandler(BaseFetchHandler):
if not os.path.exists(dest_dir):
mkdir(dest_dir, perms=0o755)
try:
self.clone(source, dest_dir, branch)
self.clone(source, dest_dir, branch, depth)
except GitCommandError as e:
raise UnhandledSource(e.message)
except OSError as e:

52
hooks/cinder_utils.py
View File

@ -5,6 +5,10 @@ import subprocess
from collections import OrderedDict
from copy import copy
from charmhelpers.contrib.python.packages import (
pip_install,
)
from charmhelpers.core.hookenv import (
charm_dir,
config,
@ -69,6 +73,8 @@ from charmhelpers.contrib.openstack.utils import (
git_install_requested,
git_clone_and_install,
git_src_dir,
git_yaml_value,
git_pip_venv_dir,
os_release,
)
@ -94,8 +100,12 @@ VOLUME_PACKAGES = ['cinder-volume']
SCHEDULER_PACKAGES = ['cinder-scheduler']
BASE_GIT_PACKAGES = [
'libffi-dev',
'libmysqlclient-dev',
'libssl-dev',
'libxml2-dev',
'libxslt1-dev',
'libyaml-dev',
'lvm2',
'python-dev',
'python-pip',
@ -564,6 +574,16 @@ def git_pre_install():
def git_post_install(projects_yaml):
"""Perform cinder post-install setup."""
http_proxy = git_yaml_value(projects_yaml, 'http_proxy')
base_packages = ['mysql-python']
for pkg in base_packages:
if http_proxy:
pip_install(pkg, proxy=http_proxy,
venv=git_pip_venv_dir(projects_yaml))
else:
pip_install(pkg,
venv=git_pip_venv_dir(projects_yaml))
src_etc = os.path.join(git_src_dir(projects_yaml, 'cinder'), 'etc/cinder')
configs = {
'src': src_etc,
@ -574,6 +594,29 @@ def git_post_install(projects_yaml):
shutil.rmtree(configs['dest'])
shutil.copytree(configs['src'], configs['dest'])
# NOTE(coreycb): Need to find better solution than bin symlinks.
symlinks = [
{'src': os.path.join(git_pip_venv_dir(projects_yaml),
'bin/cinder-manage'),
'link': '/usr/local/bin/cinder-manage'},
{'src': os.path.join(git_pip_venv_dir(projects_yaml),
'bin/cinder-rootwrap'),
'link': '/usr/local/bin/cinder-rootwrap'},
# NOTE(coreycb): This is ugly but couldn't find pypi package that
# installs rbd.py and rados.py.
{'src': '/usr/lib/python2.7/dist-packages/rbd.py',
'link': os.path.join(git_pip_venv_dir(projects_yaml),
'lib/python2.7/site-packages/rbd.py')},
{'src': '/usr/lib/python2.7/dist-packages/rados.py',
'link': os.path.join(git_pip_venv_dir(projects_yaml),
'lib/python2.7/site-packages/rados.py')},
]
for s in symlinks:
if os.path.lexists(s['link']):
os.remove(s['link'])
os.symlink(s['src'], s['link'])
render('cinder.conf', '/etc/cinder/cinder.conf', {}, owner='cinder',
group='cinder', perms=0o644)
render('git/cinder_tgt.conf', '/etc/tgt/conf.d', {}, owner='cinder',
@ -585,13 +628,14 @@ def git_post_install(projects_yaml):
os.chmod('/etc/sudoers.d', 0o750)
bin_dir = os.path.join(git_pip_venv_dir(projects_yaml), 'bin')
cinder_api_context = {
'service_description': 'Cinder API server',
'service_name': 'Cinder',
'user_name': 'cinder',
'start_dir': '/var/lib/cinder',
'process_name': 'cinder-api',
'executable_name': '/usr/local/bin/cinder-api',
'executable_name': os.path.join(bin_dir, 'cinder-api'),
'config_files': ['/etc/cinder/cinder.conf'],
'log_file': '/var/log/cinder/cinder-api.log',
}
@ -602,7 +646,7 @@ def git_post_install(projects_yaml):
'user_name': 'cinder',
'start_dir': '/var/lib/cinder',
'process_name': 'cinder-backup',
'executable_name': '/usr/local/bin/cinder-backup',
'executable_name': os.path.join(bin_dir, 'cinder-backup'),
'config_files': ['/etc/cinder/cinder.conf'],
'log_file': '/var/log/cinder/cinder-backup.log',
}
@ -613,7 +657,7 @@ def git_post_install(projects_yaml):
'user_name': 'cinder',
'start_dir': '/var/lib/cinder',
'process_name': 'cinder-scheduler',
'executable_name': '/usr/local/bin/cinder-scheduler',
'executable_name': os.path.join(bin_dir, 'cinder-scheduler'),
'config_files': ['/etc/cinder/cinder.conf'],
'log_file': '/var/log/cinder/cinder-scheduler.log',
}
@ -624,7 +668,7 @@ def git_post_install(projects_yaml):
'user_name': 'cinder',
'start_dir': '/var/lib/cinder',
'process_name': 'cinder-volume',
'executable_name': '/usr/local/bin/cinder-volume',
'executable_name': os.path.join(bin_dir, 'cinder-volume'),
'config_files': ['/etc/cinder/cinder.conf'],
'log_file': '/var/log/cinder/cinder-volume.log',
}

5
tests/basic_deployment.py
View File

@ -80,11 +80,10 @@ class CinderBasicDeployment(OpenStackAmuletDeployment):
openstack_origin_git = {
'repositories': [
{'name': 'requirements',
'repository':
'git://git.openstack.org/openstack/requirements',
'repository': 'git://github.com/openstack/requirements',
'branch': branch},
{'name': 'cinder',
'repository': 'git://git.openstack.org/openstack/cinder',
'repository': 'git://github.com/openstack/cinder',
'branch': branch},
],
'directory': '/mnt/openstack-git',

24
unit_tests/test_cinder_utils.py
View File

@ -572,6 +572,7 @@ class TestCinderUtils(CharmTestCase):
@patch.object(cinder_utils, 'git_src_dir')
@patch.object(cinder_utils, 'service_restart')
@patch.object(cinder_utils, 'render')
@patch.object(cinder_utils, 'pip_install')
@patch('os.path.join')
@patch('os.path.exists')
@patch('shutil.copytree')
@ -580,24 +581,33 @@ class TestCinderUtils(CharmTestCase):
@patch('grp.getgrnam')
@patch('os.chown')
@patch('os.chmod')
def test_git_post_install(self, chmod, chown, grp, pwd, rmtree, copytree,
exists, join, render, service_restart,
git_src_dir):
@patch('os.symlink')
def test_git_post_install(self, symlink, chmod, chown, grp, pwd, rmtree,
copytree, exists, join, pip_install, render,
service_restart, git_src_dir):
projects_yaml = openstack_origin_git
join.return_value = 'joined-string'
cinder_utils.git_post_install(projects_yaml)
pip_install('mysql-python', venv='joined-string')
expected = [
call('joined-string', '/etc/cinder'),
]
copytree.assert_has_calls(expected)
expected = [
call('joined-string', '/usr/local/bin/cinder-manage'),
call('/usr/lib/python2.7/dist-packages/rbd.py', 'joined-string'),
call('/usr/lib/python2.7/dist-packages/rados.py', 'joined-string'),
]
symlink.assert_has_calls(expected, any_order=True)
cinder_api_context = {
'service_description': 'Cinder API server',
'service_name': 'Cinder',
'user_name': 'cinder',
'start_dir': '/var/lib/cinder',
'process_name': 'cinder-api',
'executable_name': '/usr/local/bin/cinder-api',
'executable_name': 'joined-string',
'config_files': ['/etc/cinder/cinder.conf'],
'log_file': '/var/log/cinder/cinder-api.log',
}
@ -608,7 +618,7 @@ class TestCinderUtils(CharmTestCase):
'user_name': 'cinder',
'start_dir': '/var/lib/cinder',
'process_name': 'cinder-backup',
'executable_name': '/usr/local/bin/cinder-backup',
'executable_name': 'joined-string',
'config_files': ['/etc/cinder/cinder.conf'],
'log_file': '/var/log/cinder/cinder-backup.log',
}
@ -619,7 +629,7 @@ class TestCinderUtils(CharmTestCase):
'user_name': 'cinder',
'start_dir': '/var/lib/cinder',
'process_name': 'cinder-scheduler',
'executable_name': '/usr/local/bin/cinder-scheduler',
'executable_name': 'joined-string',
'config_files': ['/etc/cinder/cinder.conf'],
'log_file': '/var/log/cinder/cinder-scheduler.log',
}
@ -630,7 +640,7 @@ class TestCinderUtils(CharmTestCase):
'user_name': 'cinder',
'start_dir': '/var/lib/cinder',
'process_name': 'cinder-volume',
'executable_name': '/usr/local/bin/cinder-volume',
'executable_name': 'joined-string',
'config_files': ['/etc/cinder/cinder.conf'],
'log_file': '/var/log/cinder/cinder-volume.log',
}