diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index f90a0289..d795a358 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -53,9 +53,13 @@ from charmhelpers.contrib.network.ip import ( get_ipv6_addr ) +from charmhelpers.contrib.python.packages import ( + pip_create_virtualenv, + pip_install, +) + from charmhelpers.core.host import lsb_release, mounts, umount from charmhelpers.fetch import apt_install, apt_cache, install_remote -from charmhelpers.contrib.python.packages import pip_install from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device @@ -497,7 +501,17 @@ def git_install_requested(): requirements_dir = None -def git_clone_and_install(projects_yaml, core_project): +def _git_yaml_load(projects_yaml): + """ + Load the specified yaml into a dictionary. + """ + if not projects_yaml: + return None + + return yaml.load(projects_yaml) + + +def git_clone_and_install(projects_yaml, core_project, depth=1): """ Clone/install all specified OpenStack repositories. @@ -510,23 +524,22 @@ def git_clone_and_install(projects_yaml, core_project): repository: 'git://git.openstack.org/openstack/requirements.git', branch: 'stable/icehouse'} directory: /mnt/openstack-git - http_proxy: http://squid.internal:3128 - https_proxy: https://squid.internal:3128 + http_proxy: squid-proxy-url + https_proxy: squid-proxy-url The directory, http_proxy, and https_proxy keys are optional. """ global requirements_dir parent_dir = '/mnt/openstack-git' + http_proxy = None - if not projects_yaml: - return - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) _git_validate_projects_yaml(projects, core_project) old_environ = dict(os.environ) if 'http_proxy' in projects.keys(): + http_proxy = projects['http_proxy'] os.environ['http_proxy'] = projects['http_proxy'] if 'https_proxy' in projects.keys(): os.environ['https_proxy'] = projects['https_proxy'] @@ -534,15 +547,19 @@ def git_clone_and_install(projects_yaml, core_project): if 'directory' in projects.keys(): parent_dir = projects['directory'] + pip_create_virtualenv(os.path.join(parent_dir, 'venv')) + for p in projects['repositories']: repo = p['repository'] branch = p['branch'] if p['name'] == 'requirements': - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + repo_dir = _git_clone_and_install_single(repo, branch, depth, + parent_dir, http_proxy, update_requirements=False) requirements_dir = repo_dir else: - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + repo_dir = _git_clone_and_install_single(repo, branch, depth, + parent_dir, http_proxy, update_requirements=True) os.environ = old_environ @@ -574,7 +591,8 @@ def _git_ensure_key_exists(key, keys): error_out('openstack-origin-git key \'{}\' is missing'.format(key)) -def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements): +def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, + update_requirements): """ Clone and install a single git repository. """ @@ -587,7 +605,8 @@ def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements) if not os.path.exists(dest_dir): juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) - repo_dir = install_remote(repo, dest=parent_dir, branch=branch) + repo_dir = install_remote(repo, dest=parent_dir, branch=branch, + depth=depth) else: repo_dir = dest_dir @@ -598,7 +617,12 @@ def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements) _git_update_requirements(repo_dir, requirements_dir) juju_log('Installing git repo from dir: {}'.format(repo_dir)) - pip_install(repo_dir) + if http_proxy: + pip_install(repo_dir, proxy=http_proxy, + venv=os.path.join(parent_dir, 'venv')) + else: + pip_install(repo_dir, + venv=os.path.join(parent_dir, 'venv')) return repo_dir @@ -621,16 +645,27 @@ def _git_update_requirements(package_dir, reqs_dir): os.chdir(orig_dir) +def git_pip_venv_dir(projects_yaml): + """ + Return the pip virtualenv path. + """ + parent_dir = '/mnt/openstack-git' + + projects = _git_yaml_load(projects_yaml) + + if 'directory' in projects.keys(): + parent_dir = projects['directory'] + + return os.path.join(parent_dir, 'venv') + + def git_src_dir(projects_yaml, project): """ Return the directory where the specified project's source is located. """ parent_dir = '/mnt/openstack-git' - if not projects_yaml: - return - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) if 'directory' in projects.keys(): parent_dir = projects['directory'] @@ -640,3 +675,15 @@ def git_src_dir(projects_yaml, project): return os.path.join(parent_dir, os.path.basename(p['repository'])) return None + + +def git_yaml_value(projects_yaml, key): + """ + Return the value in projects_yaml for the specified key. + """ + projects = _git_yaml_load(projects_yaml) + + if key in projects.keys(): + return projects[key] + + return None diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 8659516b..07b0c1d7 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -17,8 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . +import os +import subprocess + from charmhelpers.fetch import apt_install, apt_update -from charmhelpers.core.hookenv import log +from charmhelpers.core.hookenv import charm_dir, log try: from pip import main as pip_execute @@ -51,11 +54,15 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, **options): +def pip_install(package, fatal=False, upgrade=False, venv=None, **options): """Install a python package""" - command = ["install"] + if venv: + venv_python = os.path.join(venv, 'bin/pip') + command = [venv_python, "install"] + else: + command = ["install"] - available_options = ('proxy', 'src', 'log', "index-url", ) + available_options = ('proxy', 'src', 'log', 'index-url', ) for option in parse_options(options, available_options): command.append(option) @@ -69,7 +76,10 @@ def pip_install(package, fatal=False, upgrade=False, **options): log("Installing {} package with options: {}".format(package, command)) - pip_execute(command) + if venv: + subprocess.check_call(command) + else: + pip_execute(command) def pip_uninstall(package, **options): @@ -94,3 +104,16 @@ def pip_list(): """Returns the list of current python installed packages """ return pip_execute(["list"]) + + +def pip_create_virtualenv(path=None): + """Create an isolated Python environment.""" + apt_install('python-virtualenv') + + if path: + venv_path = path + else: + venv_path = os.path.join(charm_dir(), 'venv') + + if not os.path.exists(venv_path): + subprocess.check_call(['virtualenv', venv_path]) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 93aae87b..ddc25b7e 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -45,14 +45,16 @@ class GitUrlFetchHandler(BaseFetchHandler): else: return True - def clone(self, source, dest, branch): + def clone(self, source, dest, branch, depth=None): if not self.can_handle(source): raise UnhandledSource("Cannot handle {}".format(source)) - repo = Repo.clone_from(source, dest) - repo.git.checkout(branch) + if depth: + Repo.clone_from(source, dest, branch=branch, depth=depth) + else: + Repo.clone_from(source, dest, branch=branch) - def install(self, source, branch="master", dest=None): + def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) branch_name = url_parts.path.strip("/").split("/")[-1] if dest: @@ -63,7 +65,7 @@ class GitUrlFetchHandler(BaseFetchHandler): if not os.path.exists(dest_dir): mkdir(dest_dir, perms=0o755) try: - self.clone(source, dest_dir, branch) + self.clone(source, dest_dir, branch, depth) except GitCommandError as e: raise UnhandledSource(e.message) except OSError as e: diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index 94b2937d..4550dddb 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -5,6 +5,10 @@ import subprocess from collections import OrderedDict from copy import copy +from charmhelpers.contrib.python.packages import ( + pip_install, +) + from charmhelpers.core.hookenv import ( charm_dir, config, @@ -69,6 +73,8 @@ from charmhelpers.contrib.openstack.utils import ( git_install_requested, git_clone_and_install, git_src_dir, + git_yaml_value, + git_pip_venv_dir, os_release, ) @@ -94,8 +100,12 @@ VOLUME_PACKAGES = ['cinder-volume'] SCHEDULER_PACKAGES = ['cinder-scheduler'] BASE_GIT_PACKAGES = [ + 'libffi-dev', + 'libmysqlclient-dev', + 'libssl-dev', 'libxml2-dev', 'libxslt1-dev', + 'libyaml-dev', 'lvm2', 'python-dev', 'python-pip', @@ -564,6 +574,16 @@ def git_pre_install(): def git_post_install(projects_yaml): """Perform cinder post-install setup.""" + http_proxy = git_yaml_value(projects_yaml, 'http_proxy') + base_packages = ['mysql-python'] + for pkg in base_packages: + if http_proxy: + pip_install(pkg, proxy=http_proxy, + venv=git_pip_venv_dir(projects_yaml)) + else: + pip_install(pkg, + venv=git_pip_venv_dir(projects_yaml)) + src_etc = os.path.join(git_src_dir(projects_yaml, 'cinder'), 'etc/cinder') configs = { 'src': src_etc, @@ -574,6 +594,29 @@ def git_post_install(projects_yaml): shutil.rmtree(configs['dest']) shutil.copytree(configs['src'], configs['dest']) + # NOTE(coreycb): Need to find better solution than bin symlinks. + symlinks = [ + {'src': os.path.join(git_pip_venv_dir(projects_yaml), + 'bin/cinder-manage'), + 'link': '/usr/local/bin/cinder-manage'}, + {'src': os.path.join(git_pip_venv_dir(projects_yaml), + 'bin/cinder-rootwrap'), + 'link': '/usr/local/bin/cinder-rootwrap'}, + # NOTE(coreycb): This is ugly but couldn't find pypi package that + # installs rbd.py and rados.py. + {'src': '/usr/lib/python2.7/dist-packages/rbd.py', + 'link': os.path.join(git_pip_venv_dir(projects_yaml), + 'lib/python2.7/site-packages/rbd.py')}, + {'src': '/usr/lib/python2.7/dist-packages/rados.py', + 'link': os.path.join(git_pip_venv_dir(projects_yaml), + 'lib/python2.7/site-packages/rados.py')}, + ] + + for s in symlinks: + if os.path.lexists(s['link']): + os.remove(s['link']) + os.symlink(s['src'], s['link']) + render('cinder.conf', '/etc/cinder/cinder.conf', {}, owner='cinder', group='cinder', perms=0o644) render('git/cinder_tgt.conf', '/etc/tgt/conf.d', {}, owner='cinder', @@ -585,13 +628,14 @@ def git_post_install(projects_yaml): os.chmod('/etc/sudoers.d', 0o750) + bin_dir = os.path.join(git_pip_venv_dir(projects_yaml), 'bin') cinder_api_context = { 'service_description': 'Cinder API server', 'service_name': 'Cinder', 'user_name': 'cinder', 'start_dir': '/var/lib/cinder', 'process_name': 'cinder-api', - 'executable_name': '/usr/local/bin/cinder-api', + 'executable_name': os.path.join(bin_dir, 'cinder-api'), 'config_files': ['/etc/cinder/cinder.conf'], 'log_file': '/var/log/cinder/cinder-api.log', } @@ -602,7 +646,7 @@ def git_post_install(projects_yaml): 'user_name': 'cinder', 'start_dir': '/var/lib/cinder', 'process_name': 'cinder-backup', - 'executable_name': '/usr/local/bin/cinder-backup', + 'executable_name': os.path.join(bin_dir, 'cinder-backup'), 'config_files': ['/etc/cinder/cinder.conf'], 'log_file': '/var/log/cinder/cinder-backup.log', } @@ -613,7 +657,7 @@ def git_post_install(projects_yaml): 'user_name': 'cinder', 'start_dir': '/var/lib/cinder', 'process_name': 'cinder-scheduler', - 'executable_name': '/usr/local/bin/cinder-scheduler', + 'executable_name': os.path.join(bin_dir, 'cinder-scheduler'), 'config_files': ['/etc/cinder/cinder.conf'], 'log_file': '/var/log/cinder/cinder-scheduler.log', } @@ -624,7 +668,7 @@ def git_post_install(projects_yaml): 'user_name': 'cinder', 'start_dir': '/var/lib/cinder', 'process_name': 'cinder-volume', - 'executable_name': '/usr/local/bin/cinder-volume', + 'executable_name': os.path.join(bin_dir, 'cinder-volume'), 'config_files': ['/etc/cinder/cinder.conf'], 'log_file': '/var/log/cinder/cinder-volume.log', } diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index e1afa7fc..0bfcbeb0 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -80,11 +80,10 @@ class CinderBasicDeployment(OpenStackAmuletDeployment): openstack_origin_git = { 'repositories': [ {'name': 'requirements', - 'repository': - 'git://git.openstack.org/openstack/requirements', + 'repository': 'git://github.com/openstack/requirements', 'branch': branch}, {'name': 'cinder', - 'repository': 'git://git.openstack.org/openstack/cinder', + 'repository': 'git://github.com/openstack/cinder', 'branch': branch}, ], 'directory': '/mnt/openstack-git', diff --git a/unit_tests/test_cinder_utils.py b/unit_tests/test_cinder_utils.py index 03360bf4..761e4ab7 100644 --- a/unit_tests/test_cinder_utils.py +++ b/unit_tests/test_cinder_utils.py @@ -572,6 +572,7 @@ class TestCinderUtils(CharmTestCase): @patch.object(cinder_utils, 'git_src_dir') @patch.object(cinder_utils, 'service_restart') @patch.object(cinder_utils, 'render') + @patch.object(cinder_utils, 'pip_install') @patch('os.path.join') @patch('os.path.exists') @patch('shutil.copytree') @@ -580,24 +581,33 @@ class TestCinderUtils(CharmTestCase): @patch('grp.getgrnam') @patch('os.chown') @patch('os.chmod') - def test_git_post_install(self, chmod, chown, grp, pwd, rmtree, copytree, - exists, join, render, service_restart, - git_src_dir): + @patch('os.symlink') + def test_git_post_install(self, symlink, chmod, chown, grp, pwd, rmtree, + copytree, exists, join, pip_install, render, + service_restart, git_src_dir): projects_yaml = openstack_origin_git join.return_value = 'joined-string' cinder_utils.git_post_install(projects_yaml) + pip_install('mysql-python', venv='joined-string') expected = [ call('joined-string', '/etc/cinder'), ] copytree.assert_has_calls(expected) + expected = [ + call('joined-string', '/usr/local/bin/cinder-manage'), + call('/usr/lib/python2.7/dist-packages/rbd.py', 'joined-string'), + call('/usr/lib/python2.7/dist-packages/rados.py', 'joined-string'), + ] + symlink.assert_has_calls(expected, any_order=True) + cinder_api_context = { 'service_description': 'Cinder API server', 'service_name': 'Cinder', 'user_name': 'cinder', 'start_dir': '/var/lib/cinder', 'process_name': 'cinder-api', - 'executable_name': '/usr/local/bin/cinder-api', + 'executable_name': 'joined-string', 'config_files': ['/etc/cinder/cinder.conf'], 'log_file': '/var/log/cinder/cinder-api.log', } @@ -608,7 +618,7 @@ class TestCinderUtils(CharmTestCase): 'user_name': 'cinder', 'start_dir': '/var/lib/cinder', 'process_name': 'cinder-backup', - 'executable_name': '/usr/local/bin/cinder-backup', + 'executable_name': 'joined-string', 'config_files': ['/etc/cinder/cinder.conf'], 'log_file': '/var/log/cinder/cinder-backup.log', } @@ -619,7 +629,7 @@ class TestCinderUtils(CharmTestCase): 'user_name': 'cinder', 'start_dir': '/var/lib/cinder', 'process_name': 'cinder-scheduler', - 'executable_name': '/usr/local/bin/cinder-scheduler', + 'executable_name': 'joined-string', 'config_files': ['/etc/cinder/cinder.conf'], 'log_file': '/var/log/cinder/cinder-scheduler.log', } @@ -630,7 +640,7 @@ class TestCinderUtils(CharmTestCase): 'user_name': 'cinder', 'start_dir': '/var/lib/cinder', 'process_name': 'cinder-volume', - 'executable_name': '/usr/local/bin/cinder-volume', + 'executable_name': 'joined-string', 'config_files': ['/etc/cinder/cinder.conf'], 'log_file': '/var/log/cinder/cinder-volume.log', }