From c5202158f303ff6689c423a809fb173aaaaddec7 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 22 Apr 2015 18:45:46 +0000 Subject: [PATCH 01/38] Sync charm-helpers --- .../contrib/openstack/amulet/deployment.py | 21 +++++++++++++------ hooks/charmhelpers/contrib/openstack/utils.py | 12 +++++++++-- hooks/charmhelpers/contrib/python/packages.py | 7 +++++-- tests/charmhelpers/contrib/amulet/utils.py | 9 +++++++- .../contrib/openstack/amulet/deployment.py | 21 +++++++++++++------ 5 files changed, 53 insertions(+), 17 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 11d49a7c..461a702f 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -46,15 +46,22 @@ class OpenStackAmuletDeployment(AmuletDeployment): stable or next branches for the other_services.""" base_charms = ['mysql', 'mongodb'] + if self.series in ['precise', 'trusty']: + base_series = self.series + else: + base_series = self.current_next + if self.stable: for svc in other_services: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: for svc in other_services: if svc['name'] in base_charms: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: temp = 'lp:~openstack-charmers/charms/{}/{}/next' svc['location'] = temp.format(self.current_next, @@ -99,10 +106,12 @@ class OpenStackAmuletDeployment(AmuletDeployment): Return an integer representing the enum value of the openstack release. """ + # Must be ordered by OpenStack release (not by Ubuntu release): (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse, self.trusty_juno, self.trusty_kilo, - self.utopic_juno, self.vivid_kilo) = range(10) + self.trusty_icehouse, self.trusty_juno, self.utopic_juno, + self.trusty_kilo, self.vivid_kilo) = range(10) + releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index f90a0289..f4824f99 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -517,6 +517,7 @@ def git_clone_and_install(projects_yaml, core_project): """ global requirements_dir parent_dir = '/mnt/openstack-git' + http_proxy = None if not projects_yaml: return @@ -527,6 +528,7 @@ def git_clone_and_install(projects_yaml, core_project): old_environ = dict(os.environ) if 'http_proxy' in projects.keys(): + http_proxy = projects['http_proxy'] os.environ['http_proxy'] = projects['http_proxy'] if 'https_proxy' in projects.keys(): os.environ['https_proxy'] = projects['https_proxy'] @@ -539,10 +541,12 @@ def git_clone_and_install(projects_yaml, core_project): branch = p['branch'] if p['name'] == 'requirements': repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + http_proxy, update_requirements=False) requirements_dir = repo_dir else: repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + http_proxy, update_requirements=True) os.environ = old_environ @@ -574,7 +578,8 @@ def _git_ensure_key_exists(key, keys): error_out('openstack-origin-git key \'{}\' is missing'.format(key)) -def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements): +def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, + update_requirements): """ Clone and install a single git repository. """ @@ -598,7 +603,10 @@ def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements) _git_update_requirements(repo_dir, requirements_dir) juju_log('Installing git repo from dir: {}'.format(repo_dir)) - pip_install(repo_dir) + if http_proxy: + pip_install(repo_dir, ignore=True, proxy=http_proxy) + else: + pip_install(repo_dir, ignore=True) return repo_dir diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 8659516b..ceaa5971 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -51,17 +51,20 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, **options): +def pip_install(package, fatal=False, upgrade=False, ignore=False, **options): """Install a python package""" command = ["install"] - available_options = ('proxy', 'src', 'log', "index-url", ) + available_options = ('proxy', 'src', 'log', 'index-url', ) for option in parse_options(options, available_options): command.append(option) if upgrade: command.append('--upgrade') + if ignore: + command.append('--ignore-installed') + if isinstance(package, list): command.extend(package) else: diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 5088b1d1..f61c2e8b 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -79,6 +79,9 @@ class AmuletUtils(object): for k, v in six.iteritems(commands): for cmd in v: output, code = k.run(cmd) + self.log.debug('{} `{}` returned ' + '{}'.format(k.info['unit_name'], + cmd, code)) if code != 0: return "command `{}` returned {}".format(cmd, str(code)) return None @@ -86,7 +89,11 @@ class AmuletUtils(object): def _get_config(self, unit, filename): """Get a ConfigParser object for parsing a unit's config file.""" file_contents = unit.file_contents(filename) - config = ConfigParser.ConfigParser() + + # NOTE(beisner): by default, ConfigParser does not handle options + # with no value, such as the flags used in the mysql my.cnf file. + # https://bugs.python.org/issue7005 + config = ConfigParser.ConfigParser(allow_no_value=True) config.readfp(io.StringIO(file_contents)) return config diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 11d49a7c..461a702f 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -46,15 +46,22 @@ class OpenStackAmuletDeployment(AmuletDeployment): stable or next branches for the other_services.""" base_charms = ['mysql', 'mongodb'] + if self.series in ['precise', 'trusty']: + base_series = self.series + else: + base_series = self.current_next + if self.stable: for svc in other_services: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: for svc in other_services: if svc['name'] in base_charms: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: temp = 'lp:~openstack-charmers/charms/{}/{}/next' svc['location'] = temp.format(self.current_next, @@ -99,10 +106,12 @@ class OpenStackAmuletDeployment(AmuletDeployment): Return an integer representing the enum value of the openstack release. """ + # Must be ordered by OpenStack release (not by Ubuntu release): (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse, self.trusty_juno, self.trusty_kilo, - self.utopic_juno, self.vivid_kilo) = range(10) + self.trusty_icehouse, self.trusty_juno, self.utopic_juno, + self.trusty_kilo, self.vivid_kilo) = range(10) + releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, From c7c06216fabe62c726177c33be204f2b4be777b6 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 22 Apr 2015 19:31:40 +0000 Subject: [PATCH 02/38] Add libffi-dev, libssl-dev, and libyaml-dev to base git install packages --- hooks/cinder_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index 94b2937d..5e05e5d8 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -94,8 +94,11 @@ VOLUME_PACKAGES = ['cinder-volume'] SCHEDULER_PACKAGES = ['cinder-scheduler'] BASE_GIT_PACKAGES = [ + 'libffi-dev', + 'libssl-dev', 'libxml2-dev', 'libxslt1-dev', + 'libyaml-dev', 'lvm2', 'python-dev', 'python-pip', From 00d2cefb30aa8c4cac115bdacfb2d26520b0a364 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 11:55:31 +0000 Subject: [PATCH 03/38] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- hooks/charmhelpers/contrib/python/packages.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index f4824f99..0460e076 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -604,9 +604,9 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, ignore=True, proxy=http_proxy) + pip_install(repo_dir, proxy=http_proxy) else: - pip_install(repo_dir, ignore=True) + pip_install(repo_dir) return repo_dir diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index ceaa5971..3f1fb09d 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -51,7 +51,7 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, ignore=False, **options): +def pip_install(package, fatal=False, upgrade=False, **options): """Install a python package""" command = ["install"] @@ -62,9 +62,6 @@ def pip_install(package, fatal=False, upgrade=False, ignore=False, **options): if upgrade: command.append('--upgrade') - if ignore: - command.append('--ignore-installed') - if isinstance(package, list): command.extend(package) else: From b395a2620a7f8922ee696119f65f1be20a18638e Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 13:01:33 +0000 Subject: [PATCH 04/38] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 0460e076..86bf91f5 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -604,9 +604,9 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, proxy=http_proxy) + pip_install(repo_dir, upgrade=True, proxy=http_proxy) else: - pip_install(repo_dir) + pip_install(repo_dir, upgrade=True) return repo_dir From d10d8fb5a995484abde5928c8a8acc4e2406ee61 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 14:43:26 +0000 Subject: [PATCH 05/38] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 86bf91f5..0460e076 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -604,9 +604,9 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, upgrade=True, proxy=http_proxy) + pip_install(repo_dir, proxy=http_proxy) else: - pip_install(repo_dir, upgrade=True) + pip_install(repo_dir) return repo_dir From 9d070616f1ffe0772285342423d687afd2f55c35 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 15:57:32 +0000 Subject: [PATCH 06/38] Drop added packages --- hooks/cinder_utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index 5e05e5d8..94b2937d 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -94,11 +94,8 @@ VOLUME_PACKAGES = ['cinder-volume'] SCHEDULER_PACKAGES = ['cinder-scheduler'] BASE_GIT_PACKAGES = [ - 'libffi-dev', - 'libssl-dev', 'libxml2-dev', 'libxslt1-dev', - 'libyaml-dev', 'lvm2', 'python-dev', 'python-pip', From 9cc1e7cfe189e4efecf47f55bee46d1a7fab64f5 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 28 Apr 2015 19:04:06 +0000 Subject: [PATCH 07/38] Sync charm-helpers --- hooks/charmhelpers/fetch/__init__.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 792e629a..57a179b8 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -367,13 +367,15 @@ def install_remote(source, *args, **kwargs): # explaining why it can't handle a given source. handlers = [h for h in plugins() if h.can_handle(source) is True] installed_to = None - for handler in handlers: - try: - installed_to = handler.install(source, *args, **kwargs) - except UnhandledSource: - pass - if not installed_to: - raise UnhandledSource("No handler found for source {}".format(source)) + while not installed_to: + for handler in handlers: + try: + installed_to = handler.install(source, *args, **kwargs) + except UnhandledSource: + pass + log("CB: install loop - installed_to = |%s|".format(installed_to)) + #if not installed_to: + # raise UnhandledSource("No handler found for source {}".format(source)) return installed_to From 8fed9963f9f2c3d7ce4c49fe346c2ad74cb82b19 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 1 May 2015 14:55:54 +0000 Subject: [PATCH 08/38] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 15 ++++++++------- hooks/charmhelpers/fetch/__init__.py | 16 +++++++--------- hooks/charmhelpers/fetch/giturl.py | 11 +++++++---- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 0460e076..b47ed937 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -497,7 +497,7 @@ def git_install_requested(): requirements_dir = None -def git_clone_and_install(projects_yaml, core_project): +def git_clone_and_install(projects_yaml, core_project, depth=1): """ Clone/install all specified OpenStack repositories. @@ -540,13 +540,13 @@ def git_clone_and_install(projects_yaml, core_project): repo = p['repository'] branch = p['branch'] if p['name'] == 'requirements': - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, - http_proxy, + repo_dir = _git_clone_and_install_single(repo, branch, depth, + parent_dir, http_proxy, update_requirements=False) requirements_dir = repo_dir else: - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, - http_proxy, + repo_dir = _git_clone_and_install_single(repo, branch, depth, + parent_dir, http_proxy, update_requirements=True) os.environ = old_environ @@ -578,7 +578,7 @@ def _git_ensure_key_exists(key, keys): error_out('openstack-origin-git key \'{}\' is missing'.format(key)) -def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, +def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, update_requirements): """ Clone and install a single git repository. @@ -592,7 +592,8 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, if not os.path.exists(dest_dir): juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) - repo_dir = install_remote(repo, dest=parent_dir, branch=branch) + repo_dir = install_remote(repo, dest=parent_dir, branch=branch, + depth=depth) else: repo_dir = dest_dir diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 57a179b8..792e629a 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -367,15 +367,13 @@ def install_remote(source, *args, **kwargs): # explaining why it can't handle a given source. handlers = [h for h in plugins() if h.can_handle(source) is True] installed_to = None - while not installed_to: - for handler in handlers: - try: - installed_to = handler.install(source, *args, **kwargs) - except UnhandledSource: - pass - log("CB: install loop - installed_to = |%s|".format(installed_to)) - #if not installed_to: - # raise UnhandledSource("No handler found for source {}".format(source)) + for handler in handlers: + try: + installed_to = handler.install(source, *args, **kwargs) + except UnhandledSource: + pass + if not installed_to: + raise UnhandledSource("No handler found for source {}".format(source)) return installed_to diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 93aae87b..06b0e767 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -45,14 +45,17 @@ class GitUrlFetchHandler(BaseFetchHandler): else: return True - def clone(self, source, dest, branch): + def clone(self, source, dest, branch, depth): if not self.can_handle(source): raise UnhandledSource("Cannot handle {}".format(source)) - repo = Repo.clone_from(source, dest) + if depth: + repo = Repo.clone_from(source, dest, depth) + else: + repo = Repo.clone_from(source, dest) repo.git.checkout(branch) - def install(self, source, branch="master", dest=None): + def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) branch_name = url_parts.path.strip("/").split("/")[-1] if dest: @@ -63,7 +66,7 @@ class GitUrlFetchHandler(BaseFetchHandler): if not os.path.exists(dest_dir): mkdir(dest_dir, perms=0o755) try: - self.clone(source, dest_dir, branch) + self.clone(source, dest_dir, branch, depth) except GitCommandError as e: raise UnhandledSource(e.message) except OSError as e: From dbb56bf872cbb5895655548aca27e0727d96d601 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 2 May 2015 18:30:16 +0000 Subject: [PATCH 09/38] Sync charm-helpers --- hooks/charmhelpers/fetch/giturl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 06b0e767..742cf319 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -50,7 +50,7 @@ class GitUrlFetchHandler(BaseFetchHandler): raise UnhandledSource("Cannot handle {}".format(source)) if depth: - repo = Repo.clone_from(source, dest, depth) + repo = Repo.clone_from(source, dest, depth=depth) else: repo = Repo.clone_from(source, dest) repo.git.checkout(branch) From 467a548667cb853aa9d3f825a5ddc4835253b452 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 2 May 2015 20:09:17 +0000 Subject: [PATCH 10/38] Sync charm-helpers --- hooks/charmhelpers/fetch/giturl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 742cf319..4e05c08a 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -50,10 +50,9 @@ class GitUrlFetchHandler(BaseFetchHandler): raise UnhandledSource("Cannot handle {}".format(source)) if depth: - repo = Repo.clone_from(source, dest, depth=depth) + repo = Repo.clone_from(source, dest, branch=branch, depth=depth) else: - repo = Repo.clone_from(source, dest) - repo.git.checkout(branch) + repo = Repo.clone_from(source, dest, branch=branch) def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) From 6c84d3f58b1d9f1ea96c661d9f1724c5c5ec2a03 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 5 May 2015 19:08:30 +0000 Subject: [PATCH 11/38] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 8 +++++++- hooks/charmhelpers/contrib/python/packages.py | 19 ++++++++++++++++++- hooks/charmhelpers/fetch/giturl.py | 4 ++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index b47ed937..ce8c8213 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -53,9 +53,13 @@ from charmhelpers.contrib.network.ip import ( get_ipv6_addr ) +from charmhelpers.contrib.python.packages import ( + pip_create_virtualenv, + pip_install, +) + from charmhelpers.core.host import lsb_release, mounts, umount from charmhelpers.fetch import apt_install, apt_cache, install_remote -from charmhelpers.contrib.python.packages import pip_install from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device @@ -536,6 +540,8 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): if 'directory' in projects.keys(): parent_dir = projects['directory'] + pip_create_virtualenv(proxy=http_proxy) + for p in projects['repositories']: repo = p['repository'] branch = p['branch'] diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 3f1fb09d..e62240b4 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -17,8 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . +import os +import subprocess + from charmhelpers.fetch import apt_install, apt_update -from charmhelpers.core.hookenv import log +from charmhelpers.core.hookenv import charm_dir, log try: from pip import main as pip_execute @@ -94,3 +97,17 @@ def pip_list(): """Returns the list of current python installed packages """ return pip_execute(["list"]) + + +def pip_create_virtualenv(parent_dir=charm_dir(), proxy=None): + """Create and activate an isolated Python environment (virtualenv).""" + if proxy: + pip_install('virtualenv', proxy=proxy) + else: + pip_install('virtualenv') + + venv_dir = os.path.join(parent_dir, 'venv') + subprocess.check_call(['virtualenv', '--no-site-packages', venv_dir]) + + venv_activate = os.path.join(venv_dir, 'bin/activate') + subprocess.check_call(['.', venv_activate], shell=True) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 4e05c08a..7d842d49 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -50,9 +50,9 @@ class GitUrlFetchHandler(BaseFetchHandler): raise UnhandledSource("Cannot handle {}".format(source)) if depth: - repo = Repo.clone_from(source, dest, branch=branch, depth=depth) + Repo.clone_from(source, dest, branch=branch, depth=depth) else: - repo = Repo.clone_from(source, dest, branch=branch) + Repo.clone_from(source, dest, branch=branch) def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) From 407bf91e2b521a3c912e83d36e3eb6c61c50e12c Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 5 May 2015 19:16:38 +0000 Subject: [PATCH 12/38] Point upstart scripts at venv binaries --- hooks/cinder_utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index 94b2937d..b24bbffa 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -585,13 +585,14 @@ def git_post_install(projects_yaml): os.chmod('/etc/sudoers.d', 0o750) + bin_dir = os.path.join(charm_dir(), 'venv/bin') cinder_api_context = { 'service_description': 'Cinder API server', 'service_name': 'Cinder', 'user_name': 'cinder', 'start_dir': '/var/lib/cinder', 'process_name': 'cinder-api', - 'executable_name': '/usr/local/bin/cinder-api', + 'executable_name': os.path.join(bin_dir, 'cinder-api'), 'config_files': ['/etc/cinder/cinder.conf'], 'log_file': '/var/log/cinder/cinder-api.log', } @@ -602,7 +603,7 @@ def git_post_install(projects_yaml): 'user_name': 'cinder', 'start_dir': '/var/lib/cinder', 'process_name': 'cinder-backup', - 'executable_name': '/usr/local/bin/cinder-backup', + 'executable_name': os.path.join(bin_dir, 'cinder-backup'), 'config_files': ['/etc/cinder/cinder.conf'], 'log_file': '/var/log/cinder/cinder-backup.log', } @@ -613,7 +614,7 @@ def git_post_install(projects_yaml): 'user_name': 'cinder', 'start_dir': '/var/lib/cinder', 'process_name': 'cinder-scheduler', - 'executable_name': '/usr/local/bin/cinder-scheduler', + 'executable_name': os.path.join(bin_dir, 'cinder-scheduler'), 'config_files': ['/etc/cinder/cinder.conf'], 'log_file': '/var/log/cinder/cinder-scheduler.log', } @@ -624,7 +625,7 @@ def git_post_install(projects_yaml): 'user_name': 'cinder', 'start_dir': '/var/lib/cinder', 'process_name': 'cinder-volume', - 'executable_name': '/usr/local/bin/cinder-volume', + 'executable_name': os.path.join(bin_dir, 'cinder-volume'), 'config_files': ['/etc/cinder/cinder.conf'], 'log_file': '/var/log/cinder/cinder-volume.log', } From 83c007bdc39bd7e83b59ff2a19237211d3ff92a6 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 5 May 2015 19:53:21 +0000 Subject: [PATCH 13/38] Update unit tests --- unit_tests/test_cinder_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/unit_tests/test_cinder_utils.py b/unit_tests/test_cinder_utils.py index 03360bf4..2f1a173c 100644 --- a/unit_tests/test_cinder_utils.py +++ b/unit_tests/test_cinder_utils.py @@ -597,7 +597,7 @@ class TestCinderUtils(CharmTestCase): 'user_name': 'cinder', 'start_dir': '/var/lib/cinder', 'process_name': 'cinder-api', - 'executable_name': '/usr/local/bin/cinder-api', + 'executable_name': 'joined-string', 'config_files': ['/etc/cinder/cinder.conf'], 'log_file': '/var/log/cinder/cinder-api.log', } @@ -608,7 +608,7 @@ class TestCinderUtils(CharmTestCase): 'user_name': 'cinder', 'start_dir': '/var/lib/cinder', 'process_name': 'cinder-backup', - 'executable_name': '/usr/local/bin/cinder-backup', + 'executable_name': 'joined-string', 'config_files': ['/etc/cinder/cinder.conf'], 'log_file': '/var/log/cinder/cinder-backup.log', } @@ -619,7 +619,7 @@ class TestCinderUtils(CharmTestCase): 'user_name': 'cinder', 'start_dir': '/var/lib/cinder', 'process_name': 'cinder-scheduler', - 'executable_name': '/usr/local/bin/cinder-scheduler', + 'executable_name': 'joined-string', 'config_files': ['/etc/cinder/cinder.conf'], 'log_file': '/var/log/cinder/cinder-scheduler.log', } @@ -630,7 +630,7 @@ class TestCinderUtils(CharmTestCase): 'user_name': 'cinder', 'start_dir': '/var/lib/cinder', 'process_name': 'cinder-volume', - 'executable_name': '/usr/local/bin/cinder-volume', + 'executable_name': 'joined-string', 'config_files': ['/etc/cinder/cinder.conf'], 'log_file': '/var/log/cinder/cinder-volume.log', } From 56901c33ff6e45e0c5c27c6a8987782725e2c98f Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 6 May 2015 15:59:46 +0000 Subject: [PATCH 14/38] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 6 ++-- hooks/charmhelpers/contrib/python/packages.py | 29 +++++++++---------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index ce8c8213..5909b0f5 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -540,7 +540,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): if 'directory' in projects.keys(): parent_dir = projects['directory'] - pip_create_virtualenv(proxy=http_proxy) + pip_create_virtualenv() for p in projects['repositories']: repo = p['repository'] @@ -611,9 +611,9 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, proxy=http_proxy) + pip_install(repo_dir, proxy=http_proxy, venv=True) else: - pip_install(repo_dir) + pip_install(repo_dir, venv=True) return repo_dir diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index e62240b4..740eaa51 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -54,9 +54,13 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, **options): +def pip_install(package, fatal=False, upgrade=False, venv=False, **options): """Install a python package""" - command = ["install"] + if venv: + venv_python = os.path.join(charm_dir(), 'venv/bin/pip') + command = [venv_python, "install"] + else: + command = ["install"] available_options = ('proxy', 'src', 'log', 'index-url', ) for option in parse_options(options, available_options): @@ -72,7 +76,10 @@ def pip_install(package, fatal=False, upgrade=False, **options): log("Installing {} package with options: {}".format(package, command)) - pip_execute(command) + if venv: + subprocess.check_call(command) + else: + pip_execute(command) def pip_uninstall(package, **options): @@ -99,15 +106,7 @@ def pip_list(): return pip_execute(["list"]) -def pip_create_virtualenv(parent_dir=charm_dir(), proxy=None): - """Create and activate an isolated Python environment (virtualenv).""" - if proxy: - pip_install('virtualenv', proxy=proxy) - else: - pip_install('virtualenv') - - venv_dir = os.path.join(parent_dir, 'venv') - subprocess.check_call(['virtualenv', '--no-site-packages', venv_dir]) - - venv_activate = os.path.join(venv_dir, 'bin/activate') - subprocess.check_call(['.', venv_activate], shell=True) +def pip_create_virtualenv(): + """Create an isolated Python environment.""" + apt_install('python-virtualenv') + subprocess.check_call(['virtualenv', os.path.join(charm_dir(), 'venv')]) From fbc1a5b1e4828f5b9b791bf3ae6de2ae519c48be Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 6 May 2015 16:14:25 +0000 Subject: [PATCH 15/38] Add libffi-dev to base git install packages --- hooks/cinder_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index b24bbffa..8763630e 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -94,6 +94,7 @@ VOLUME_PACKAGES = ['cinder-volume'] SCHEDULER_PACKAGES = ['cinder-scheduler'] BASE_GIT_PACKAGES = [ + 'libffi-dev', 'libxml2-dev', 'libxslt1-dev', 'lvm2', From 51b7082b7c3a935d9b184949d9c05eea64ad27dc Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 6 May 2015 18:20:29 +0000 Subject: [PATCH 16/38] Add libssl-dev to base git install packages --- hooks/cinder_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index 8763630e..23bbe2fa 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -95,6 +95,7 @@ SCHEDULER_PACKAGES = ['cinder-scheduler'] BASE_GIT_PACKAGES = [ 'libffi-dev', + 'libssl-dev', 'libxml2-dev', 'libxslt1-dev', 'lvm2', From 420e857bba3f4d5d3d9d8da02b04685631f80f74 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 6 May 2015 19:11:05 +0000 Subject: [PATCH 17/38] Symlink cinder-manage to /usr/local/bin/cinder-manage for 'cinder-manage db sync' --- hooks/cinder_utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index 23bbe2fa..484eae77 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -576,6 +576,16 @@ def git_post_install(projects_yaml): shutil.rmtree(configs['dest']) shutil.copytree(configs['src'], configs['dest']) + symlinks = [ + {'src': os.path.join(charm_dir(), 'venv/bin/cinder-manage'), + 'link': '/usr/local/bin/cinder-manage'}, + ] + + for s in symlinks: + if os.path.lexists(s['link']): + os.remove(s['link']) + os.symlink(s['src'], s['link']) + render('cinder.conf', '/etc/cinder/cinder.conf', {}, owner='cinder', group='cinder', perms=0o644) render('git/cinder_tgt.conf', '/etc/tgt/conf.d', {}, owner='cinder', From 8c152634124bde21c2c0a6d52861dccba5ff9e4e Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 6 May 2015 19:46:38 +0000 Subject: [PATCH 18/38] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 5909b0f5..6d91490e 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -655,3 +655,18 @@ def git_src_dir(projects_yaml, project): return os.path.join(parent_dir, os.path.basename(p['repository'])) return None + + +def git_http_proxy(projects_yaml): + """ + Return the http_proxy value if it is specified in projects_yaml. + """ + if not projects_yaml: + return + + projects = yaml.load(projects_yaml) + + if 'http_proxy' in projects.keys(): + return projects['http_proxy'] + + return None From ef1a83919e298a29a96fcd0fbb5e7bfda1ecd9dd Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 6 May 2015 20:01:50 +0000 Subject: [PATCH 19/38] Install mysql client/interface packages --- hooks/cinder_utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index 484eae77..733b79e5 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -5,6 +5,10 @@ import subprocess from collections import OrderedDict from copy import copy +from charmhelpers.contrib.python.packages import ( + pip_install, +) + from charmhelpers.core.hookenv import ( charm_dir, config, @@ -69,6 +73,7 @@ from charmhelpers.contrib.openstack.utils import ( git_install_requested, git_clone_and_install, git_src_dir, + git_http_proxy, os_release, ) @@ -95,6 +100,7 @@ SCHEDULER_PACKAGES = ['cinder-scheduler'] BASE_GIT_PACKAGES = [ 'libffi-dev', + 'libmysqlclient-dev', 'libssl-dev', 'libxml2-dev', 'libxslt1-dev', @@ -566,6 +572,12 @@ def git_pre_install(): def git_post_install(projects_yaml): """Perform cinder post-install setup.""" + http_proxy = git_http_proxy(projects_yaml) + if http_proxy: + pip_install('mysql-python', proxy=http_proxy, venv=True) + else: + pip_install('mysql-python', venv=True) + src_etc = os.path.join(git_src_dir(projects_yaml, 'cinder'), 'etc/cinder') configs = { 'src': src_etc, From 7fdc4e77b0bfc54bb0dca17da4236eb9c0bcdd2a Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 11:53:30 +0000 Subject: [PATCH 20/38] Unit test updates --- unit_tests/test_cinder_utils.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/unit_tests/test_cinder_utils.py b/unit_tests/test_cinder_utils.py index 2f1a173c..0dc0f7ff 100644 --- a/unit_tests/test_cinder_utils.py +++ b/unit_tests/test_cinder_utils.py @@ -572,6 +572,7 @@ class TestCinderUtils(CharmTestCase): @patch.object(cinder_utils, 'git_src_dir') @patch.object(cinder_utils, 'service_restart') @patch.object(cinder_utils, 'render') + @patch.object(cinder_utils, 'pip_install') @patch('os.path.join') @patch('os.path.exists') @patch('shutil.copytree') @@ -580,17 +581,24 @@ class TestCinderUtils(CharmTestCase): @patch('grp.getgrnam') @patch('os.chown') @patch('os.chmod') - def test_git_post_install(self, chmod, chown, grp, pwd, rmtree, copytree, - exists, join, render, service_restart, - git_src_dir): + @patch('os.symlink') + def test_git_post_install(self, symlink, chmod, chown, grp, pwd, rmtree, + copytree, exists, join, pip_install, render, + service_restart, git_src_dir): projects_yaml = openstack_origin_git join.return_value = 'joined-string' cinder_utils.git_post_install(projects_yaml) + pip_install.assert_called_with('mysql-python', venv=True) expected = [ call('joined-string', '/etc/cinder'), ] copytree.assert_has_calls(expected) + expected = [ + call('joined-string', '/usr/local/bin/cinder-manage'), + ] + symlink.assert_has_calls(expected, any_order=True) + cinder_api_context = { 'service_description': 'Cinder API server', 'service_name': 'Cinder', From 8a689f4dd04e11e62e9e0fbd16a4b9c7aed9fd94 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 12:21:54 +0000 Subject: [PATCH 21/38] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 10 +++++----- hooks/charmhelpers/contrib/python/packages.py | 19 ++++++++++++++++--- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 6d91490e..a1d09981 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -657,16 +657,16 @@ def git_src_dir(projects_yaml, project): return None -def git_http_proxy(projects_yaml): +def git_yaml_value(projects_yaml, key): """ - Return the http_proxy value if it is specified in projects_yaml. + Return the value in projects_yaml for the specified key. """ if not projects_yaml: - return + return None projects = yaml.load(projects_yaml) - if 'http_proxy' in projects.keys(): - return projects['http_proxy'] + if key in projects.keys(): + return projects[key] return None diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 740eaa51..1a34afff 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -30,6 +30,8 @@ except ImportError: apt_install('python-pip') from pip import main as pip_execute +pip_venv_path = None + __author__ = "Jorge Niedbalski " @@ -57,7 +59,7 @@ def pip_install_requirements(requirements, **options): def pip_install(package, fatal=False, upgrade=False, venv=False, **options): """Install a python package""" if venv: - venv_python = os.path.join(charm_dir(), 'venv/bin/pip') + venv_python = os.path.join(pip_venv_path, 'bin/pip') command = [venv_python, "install"] else: command = ["install"] @@ -106,7 +108,18 @@ def pip_list(): return pip_execute(["list"]) -def pip_create_virtualenv(): +def pip_create_virtualenv(path=None): """Create an isolated Python environment.""" + global pip_venv_path + apt_install('python-virtualenv') - subprocess.check_call(['virtualenv', os.path.join(charm_dir(), 'venv')]) + + if path: + pip_venv_path = path + else: + pip_venv_path = os.path.join(charm_dir(), 'venv') + + subprocess.check_call(['virtualenv', pip_venv_path]) + +def pip_get_virtualenv_path(): + return pip_venv_path From 6dc7254ec5081c63f7387a524a172b264fc3e0f8 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 12:23:38 +0000 Subject: [PATCH 22/38] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index a1d09981..e00d19c5 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -540,7 +540,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): if 'directory' in projects.keys(): parent_dir = projects['directory'] - pip_create_virtualenv() + pip_create_virtualenv(parent_dir) for p in projects['repositories']: repo = p['repository'] From 4b8a352341f0c000d10b96fc02c2e3e9f215ce3a Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 12:30:55 +0000 Subject: [PATCH 23/38] Use function to get pip venv path --- hooks/cinder_utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index 733b79e5..f86d10aa 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -7,6 +7,7 @@ from copy import copy from charmhelpers.contrib.python.packages import ( pip_install, + pip_get_virtualenv_path, ) from charmhelpers.core.hookenv import ( @@ -73,7 +74,7 @@ from charmhelpers.contrib.openstack.utils import ( git_install_requested, git_clone_and_install, git_src_dir, - git_http_proxy, + git_yaml_value, os_release, ) @@ -572,7 +573,7 @@ def git_pre_install(): def git_post_install(projects_yaml): """Perform cinder post-install setup.""" - http_proxy = git_http_proxy(projects_yaml) + http_proxy = git_yaml_value(projects_yaml, 'http_proxy') if http_proxy: pip_install('mysql-python', proxy=http_proxy, venv=True) else: @@ -589,7 +590,7 @@ def git_post_install(projects_yaml): shutil.copytree(configs['src'], configs['dest']) symlinks = [ - {'src': os.path.join(charm_dir(), 'venv/bin/cinder-manage'), + {'src': os.path.join(pip_get_virtualenv_path(), 'bin/cinder-manage'), 'link': '/usr/local/bin/cinder-manage'}, ] @@ -609,7 +610,7 @@ def git_post_install(projects_yaml): os.chmod('/etc/sudoers.d', 0o750) - bin_dir = os.path.join(charm_dir(), 'venv/bin') + bin_dir = os.path.join(pip_get_virtualenv_path(), 'bin') cinder_api_context = { 'service_description': 'Cinder API server', 'service_name': 'Cinder', From 1c594dc89b379dc4f4cdc633c7d4a41d4737c729 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 12:37:05 +0000 Subject: [PATCH 24/38] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 2 +- hooks/charmhelpers/contrib/python/packages.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index e00d19c5..29ac7b78 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -540,7 +540,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): if 'directory' in projects.keys(): parent_dir = projects['directory'] - pip_create_virtualenv(parent_dir) + pip_create_virtualenv(os.path.join(parent_dir, 'venv')) for p in projects['repositories']: repo = p['repository'] diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 1a34afff..12838d24 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -121,5 +121,6 @@ def pip_create_virtualenv(path=None): subprocess.check_call(['virtualenv', pip_venv_path]) + def pip_get_virtualenv_path(): return pip_venv_path From 426bce4c99b6b243730f95afe9b379193b85d5da Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 14:39:35 +0000 Subject: [PATCH 25/38] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 23 +++++++++++++++++-- hooks/charmhelpers/contrib/python/packages.py | 18 ++++----------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 29ac7b78..c5e08ec2 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -611,9 +611,11 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, proxy=http_proxy, venv=True) + pip_install(repo_dir, proxy=http_proxy, + venv=os.path.join(parent_dir, 'venv')) else: - pip_install(repo_dir, venv=True) + pip_install(repo_dir, + venv=os.path.join(parent_dir, 'venv')) return repo_dir @@ -636,6 +638,23 @@ def _git_update_requirements(package_dir, reqs_dir): os.chdir(orig_dir) +def git_pip_venv_dir(projects_yaml): + """ + Return the pip virtualenv path. + """ + parent_dir = '/mnt/openstack-git' + + if not projects_yaml: + return + + projects = yaml.load(projects_yaml) + + if 'directory' in projects.keys(): + parent_dir = projects['directory'] + + return os.path.join(parent_dir, 'venv') + + def git_src_dir(projects_yaml, project): """ Return the directory where the specified project's source is located. diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 12838d24..88c8887d 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -30,8 +30,6 @@ except ImportError: apt_install('python-pip') from pip import main as pip_execute -pip_venv_path = None - __author__ = "Jorge Niedbalski " @@ -56,10 +54,10 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, venv=False, **options): +def pip_install(package, fatal=False, upgrade=False, venv=None, **options): """Install a python package""" if venv: - venv_python = os.path.join(pip_venv_path, 'bin/pip') + venv_python = os.path.join(venv, 'bin/pip') command = [venv_python, "install"] else: command = ["install"] @@ -110,17 +108,11 @@ def pip_list(): def pip_create_virtualenv(path=None): """Create an isolated Python environment.""" - global pip_venv_path - apt_install('python-virtualenv') if path: - pip_venv_path = path + venv_path = path else: - pip_venv_path = os.path.join(charm_dir(), 'venv') + venv_path = os.path.join(charm_dir(), 'venv') - subprocess.check_call(['virtualenv', pip_venv_path]) - - -def pip_get_virtualenv_path(): - return pip_venv_path + subprocess.check_call(['virtualenv', venv_path]) From 7a9552d892a1d24503b0b7226314d44c8ce73d57 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 14:48:10 +0000 Subject: [PATCH 26/38] Use git_pip_venv_dir to get venv path --- hooks/cinder_utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index f86d10aa..c6450aa5 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -7,7 +7,6 @@ from copy import copy from charmhelpers.contrib.python.packages import ( pip_install, - pip_get_virtualenv_path, ) from charmhelpers.core.hookenv import ( @@ -75,6 +74,7 @@ from charmhelpers.contrib.openstack.utils import ( git_clone_and_install, git_src_dir, git_yaml_value, + git_pip_venv_dir, os_release, ) @@ -575,9 +575,11 @@ def git_post_install(projects_yaml): """Perform cinder post-install setup.""" http_proxy = git_yaml_value(projects_yaml, 'http_proxy') if http_proxy: - pip_install('mysql-python', proxy=http_proxy, venv=True) + pip_install('mysql-python', proxy=http_proxy, + venv=git_pip_venv_dir(projects_yaml)) else: - pip_install('mysql-python', venv=True) + pip_install('mysql-python', + venv=git_pip_venv_dir(projects_yaml)) src_etc = os.path.join(git_src_dir(projects_yaml, 'cinder'), 'etc/cinder') configs = { @@ -590,7 +592,8 @@ def git_post_install(projects_yaml): shutil.copytree(configs['src'], configs['dest']) symlinks = [ - {'src': os.path.join(pip_get_virtualenv_path(), 'bin/cinder-manage'), + {'src': os.path.join(git_pip_venv_dir(projects_yaml), + 'bin/cinder-manage'), 'link': '/usr/local/bin/cinder-manage'}, ] @@ -610,7 +613,7 @@ def git_post_install(projects_yaml): os.chmod('/etc/sudoers.d', 0o750) - bin_dir = os.path.join(pip_get_virtualenv_path(), 'bin') + bin_dir = os.path.join(git_pip_venv_dir(projects_yaml), 'bin') cinder_api_context = { 'service_description': 'Cinder API server', 'service_name': 'Cinder', From bb411ad6d3930a43acc7ce358953fbc39c10bf7e Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 15:24:24 +0000 Subject: [PATCH 27/38] Unit test update --- unit_tests/test_cinder_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/test_cinder_utils.py b/unit_tests/test_cinder_utils.py index 0dc0f7ff..e2e025b2 100644 --- a/unit_tests/test_cinder_utils.py +++ b/unit_tests/test_cinder_utils.py @@ -588,7 +588,7 @@ class TestCinderUtils(CharmTestCase): projects_yaml = openstack_origin_git join.return_value = 'joined-string' cinder_utils.git_post_install(projects_yaml) - pip_install.assert_called_with('mysql-python', venv=True) + pip_install('mysql-python', venv='joined-string') expected = [ call('joined-string', '/etc/cinder'), ] From ff140f9ae76180b86b5f5ce96f7e5e23a0cfc857 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 16:37:48 +0000 Subject: [PATCH 28/38] Add rootwrap symlink --- hooks/cinder_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index c6450aa5..c8d832fc 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -595,6 +595,9 @@ def git_post_install(projects_yaml): {'src': os.path.join(git_pip_venv_dir(projects_yaml), 'bin/cinder-manage'), 'link': '/usr/local/bin/cinder-manage'}, + {'src': os.path.join(git_pip_venv_dir(projects_yaml), + 'bin/cinder-rootwrap'), + 'link': '/usr/local/bin/cinder-rootwrap'}, ] for s in symlinks: From d5cfa23dac1f1f94421f8e155aecc495a1ce3fbe Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 18:53:45 +0000 Subject: [PATCH 29/38] Sync charm-helpers --- hooks/charmhelpers/contrib/python/packages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 88c8887d..07b0c1d7 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -115,4 +115,5 @@ def pip_create_virtualenv(path=None): else: venv_path = os.path.join(charm_dir(), 'venv') - subprocess.check_call(['virtualenv', venv_path]) + if not os.path.exists(venv_path): + subprocess.check_call(['virtualenv', venv_path]) From e5b49094153c6f681245baeaac65c04f4cdd9700 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 11 May 2015 12:32:17 +0000 Subject: [PATCH 30/38] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index c5e08ec2..774eaba1 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -501,6 +501,16 @@ def git_install_requested(): requirements_dir = None +def _git_yaml_load(projects_yaml): + """ + Load the specified yaml into a dictionary. + """ + if not projects_yaml: + return None + + projects = yaml.load(projects_yaml) + + def git_clone_and_install(projects_yaml, core_project, depth=1): """ Clone/install all specified OpenStack repositories. @@ -523,10 +533,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): parent_dir = '/mnt/openstack-git' http_proxy = None - if not projects_yaml: - return - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) _git_validate_projects_yaml(projects, core_project) old_environ = dict(os.environ) @@ -644,10 +651,7 @@ def git_pip_venv_dir(projects_yaml): """ parent_dir = '/mnt/openstack-git' - if not projects_yaml: - return - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) if 'directory' in projects.keys(): parent_dir = projects['directory'] @@ -661,10 +665,7 @@ def git_src_dir(projects_yaml, project): """ parent_dir = '/mnt/openstack-git' - if not projects_yaml: - return - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) if 'directory' in projects.keys(): parent_dir = projects['directory'] @@ -680,10 +681,7 @@ def git_yaml_value(projects_yaml, key): """ Return the value in projects_yaml for the specified key. """ - if not projects_yaml: - return None - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) if key in projects.keys(): return projects[key] From e5e3000bc60dfe71afac8b73c31b52ec6f8f93f5 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 11 May 2015 12:35:05 +0000 Subject: [PATCH 31/38] Add comment to fix bin symlinks --- hooks/cinder_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index c8d832fc..ef34cb71 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -591,6 +591,7 @@ def git_post_install(projects_yaml): shutil.rmtree(configs['dest']) shutil.copytree(configs['src'], configs['dest']) + # NOTE(coreycb): Need to find better solution than bin symlinks. symlinks = [ {'src': os.path.join(git_pip_venv_dir(projects_yaml), 'bin/cinder-manage'), From b0f5acef905385793b5bd4803540c77c8135f783 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 11 May 2015 12:45:25 +0000 Subject: [PATCH 32/38] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 774eaba1..8bd2356e 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -508,7 +508,7 @@ def _git_yaml_load(projects_yaml): if not projects_yaml: return None - projects = yaml.load(projects_yaml) + return yaml.load(projects_yaml) def git_clone_and_install(projects_yaml, core_project, depth=1): From 6a71be87f05d2bfb24bb82034b56f926accd33ef Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 12 May 2015 14:23:55 +0000 Subject: [PATCH 33/38] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 8bd2356e..d795a358 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -524,8 +524,8 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): repository: 'git://git.openstack.org/openstack/requirements.git', branch: 'stable/icehouse'} directory: /mnt/openstack-git - http_proxy: http://squid.internal:3128 - https_proxy: https://squid.internal:3128 + http_proxy: squid-proxy-url + https_proxy: squid-proxy-url The directory, http_proxy, and https_proxy keys are optional. """ From 1afd56b5b0de3d0f0f12284b49c8fd9987fb55b6 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 12 May 2015 14:49:27 +0000 Subject: [PATCH 34/38] Clone from github in deploy from source amulet tests --- tests/basic_deployment.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index e1afa7fc..0bfcbeb0 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -80,11 +80,10 @@ class CinderBasicDeployment(OpenStackAmuletDeployment): openstack_origin_git = { 'repositories': [ {'name': 'requirements', - 'repository': - 'git://git.openstack.org/openstack/requirements', + 'repository': 'git://github.com/openstack/requirements', 'branch': branch}, {'name': 'cinder', - 'repository': 'git://git.openstack.org/openstack/cinder', + 'repository': 'git://github.com/openstack/cinder', 'branch': branch}, ], 'directory': '/mnt/openstack-git', From bb5e5a035e95ab145a78f039e44587994010f11d Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 12 May 2015 19:26:39 +0000 Subject: [PATCH 35/38] Install python-cephclient as part of base deploy from source packages --- hooks/cinder_utils.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index ef34cb71..614b0c7f 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -574,12 +574,14 @@ def git_pre_install(): def git_post_install(projects_yaml): """Perform cinder post-install setup.""" http_proxy = git_yaml_value(projects_yaml, 'http_proxy') - if http_proxy: - pip_install('mysql-python', proxy=http_proxy, - venv=git_pip_venv_dir(projects_yaml)) - else: - pip_install('mysql-python', - venv=git_pip_venv_dir(projects_yaml)) + base_packages = ['mysql-python', 'python-cephclient'] + for pkg in base_packages: + if http_proxy: + pip_install(pkg, proxy=http_proxy, + venv=git_pip_venv_dir(projects_yaml)) + else: + pip_install(pkg, + venv=git_pip_venv_dir(projects_yaml)) src_etc = os.path.join(git_src_dir(projects_yaml, 'cinder'), 'etc/cinder') configs = { From 48a98551752987673b2aad9ed1afca9f3fa0336e Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 12 May 2015 19:47:49 +0000 Subject: [PATCH 36/38] Add libyaml-dev as base git package --- hooks/cinder_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index 614b0c7f..8905c7f2 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -105,6 +105,7 @@ BASE_GIT_PACKAGES = [ 'libssl-dev', 'libxml2-dev', 'libxslt1-dev', + 'libyaml-dev', 'lvm2', 'python-dev', 'python-pip', From 222a0e66f732ea0331c566d1f6b4d6abb990cc04 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 12 May 2015 20:43:48 +0000 Subject: [PATCH 37/38] Add rbd.py and rados.py symlinks and drop python-cephclient --- hooks/cinder_utils.py | 10 +++++++++- unit_tests/test_cinder_utils.py | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index 8905c7f2..4550dddb 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -575,7 +575,7 @@ def git_pre_install(): def git_post_install(projects_yaml): """Perform cinder post-install setup.""" http_proxy = git_yaml_value(projects_yaml, 'http_proxy') - base_packages = ['mysql-python', 'python-cephclient'] + base_packages = ['mysql-python'] for pkg in base_packages: if http_proxy: pip_install(pkg, proxy=http_proxy, @@ -602,6 +602,14 @@ def git_post_install(projects_yaml): {'src': os.path.join(git_pip_venv_dir(projects_yaml), 'bin/cinder-rootwrap'), 'link': '/usr/local/bin/cinder-rootwrap'}, + # NOTE(coreycb): This is ugly but couldn't find pypi package that + # installs rbd.py and rados.py. + {'src': '/usr/lib/python2.7/dist-packages/rbd.py', + 'link': os.path.join(git_pip_venv_dir(projects_yaml), + 'lib/python2.7/site-packages/rbd.py')}, + {'src': '/usr/lib/python2.7/dist-packages/rados.py', + 'link': os.path.join(git_pip_venv_dir(projects_yaml), + 'lib/python2.7/site-packages/rados.py')}, ] for s in symlinks: diff --git a/unit_tests/test_cinder_utils.py b/unit_tests/test_cinder_utils.py index e2e025b2..761e4ab7 100644 --- a/unit_tests/test_cinder_utils.py +++ b/unit_tests/test_cinder_utils.py @@ -596,6 +596,8 @@ class TestCinderUtils(CharmTestCase): expected = [ call('joined-string', '/usr/local/bin/cinder-manage'), + call('/usr/lib/python2.7/dist-packages/rbd.py', 'joined-string'), + call('/usr/lib/python2.7/dist-packages/rados.py', 'joined-string'), ] symlink.assert_has_calls(expected, any_order=True) From 94ea1cd393257e4ad480e95103bcae4512386a9f Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 27 May 2015 13:01:09 +0000 Subject: [PATCH 38/38] Sync charm-helpers --- hooks/charmhelpers/fetch/giturl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 7d842d49..ddc25b7e 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -45,7 +45,7 @@ class GitUrlFetchHandler(BaseFetchHandler): else: return True - def clone(self, source, dest, branch, depth): + def clone(self, source, dest, branch, depth=None): if not self.can_handle(source): raise UnhandledSource("Cannot handle {}".format(source))