From 1e14d335318598c3ebeabca292b32bcf446404f4 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 22 Apr 2015 18:56:00 +0000 Subject: [PATCH 01/31] Sync charm-helpers --- .../contrib/openstack/amulet/deployment.py | 21 +++++++++++++------ hooks/charmhelpers/contrib/openstack/utils.py | 12 +++++++++-- hooks/charmhelpers/contrib/python/packages.py | 7 +++++-- tests/charmhelpers/contrib/amulet/utils.py | 9 +++++++- .../contrib/openstack/amulet/deployment.py | 21 +++++++++++++------ 5 files changed, 53 insertions(+), 17 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 11d49a7c..461a702f 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -46,15 +46,22 @@ class OpenStackAmuletDeployment(AmuletDeployment): stable or next branches for the other_services.""" base_charms = ['mysql', 'mongodb'] + if self.series in ['precise', 'trusty']: + base_series = self.series + else: + base_series = self.current_next + if self.stable: for svc in other_services: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: for svc in other_services: if svc['name'] in base_charms: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: temp = 'lp:~openstack-charmers/charms/{}/{}/next' svc['location'] = temp.format(self.current_next, @@ -99,10 +106,12 @@ class OpenStackAmuletDeployment(AmuletDeployment): Return an integer representing the enum value of the openstack release. """ + # Must be ordered by OpenStack release (not by Ubuntu release): (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse, self.trusty_juno, self.trusty_kilo, - self.utopic_juno, self.vivid_kilo) = range(10) + self.trusty_icehouse, self.trusty_juno, self.utopic_juno, + self.trusty_kilo, self.vivid_kilo) = range(10) + releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index f90a0289..f4824f99 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -517,6 +517,7 @@ def git_clone_and_install(projects_yaml, core_project): """ global requirements_dir parent_dir = '/mnt/openstack-git' + http_proxy = None if not projects_yaml: return @@ -527,6 +528,7 @@ def git_clone_and_install(projects_yaml, core_project): old_environ = dict(os.environ) if 'http_proxy' in projects.keys(): + http_proxy = projects['http_proxy'] os.environ['http_proxy'] = projects['http_proxy'] if 'https_proxy' in projects.keys(): os.environ['https_proxy'] = projects['https_proxy'] @@ -539,10 +541,12 @@ def git_clone_and_install(projects_yaml, core_project): branch = p['branch'] if p['name'] == 'requirements': repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + http_proxy, update_requirements=False) requirements_dir = repo_dir else: repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + http_proxy, update_requirements=True) os.environ = old_environ @@ -574,7 +578,8 @@ def _git_ensure_key_exists(key, keys): error_out('openstack-origin-git key \'{}\' is missing'.format(key)) -def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements): +def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, + update_requirements): """ Clone and install a single git repository. """ @@ -598,7 +603,10 @@ def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements) _git_update_requirements(repo_dir, requirements_dir) juju_log('Installing git repo from dir: {}'.format(repo_dir)) - pip_install(repo_dir) + if http_proxy: + pip_install(repo_dir, ignore=True, proxy=http_proxy) + else: + pip_install(repo_dir, ignore=True) return repo_dir diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 8659516b..ceaa5971 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -51,17 +51,20 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, **options): +def pip_install(package, fatal=False, upgrade=False, ignore=False, **options): """Install a python package""" command = ["install"] - available_options = ('proxy', 'src', 'log', "index-url", ) + available_options = ('proxy', 'src', 'log', 'index-url', ) for option in parse_options(options, available_options): command.append(option) if upgrade: command.append('--upgrade') + if ignore: + command.append('--ignore-installed') + if isinstance(package, list): command.extend(package) else: diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 5088b1d1..f61c2e8b 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -79,6 +79,9 @@ class AmuletUtils(object): for k, v in six.iteritems(commands): for cmd in v: output, code = k.run(cmd) + self.log.debug('{} `{}` returned ' + '{}'.format(k.info['unit_name'], + cmd, code)) if code != 0: return "command `{}` returned {}".format(cmd, str(code)) return None @@ -86,7 +89,11 @@ class AmuletUtils(object): def _get_config(self, unit, filename): """Get a ConfigParser object for parsing a unit's config file.""" file_contents = unit.file_contents(filename) - config = ConfigParser.ConfigParser() + + # NOTE(beisner): by default, ConfigParser does not handle options + # with no value, such as the flags used in the mysql my.cnf file. + # https://bugs.python.org/issue7005 + config = ConfigParser.ConfigParser(allow_no_value=True) config.readfp(io.StringIO(file_contents)) return config diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 11d49a7c..461a702f 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -46,15 +46,22 @@ class OpenStackAmuletDeployment(AmuletDeployment): stable or next branches for the other_services.""" base_charms = ['mysql', 'mongodb'] + if self.series in ['precise', 'trusty']: + base_series = self.series + else: + base_series = self.current_next + if self.stable: for svc in other_services: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: for svc in other_services: if svc['name'] in base_charms: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: temp = 'lp:~openstack-charmers/charms/{}/{}/next' svc['location'] = temp.format(self.current_next, @@ -99,10 +106,12 @@ class OpenStackAmuletDeployment(AmuletDeployment): Return an integer representing the enum value of the openstack release. """ + # Must be ordered by OpenStack release (not by Ubuntu release): (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse, self.trusty_juno, self.trusty_kilo, - self.utopic_juno, self.vivid_kilo) = range(10) + self.trusty_icehouse, self.trusty_juno, self.utopic_juno, + self.trusty_kilo, self.vivid_kilo) = range(10) + releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, From c2ac30472c0d420a3230c5cd50c19e26aa08eebc Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 22 Apr 2015 19:34:46 +0000 Subject: [PATCH 02/31] Add libffi-dev, libssl-dev, and libyaml-dev to base git install packages --- hooks/quantum_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 3be8aa2b..1158d485 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -186,8 +186,11 @@ L3HA_PACKAGES = ['keepalived'] BASE_GIT_PACKAGES = [ 'dnsmasq', + 'libffi-dev', + 'libssl-dev', 'libxml2-dev', 'libxslt1-dev', + 'libyaml-dev', 'python-dev', 'python-pip', 'python-setuptools', From 516e0bf50f4d934a9e80f69bbb8a0e21e8106fa0 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 11:55:32 +0000 Subject: [PATCH 03/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- hooks/charmhelpers/contrib/python/packages.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index f4824f99..0460e076 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -604,9 +604,9 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, ignore=True, proxy=http_proxy) + pip_install(repo_dir, proxy=http_proxy) else: - pip_install(repo_dir, ignore=True) + pip_install(repo_dir) return repo_dir diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index ceaa5971..3f1fb09d 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -51,7 +51,7 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, ignore=False, **options): +def pip_install(package, fatal=False, upgrade=False, **options): """Install a python package""" command = ["install"] @@ -62,9 +62,6 @@ def pip_install(package, fatal=False, upgrade=False, ignore=False, **options): if upgrade: command.append('--upgrade') - if ignore: - command.append('--ignore-installed') - if isinstance(package, list): command.extend(package) else: From 3322f5fc64474286470464ae60c72c80305b70b6 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 13:01:34 +0000 Subject: [PATCH 04/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 0460e076..86bf91f5 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -604,9 +604,9 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, proxy=http_proxy) + pip_install(repo_dir, upgrade=True, proxy=http_proxy) else: - pip_install(repo_dir) + pip_install(repo_dir, upgrade=True) return repo_dir From d3a6bfaf7dc2a72fc19c16dfd49ffebcae8e01bd Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 14:43:27 +0000 Subject: [PATCH 05/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 86bf91f5..0460e076 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -604,9 +604,9 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, upgrade=True, proxy=http_proxy) + pip_install(repo_dir, proxy=http_proxy) else: - pip_install(repo_dir, upgrade=True) + pip_install(repo_dir) return repo_dir From b805fc39fb23af677ca5e0b61b39c8ae4f6824b0 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 16:07:05 +0000 Subject: [PATCH 06/31] Drop added packages --- hooks/quantum_utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 1158d485..3be8aa2b 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -186,11 +186,8 @@ L3HA_PACKAGES = ['keepalived'] BASE_GIT_PACKAGES = [ 'dnsmasq', - 'libffi-dev', - 'libssl-dev', 'libxml2-dev', 'libxslt1-dev', - 'libyaml-dev', 'python-dev', 'python-pip', 'python-setuptools', From d6adb84c9c3af2b6250a574c55858b40271281c5 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 28 Apr 2015 19:01:28 +0000 Subject: [PATCH 07/31] Sync charm-helpers --- hooks/charmhelpers/fetch/__init__.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 792e629a..57a179b8 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -367,13 +367,15 @@ def install_remote(source, *args, **kwargs): # explaining why it can't handle a given source. handlers = [h for h in plugins() if h.can_handle(source) is True] installed_to = None - for handler in handlers: - try: - installed_to = handler.install(source, *args, **kwargs) - except UnhandledSource: - pass - if not installed_to: - raise UnhandledSource("No handler found for source {}".format(source)) + while not installed_to: + for handler in handlers: + try: + installed_to = handler.install(source, *args, **kwargs) + except UnhandledSource: + pass + log("CB: install loop - installed_to = |%s|".format(installed_to)) + #if not installed_to: + # raise UnhandledSource("No handler found for source {}".format(source)) return installed_to From 78205534a4e65dab9bbf01986f120f91d676061b Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 1 May 2015 14:57:04 +0000 Subject: [PATCH 08/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 15 ++++++++------- hooks/charmhelpers/fetch/__init__.py | 16 +++++++--------- hooks/charmhelpers/fetch/giturl.py | 11 +++++++---- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 0460e076..b47ed937 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -497,7 +497,7 @@ def git_install_requested(): requirements_dir = None -def git_clone_and_install(projects_yaml, core_project): +def git_clone_and_install(projects_yaml, core_project, depth=1): """ Clone/install all specified OpenStack repositories. @@ -540,13 +540,13 @@ def git_clone_and_install(projects_yaml, core_project): repo = p['repository'] branch = p['branch'] if p['name'] == 'requirements': - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, - http_proxy, + repo_dir = _git_clone_and_install_single(repo, branch, depth, + parent_dir, http_proxy, update_requirements=False) requirements_dir = repo_dir else: - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, - http_proxy, + repo_dir = _git_clone_and_install_single(repo, branch, depth, + parent_dir, http_proxy, update_requirements=True) os.environ = old_environ @@ -578,7 +578,7 @@ def _git_ensure_key_exists(key, keys): error_out('openstack-origin-git key \'{}\' is missing'.format(key)) -def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, +def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, update_requirements): """ Clone and install a single git repository. @@ -592,7 +592,8 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, if not os.path.exists(dest_dir): juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) - repo_dir = install_remote(repo, dest=parent_dir, branch=branch) + repo_dir = install_remote(repo, dest=parent_dir, branch=branch, + depth=depth) else: repo_dir = dest_dir diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 57a179b8..792e629a 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -367,15 +367,13 @@ def install_remote(source, *args, **kwargs): # explaining why it can't handle a given source. handlers = [h for h in plugins() if h.can_handle(source) is True] installed_to = None - while not installed_to: - for handler in handlers: - try: - installed_to = handler.install(source, *args, **kwargs) - except UnhandledSource: - pass - log("CB: install loop - installed_to = |%s|".format(installed_to)) - #if not installed_to: - # raise UnhandledSource("No handler found for source {}".format(source)) + for handler in handlers: + try: + installed_to = handler.install(source, *args, **kwargs) + except UnhandledSource: + pass + if not installed_to: + raise UnhandledSource("No handler found for source {}".format(source)) return installed_to diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 93aae87b..06b0e767 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -45,14 +45,17 @@ class GitUrlFetchHandler(BaseFetchHandler): else: return True - def clone(self, source, dest, branch): + def clone(self, source, dest, branch, depth): if not self.can_handle(source): raise UnhandledSource("Cannot handle {}".format(source)) - repo = Repo.clone_from(source, dest) + if depth: + repo = Repo.clone_from(source, dest, depth) + else: + repo = Repo.clone_from(source, dest) repo.git.checkout(branch) - def install(self, source, branch="master", dest=None): + def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) branch_name = url_parts.path.strip("/").split("/")[-1] if dest: @@ -63,7 +66,7 @@ class GitUrlFetchHandler(BaseFetchHandler): if not os.path.exists(dest_dir): mkdir(dest_dir, perms=0o755) try: - self.clone(source, dest_dir, branch) + self.clone(source, dest_dir, branch, depth) except GitCommandError as e: raise UnhandledSource(e.message) except OSError as e: From 367ee4bddc739483a2e65d914bdd4a5fa42db8a0 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 2 May 2015 18:35:53 +0000 Subject: [PATCH 09/31] Sync charm-helpers --- hooks/charmhelpers/fetch/giturl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 06b0e767..742cf319 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -50,7 +50,7 @@ class GitUrlFetchHandler(BaseFetchHandler): raise UnhandledSource("Cannot handle {}".format(source)) if depth: - repo = Repo.clone_from(source, dest, depth) + repo = Repo.clone_from(source, dest, depth=depth) else: repo = Repo.clone_from(source, dest) repo.git.checkout(branch) From d37c7649d7b6e895ff7d074830111992d5d1f07b Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 2 May 2015 20:21:47 +0000 Subject: [PATCH 10/31] Sync charm-helpers --- hooks/charmhelpers/fetch/giturl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 742cf319..4e05c08a 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -50,10 +50,9 @@ class GitUrlFetchHandler(BaseFetchHandler): raise UnhandledSource("Cannot handle {}".format(source)) if depth: - repo = Repo.clone_from(source, dest, depth=depth) + repo = Repo.clone_from(source, dest, branch=branch, depth=depth) else: - repo = Repo.clone_from(source, dest) - repo.git.checkout(branch) + repo = Repo.clone_from(source, dest, branch=branch) def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) From 5272249b4d45b1af0c4cc83a9914bdc6cb4cff31 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 5 May 2015 19:18:25 +0000 Subject: [PATCH 11/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 8 +++++++- hooks/charmhelpers/contrib/python/packages.py | 19 ++++++++++++++++++- hooks/charmhelpers/fetch/giturl.py | 4 ++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index b47ed937..ce8c8213 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -53,9 +53,13 @@ from charmhelpers.contrib.network.ip import ( get_ipv6_addr ) +from charmhelpers.contrib.python.packages import ( + pip_create_virtualenv, + pip_install, +) + from charmhelpers.core.host import lsb_release, mounts, umount from charmhelpers.fetch import apt_install, apt_cache, install_remote -from charmhelpers.contrib.python.packages import pip_install from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device @@ -536,6 +540,8 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): if 'directory' in projects.keys(): parent_dir = projects['directory'] + pip_create_virtualenv(proxy=http_proxy) + for p in projects['repositories']: repo = p['repository'] branch = p['branch'] diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 3f1fb09d..e62240b4 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -17,8 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . +import os +import subprocess + from charmhelpers.fetch import apt_install, apt_update -from charmhelpers.core.hookenv import log +from charmhelpers.core.hookenv import charm_dir, log try: from pip import main as pip_execute @@ -94,3 +97,17 @@ def pip_list(): """Returns the list of current python installed packages """ return pip_execute(["list"]) + + +def pip_create_virtualenv(parent_dir=charm_dir(), proxy=None): + """Create and activate an isolated Python environment (virtualenv).""" + if proxy: + pip_install('virtualenv', proxy=proxy) + else: + pip_install('virtualenv') + + venv_dir = os.path.join(parent_dir, 'venv') + subprocess.check_call(['virtualenv', '--no-site-packages', venv_dir]) + + venv_activate = os.path.join(venv_dir, 'bin/activate') + subprocess.check_call(['.', venv_activate], shell=True) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 4e05c08a..7d842d49 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -50,9 +50,9 @@ class GitUrlFetchHandler(BaseFetchHandler): raise UnhandledSource("Cannot handle {}".format(source)) if depth: - repo = Repo.clone_from(source, dest, branch=branch, depth=depth) + Repo.clone_from(source, dest, branch=branch, depth=depth) else: - repo = Repo.clone_from(source, dest, branch=branch) + Repo.clone_from(source, dest, branch=branch) def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) From ccdffa98266d142294d14fad23e20ec2b4adc04f Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 5 May 2015 19:50:11 +0000 Subject: [PATCH 12/31] Point upstart scripts at venv binaries --- hooks/quantum_utils.py | 32 +++++++++++-------- templates/git/upstart/neutron-agent.upstart | 2 +- .../git/upstart/neutron-ovs-cleanup.upstart | 2 +- templates/git/upstart/neutron-server.upstart | 2 +- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 3be8aa2b..6cad7033 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -898,15 +898,18 @@ def git_post_install(projects_yaml): service_name = 'quantum-gateway' user_name = 'neutron' + bin_dir = os.path.join(charm_dir(), 'venv/bin') neutron_api_context = { 'service_description': 'Neutron API server', 'service_name': service_name, 'process_name': 'neutron-server', + 'executable_name': os.path.join(bin_dir, 'neutron-server'), } neutron_dhcp_agent_context = { 'service_description': 'Neutron DHCP Agent', 'service_name': service_name, 'process_name': 'neutron-dhcp-agent', + 'executable_name': os.path.join(bin_dir, 'neutron-dhcp-agent'), 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/dhcp_agent.ini'], 'log_file': '/var/log/neutron/dhcp-agent.log', @@ -915,6 +918,7 @@ def git_post_install(projects_yaml): 'service_description': 'Neutron L3 Agent', 'service_name': service_name, 'process_name': 'neutron-l3-agent', + 'executable_name': os.path.join(bin_dir, 'neutron-l3-agent'), 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/l3_agent.ini', '/etc/neutron/fwaas_driver.ini'], @@ -926,7 +930,7 @@ def git_post_install(projects_yaml): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-lbaas-agent', - 'executable_name': '/usr/local/bin/neutron-lbaas-agent', + 'executable_name': os.path.join(bin_dir, 'neutron-lbaas-agent'), 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/lbaas_agent.ini'], 'log_file': '/var/log/neutron/lbaas-agent.log', @@ -937,7 +941,7 @@ def git_post_install(projects_yaml): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-metadata-agent', - 'executable_name': '/usr/local/bin/neutron-metadata-agent', + 'executable_name': os.path.join(bin_dir, 'neutron-metadata-agent'), 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/metadata_agent.ini'], 'log_file': '/var/log/neutron/metadata-agent.log', @@ -948,7 +952,7 @@ def git_post_install(projects_yaml): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-metering-agent', - 'executable_name': '/usr/local/bin/neutron-metering-agent', + 'executable_name': os.path.join(bin_dir, 'neutron-metering-agent'), 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/metering_agent.ini'], 'log_file': '/var/log/neutron/metering-agent.log', @@ -957,6 +961,7 @@ def git_post_install(projects_yaml): 'service_description': 'Neutron OVS cleanup', 'service_name': service_name, 'process_name': 'neutron-ovs-cleanup', + 'executable_name': os.path.join(bin_dir, 'neutron-ovs-cleanup'), 'config_file': '/etc/neutron/neutron.conf', 'log_file': '/var/log/neutron/ovs-cleanup.log', } @@ -966,7 +971,7 @@ def git_post_install(projects_yaml): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-restproxy-agent', - 'executable_name': '/usr/local/bin/neutron-restproxy-agent', + 'executable_name': os.path.join(bin_dir, 'neutron-restproxy-agent'), 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/bigswitch/restproxy.ini'], 'log_file': '/var/log/neutron/bigswitch-agent.log', @@ -977,7 +982,7 @@ def git_post_install(projects_yaml): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-ibm-agent', - 'executable_name': '/usr/local/bin/neutron-ibm-agent', + 'executable_name': os.path.join(bin_dir, 'neutron-ibm-agent'), 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/ibm/sdnve_neutron_plugin.ini'], 'log_file': '/var/log/neutron/ibm-agent.log', @@ -988,7 +993,7 @@ def git_post_install(projects_yaml): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-linuxbridge-agent', - 'executable_name': '/usr/local/bin/neutron-linuxbridge-agent', + 'executable_name': os.path.join(bin_dir, 'neutron-linuxbridge-agent'), 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/ml2/ml2_conf.ini'], 'log_file': '/var/log/neutron/linuxbridge-agent.log', @@ -999,7 +1004,7 @@ def git_post_install(projects_yaml): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-mlnx-agent', - 'executable_name': '/usr/local/bin/neutron-mlnx-agent', + 'executable_name': os.path.join(bin_dir, 'neutron-mlnx-agent'), 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/mlnx/mlnx_conf.ini'], 'log_file': '/var/log/neutron/mlnx-agent.log', @@ -1009,7 +1014,7 @@ def git_post_install(projects_yaml): 'service_name': service_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-nec-agent', - 'executable_name': '/usr/local/bin/neutron-nec-agent', + 'executable_name': os.path.join(bin_dir, 'neutron-nec-agent'), 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/nec/nec.ini'], 'log_file': '/var/log/neutron/nec-agent.log', @@ -1020,7 +1025,7 @@ def git_post_install(projects_yaml): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-nvsd-agent', - 'executable_name': '/usr/local/bin/neutron-nvsd-agent', + 'executable_name': os.path.join(bin_dir, 'neutron-nvsd-agent'), 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/oneconvergence/nvsdplugin.ini'], 'log_file': '/var/log/neutron/nvsd-agent.log', @@ -1031,7 +1036,7 @@ def git_post_install(projects_yaml): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-ofagent-agent', - 'executable_name': '/usr/local/bin/neutron-ofagent-agent', + 'executable_name': os.path.join(bin_dir, 'neutron-ofagent-agent'), 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/ml2/ml2_conf_ofa.ini'], 'log_file': '/var/log/neutron/openflow-agent.log', @@ -1042,7 +1047,7 @@ def git_post_install(projects_yaml): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-openvswitch-agent', - 'executable_name': '/usr/local/bin/neutron-openvswitch-agent', + 'executable_name': os.path.join(bin_dir, 'neutron-openvswitch-agent'), 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/ml2/ml2_conf.ini'], 'log_file': '/var/log/neutron/openvswitch-agent.log', @@ -1053,7 +1058,7 @@ def git_post_install(projects_yaml): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-ryu-agent', - 'executable_name': '/usr/local/bin/neutron-ryu-agent', + 'executable_name': os.path.join(bin_dir, 'neutron-ryu-agent'), 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/ryu/ryu.ini'], 'log_file': '/var/log/neutron/ryu-agent.log', @@ -1064,7 +1069,7 @@ def git_post_install(projects_yaml): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-sriov-nic-agent', - 'executable_name': '/usr/local/bin/neutron-sriov-nic-agent', + 'executable_name': os.path.join(bin_dir, 'neutron-sriov-nic-agent'), 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/ml2/ml2_conf_sriov'], 'log_file': '/var/log/neutron/sriov-agent.log', @@ -1073,6 +1078,7 @@ def git_post_install(projects_yaml): 'service_description': 'Neutron VPN Agent', 'service_name': service_name, 'process_name': 'neutron-vpn-agent', + 'executable_name': os.path.join(bin_dir, 'neutron-vpn-agent'), 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/vpn_agent.ini', '/etc/neutron/l3_agent.ini', diff --git a/templates/git/upstart/neutron-agent.upstart b/templates/git/upstart/neutron-agent.upstart index 3853caa4..b3a4fd72 100644 --- a/templates/git/upstart/neutron-agent.upstart +++ b/templates/git/upstart/neutron-agent.upstart @@ -18,7 +18,7 @@ pre-start script fi end script -exec start-stop-daemon --start --chuid neutron --exec /usr/local/bin/{{ process_name }} -- \ +exec start-stop-daemon --start --chuid neutron --exec {{ executable_name }} -- \ {% for config_file in config_files -%} --config-file={{ config_file }} \ {% endfor -%} diff --git a/templates/git/upstart/neutron-ovs-cleanup.upstart b/templates/git/upstart/neutron-ovs-cleanup.upstart index fa2c2149..2bb07444 100644 --- a/templates/git/upstart/neutron-ovs-cleanup.upstart +++ b/templates/git/upstart/neutron-ovs-cleanup.upstart @@ -7,7 +7,7 @@ stop on runlevel [!2345] pre-start script [ ! -x /usr/local/bin/{{ process_name }} ] && exit 0 start-stop-daemon --start --chuid neutron \ - --exec /usr/local/bin/{{ process_name }} -- \ + --exec {{ executable_name }} -- \ --log-file {{ log_file }} \ --config-file {{ config_file }} --verbose end script diff --git a/templates/git/upstart/neutron-server.upstart b/templates/git/upstart/neutron-server.upstart index 2d2c73a2..c19288c4 100644 --- a/templates/git/upstart/neutron-server.upstart +++ b/templates/git/upstart/neutron-server.upstart @@ -16,7 +16,7 @@ end script script [ -r /etc/default/{{ process_name }} ] && . /etc/default/{{ process_name }} [ -r "$NEUTRON_PLUGIN_CONFIG" ] && CONF_ARG="--config-file $NEUTRON_PLUGIN_CONFIG" - exec start-stop-daemon --start --chuid neutron --exec /usr/local/bin/neutron-server -- \ + exec start-stop-daemon --start --chuid neutron --exec {{ executable_name }} -- \ --config-file /etc/neutron/neutron.conf \ --log-file /var/log/neutron/server.log $CONF_ARG end script From f31d6638841ae330e9481a142f31b99df0d3ddd8 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 5 May 2015 20:00:09 +0000 Subject: [PATCH 13/31] Update unit tests --- unit_tests/test_quantum_utils.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/unit_tests/test_quantum_utils.py b/unit_tests/test_quantum_utils.py index 9d7615a3..dbc25402 100644 --- a/unit_tests/test_quantum_utils.py +++ b/unit_tests/test_quantum_utils.py @@ -808,11 +808,13 @@ class TestQuantumAgentReallocation(CharmTestCase): 'service_description': 'Neutron API server', 'charm_name': 'neutron-api', 'process_name': 'neutron-server', + 'executable_name': 'joined-string', } neutron_dhcp_agent_context = { 'service_description': 'Neutron DHCP Agent', 'service_name': service_name, 'process_name': 'neutron-dhcp-agent', + 'executable_name': 'joined-string', 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/dhcp_agent.ini'], 'log_file': '/var/log/neutron/dhcp-agent.log', @@ -821,6 +823,7 @@ class TestQuantumAgentReallocation(CharmTestCase): 'service_description': 'Neutron L3 Agent', 'service_name': service_name, 'process_name': 'neutron-l3-agent', + 'executable_name': 'joined-string', 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/l3_agent.ini', '/etc/neutron/fwaas_driver.ini'], @@ -832,7 +835,7 @@ class TestQuantumAgentReallocation(CharmTestCase): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-lbaas-agent', - 'executable_name': '/usr/local/bin/neutron-lbaas-agent', + 'executable_name': 'joined-string', 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/lbaas_agent.ini'], 'log_file': '/var/log/neutron/lbaas-agent.log', @@ -843,7 +846,7 @@ class TestQuantumAgentReallocation(CharmTestCase): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-metadata-agent', - 'executable_name': '/usr/local/bin/neutron-metadata-agent', + 'executable_name': 'joined-string', 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/metadata_agent.ini'], 'log_file': '/var/log/neutron/metadata-agent.log', @@ -854,7 +857,7 @@ class TestQuantumAgentReallocation(CharmTestCase): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-metering-agent', - 'executable_name': '/usr/local/bin/neutron-metering-agent', + 'executable_name': 'joined-string', 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/metering_agent.ini'], 'log_file': '/var/log/neutron/metering-agent.log', @@ -863,6 +866,7 @@ class TestQuantumAgentReallocation(CharmTestCase): 'service_description': 'Neutron OVS cleanup', 'service_name': service_name, 'process_name': 'neutron-ovs-cleanup', + 'executable_name': 'joined-string', 'config_file': '/etc/neutron/neutron.conf', 'log_file': '/var/log/neutron/ovs-cleanup.log', } @@ -872,7 +876,7 @@ class TestQuantumAgentReallocation(CharmTestCase): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-restproxy-agent', - 'executable_name': '/usr/local/bin/neutron-restproxy-agent', + 'executable_name': 'joined-string', 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/bigswitch/restproxy.ini'], 'log_file': '/var/log/neutron/bigswitch-agent.log', @@ -883,7 +887,7 @@ class TestQuantumAgentReallocation(CharmTestCase): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-ibm-agent', - 'executable_name': '/usr/local/bin/neutron-ibm-agent', + 'executable_name': 'joined-string', 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/ibm/sdnve_neutron_plugin.ini'], @@ -895,7 +899,7 @@ class TestQuantumAgentReallocation(CharmTestCase): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-linuxbridge-agent', - 'executable_name': '/usr/local/bin/neutron-linuxbridge-agent', + 'executable_name': 'joined-string', 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/ml2/ml2_conf.ini'], 'log_file': '/var/log/neutron/linuxbridge-agent.log', @@ -906,7 +910,7 @@ class TestQuantumAgentReallocation(CharmTestCase): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-mlnx-agent', - 'executable_name': '/usr/local/bin/neutron-mlnx-agent', + 'executable_name': 'joined-string', 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/mlnx/mlnx_conf.ini'], 'log_file': '/var/log/neutron/mlnx-agent.log', @@ -916,7 +920,7 @@ class TestQuantumAgentReallocation(CharmTestCase): 'service_name': service_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-nec-agent', - 'executable_name': '/usr/local/bin/neutron-nec-agent', + 'executable_name': 'joined-string', 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/nec/nec.ini'], 'log_file': '/var/log/neutron/nec-agent.log', @@ -927,7 +931,7 @@ class TestQuantumAgentReallocation(CharmTestCase): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-nvsd-agent', - 'executable_name': '/usr/local/bin/neutron-nvsd-agent', + 'executable_name': 'joined-string', 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/oneconvergence/nvsdplugin.ini'], 'log_file': '/var/log/neutron/nvsd-agent.log', @@ -938,7 +942,7 @@ class TestQuantumAgentReallocation(CharmTestCase): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-ofagent-agent', - 'executable_name': '/usr/local/bin/neutron-ofagent-agent', + 'executable_name': 'joined-string', 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/ml2/ml2_conf_ofa.ini'], 'log_file': '/var/log/neutron/openflow-agent.log', @@ -949,7 +953,7 @@ class TestQuantumAgentReallocation(CharmTestCase): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-openvswitch-agent', - 'executable_name': '/usr/local/bin/neutron-openvswitch-agent', + 'executable_name': 'joined-string', 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/ml2/ml2_conf.ini'], 'log_file': '/var/log/neutron/openvswitch-agent.log', @@ -960,7 +964,7 @@ class TestQuantumAgentReallocation(CharmTestCase): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-ryu-agent', - 'executable_name': '/usr/local/bin/neutron-ryu-agent', + 'executable_name': 'joined-string', 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/ryu/ryu.ini'], 'log_file': '/var/log/neutron/ryu-agent.log', @@ -971,7 +975,7 @@ class TestQuantumAgentReallocation(CharmTestCase): 'user_name': user_name, 'start_dir': '/var/lib/neutron', 'process_name': 'neutron-sriov-nic-agent', - 'executable_name': '/usr/local/bin/neutron-sriov-nic-agent', + 'executable_name': 'joined-string', 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/plugins/ml2/ml2_conf_sriov'], 'log_file': '/var/log/neutron/sriov-agent.log', @@ -980,11 +984,13 @@ class TestQuantumAgentReallocation(CharmTestCase): 'service_description': 'Neutron API server', 'service_name': service_name, 'process_name': 'neutron-server', + 'executable_name': 'joined-string', } neutron_vpn_agent_context = { 'service_description': 'Neutron VPN Agent', 'service_name': service_name, 'process_name': 'neutron-vpn-agent', + 'executable_name': 'joined-string', 'config_files': ['/etc/neutron/neutron.conf', '/etc/neutron/vpn_agent.ini', '/etc/neutron/l3_agent.ini', From 89ba3eade69270c9e91f07c8ae74dd5f917ba550 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 6 May 2015 16:00:35 +0000 Subject: [PATCH 14/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 6 ++-- hooks/charmhelpers/contrib/python/packages.py | 29 +++++++++---------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index ce8c8213..5909b0f5 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -540,7 +540,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): if 'directory' in projects.keys(): parent_dir = projects['directory'] - pip_create_virtualenv(proxy=http_proxy) + pip_create_virtualenv() for p in projects['repositories']: repo = p['repository'] @@ -611,9 +611,9 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, proxy=http_proxy) + pip_install(repo_dir, proxy=http_proxy, venv=True) else: - pip_install(repo_dir) + pip_install(repo_dir, venv=True) return repo_dir diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index e62240b4..740eaa51 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -54,9 +54,13 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, **options): +def pip_install(package, fatal=False, upgrade=False, venv=False, **options): """Install a python package""" - command = ["install"] + if venv: + venv_python = os.path.join(charm_dir(), 'venv/bin/pip') + command = [venv_python, "install"] + else: + command = ["install"] available_options = ('proxy', 'src', 'log', 'index-url', ) for option in parse_options(options, available_options): @@ -72,7 +76,10 @@ def pip_install(package, fatal=False, upgrade=False, **options): log("Installing {} package with options: {}".format(package, command)) - pip_execute(command) + if venv: + subprocess.check_call(command) + else: + pip_execute(command) def pip_uninstall(package, **options): @@ -99,15 +106,7 @@ def pip_list(): return pip_execute(["list"]) -def pip_create_virtualenv(parent_dir=charm_dir(), proxy=None): - """Create and activate an isolated Python environment (virtualenv).""" - if proxy: - pip_install('virtualenv', proxy=proxy) - else: - pip_install('virtualenv') - - venv_dir = os.path.join(parent_dir, 'venv') - subprocess.check_call(['virtualenv', '--no-site-packages', venv_dir]) - - venv_activate = os.path.join(venv_dir, 'bin/activate') - subprocess.check_call(['.', venv_activate], shell=True) +def pip_create_virtualenv(): + """Create an isolated Python environment.""" + apt_install('python-virtualenv') + subprocess.check_call(['virtualenv', os.path.join(charm_dir(), 'venv')]) From 7d332e2187c9c4d40f7ca46190b0efeff2987911 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 12:44:35 +0000 Subject: [PATCH 15/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 17 +++++++++++++++- hooks/charmhelpers/contrib/python/packages.py | 20 ++++++++++++++++--- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 5909b0f5..29ac7b78 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -540,7 +540,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): if 'directory' in projects.keys(): parent_dir = projects['directory'] - pip_create_virtualenv() + pip_create_virtualenv(os.path.join(parent_dir, 'venv')) for p in projects['repositories']: repo = p['repository'] @@ -655,3 +655,18 @@ def git_src_dir(projects_yaml, project): return os.path.join(parent_dir, os.path.basename(p['repository'])) return None + + +def git_yaml_value(projects_yaml, key): + """ + Return the value in projects_yaml for the specified key. + """ + if not projects_yaml: + return None + + projects = yaml.load(projects_yaml) + + if key in projects.keys(): + return projects[key] + + return None diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 740eaa51..12838d24 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -30,6 +30,8 @@ except ImportError: apt_install('python-pip') from pip import main as pip_execute +pip_venv_path = None + __author__ = "Jorge Niedbalski " @@ -57,7 +59,7 @@ def pip_install_requirements(requirements, **options): def pip_install(package, fatal=False, upgrade=False, venv=False, **options): """Install a python package""" if venv: - venv_python = os.path.join(charm_dir(), 'venv/bin/pip') + venv_python = os.path.join(pip_venv_path, 'bin/pip') command = [venv_python, "install"] else: command = ["install"] @@ -106,7 +108,19 @@ def pip_list(): return pip_execute(["list"]) -def pip_create_virtualenv(): +def pip_create_virtualenv(path=None): """Create an isolated Python environment.""" + global pip_venv_path + apt_install('python-virtualenv') - subprocess.check_call(['virtualenv', os.path.join(charm_dir(), 'venv')]) + + if path: + pip_venv_path = path + else: + pip_venv_path = os.path.join(charm_dir(), 'venv') + + subprocess.check_call(['virtualenv', pip_venv_path]) + + +def pip_get_virtualenv_path(): + return pip_venv_path From 997d9519efc049471501ef2c15a0f80ec14c0c8e Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 12:51:57 +0000 Subject: [PATCH 16/31] Use function to get pip venv path --- hooks/quantum_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 6cad7033..95b08d58 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -60,6 +60,9 @@ from charmhelpers.contrib.openstack.context import ( ) import charmhelpers.contrib.openstack.templating as templating from charmhelpers.contrib.openstack.neutron import headers_package +from charmhelpers.contrib.python.packages import ( + pip_get_virtualenv_path, +) from quantum_contexts import ( CORE_PLUGIN, OVS, NVP, NSX, N1KV, NEUTRON, QUANTUM, @@ -898,7 +901,7 @@ def git_post_install(projects_yaml): service_name = 'quantum-gateway' user_name = 'neutron' - bin_dir = os.path.join(charm_dir(), 'venv/bin') + bin_dir = os.path.join(pip_get_virtualenv_path(), 'bin' neutron_api_context = { 'service_description': 'Neutron API server', 'service_name': service_name, From 5ed06ea40b73df82b29cd32dd3b94292df8f6c9e Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 13:05:12 +0000 Subject: [PATCH 17/31] Add missing paren --- hooks/quantum_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 95b08d58..634a2182 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -901,7 +901,7 @@ def git_post_install(projects_yaml): service_name = 'quantum-gateway' user_name = 'neutron' - bin_dir = os.path.join(pip_get_virtualenv_path(), 'bin' + bin_dir = os.path.join(pip_get_virtualenv_path(), 'bin') neutron_api_context = { 'service_description': 'Neutron API server', 'service_name': service_name, From b55f0e243bb1b34128621c6c3975b68adbf5115b Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 14:40:23 +0000 Subject: [PATCH 18/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 23 +++++++++++++++++-- hooks/charmhelpers/contrib/python/packages.py | 18 ++++----------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 29ac7b78..c5e08ec2 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -611,9 +611,11 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, proxy=http_proxy, venv=True) + pip_install(repo_dir, proxy=http_proxy, + venv=os.path.join(parent_dir, 'venv')) else: - pip_install(repo_dir, venv=True) + pip_install(repo_dir, + venv=os.path.join(parent_dir, 'venv')) return repo_dir @@ -636,6 +638,23 @@ def _git_update_requirements(package_dir, reqs_dir): os.chdir(orig_dir) +def git_pip_venv_dir(projects_yaml): + """ + Return the pip virtualenv path. + """ + parent_dir = '/mnt/openstack-git' + + if not projects_yaml: + return + + projects = yaml.load(projects_yaml) + + if 'directory' in projects.keys(): + parent_dir = projects['directory'] + + return os.path.join(parent_dir, 'venv') + + def git_src_dir(projects_yaml, project): """ Return the directory where the specified project's source is located. diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 12838d24..88c8887d 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -30,8 +30,6 @@ except ImportError: apt_install('python-pip') from pip import main as pip_execute -pip_venv_path = None - __author__ = "Jorge Niedbalski " @@ -56,10 +54,10 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, venv=False, **options): +def pip_install(package, fatal=False, upgrade=False, venv=None, **options): """Install a python package""" if venv: - venv_python = os.path.join(pip_venv_path, 'bin/pip') + venv_python = os.path.join(venv, 'bin/pip') command = [venv_python, "install"] else: command = ["install"] @@ -110,17 +108,11 @@ def pip_list(): def pip_create_virtualenv(path=None): """Create an isolated Python environment.""" - global pip_venv_path - apt_install('python-virtualenv') if path: - pip_venv_path = path + venv_path = path else: - pip_venv_path = os.path.join(charm_dir(), 'venv') + venv_path = os.path.join(charm_dir(), 'venv') - subprocess.check_call(['virtualenv', pip_venv_path]) - - -def pip_get_virtualenv_path(): - return pip_venv_path + subprocess.check_call(['virtualenv', venv_path]) From 0c3e42d42309139896cbc79631d55c585187e80a Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 14:54:52 +0000 Subject: [PATCH 19/31] Use git_pip_venv_dir to get venv path --- hooks/quantum_utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 634a2182..5bcc0cb6 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -42,6 +42,7 @@ from charmhelpers.contrib.openstack.utils import ( git_install_requested, git_clone_and_install, git_src_dir, + git_pip_venv_dir, get_hostname ) @@ -60,9 +61,6 @@ from charmhelpers.contrib.openstack.context import ( ) import charmhelpers.contrib.openstack.templating as templating from charmhelpers.contrib.openstack.neutron import headers_package -from charmhelpers.contrib.python.packages import ( - pip_get_virtualenv_path, -) from quantum_contexts import ( CORE_PLUGIN, OVS, NVP, NSX, N1KV, NEUTRON, QUANTUM, @@ -901,7 +899,7 @@ def git_post_install(projects_yaml): service_name = 'quantum-gateway' user_name = 'neutron' - bin_dir = os.path.join(pip_get_virtualenv_path(), 'bin') + bin_dir = os.path.join(git_pip_venv_dir(), 'bin') neutron_api_context = { 'service_description': 'Neutron API server', 'service_name': service_name, From 9b7b1fde47941c1c24cd7172e8b98eddf1b0a62a Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 15:11:51 +0000 Subject: [PATCH 20/31] Add missing params --- hooks/quantum_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 5bcc0cb6..a4705e0a 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -899,7 +899,7 @@ def git_post_install(projects_yaml): service_name = 'quantum-gateway' user_name = 'neutron' - bin_dir = os.path.join(git_pip_venv_dir(), 'bin') + bin_dir = os.path.join(git_pip_venv_dir(projects_yaml), 'bin') neutron_api_context = { 'service_description': 'Neutron API server', 'service_name': service_name, From 73b1457f4132630b5f9e1bee17b54ba5dafffc82 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 16:41:31 +0000 Subject: [PATCH 21/31] Add rootwrap symlink --- hooks/quantum_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index a4705e0a..debe5290 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -879,6 +879,9 @@ def git_post_install(projects_yaml): shutil.copytree(c['src'], c['dest']) symlinks = [ + {'src': os.path.join(git_pip_venv_dir(projects_yaml), + 'bin/neutron-rootwrap'), + 'link': '/usr/local/bin/neutron-rootwrap'}, {'src': '/usr/local/bin/neutron-rootwrap', 'link': '/usr/bin/neutron-rootwrap'}, ] From 14f92bab75e3241edce108468c9db049de2cd624 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 18:54:35 +0000 Subject: [PATCH 22/31] Sync charm-helpers --- hooks/charmhelpers/contrib/python/packages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 88c8887d..07b0c1d7 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -115,4 +115,5 @@ def pip_create_virtualenv(path=None): else: venv_path = os.path.join(charm_dir(), 'venv') - subprocess.check_call(['virtualenv', venv_path]) + if not os.path.exists(venv_path): + subprocess.check_call(['virtualenv', venv_path]) From eb59103cc43251a3676f34b89cfac2321be20943 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 8 May 2015 15:45:15 +0000 Subject: [PATCH 23/31] Add libffi-dev to base git pkgs --- hooks/quantum_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index debe5290..ab96e8fb 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -187,6 +187,7 @@ L3HA_PACKAGES = ['keepalived'] BASE_GIT_PACKAGES = [ 'dnsmasq', + 'libff-dev', 'libxml2-dev', 'libxslt1-dev', 'python-dev', From a66b21c0ec15a85c90285b42023c3d9df2afefa5 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 8 May 2015 16:05:05 +0000 Subject: [PATCH 24/31] It's called libffi-dev --- hooks/quantum_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index ab96e8fb..ff385b05 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -187,7 +187,7 @@ L3HA_PACKAGES = ['keepalived'] BASE_GIT_PACKAGES = [ 'dnsmasq', - 'libff-dev', + 'libffi-dev', 'libxml2-dev', 'libxslt1-dev', 'python-dev', From 951cdd244c425bc53ce035ebd6b9c6272b80408f Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 8 May 2015 16:19:49 +0000 Subject: [PATCH 25/31] Add libssl-dev to base git pkgs --- hooks/quantum_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index ff385b05..44030cad 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -188,6 +188,7 @@ L3HA_PACKAGES = ['keepalived'] BASE_GIT_PACKAGES = [ 'dnsmasq', 'libffi-dev', + 'libssl-dev', 'libxml2-dev', 'libxslt1-dev', 'python-dev', From c31fff663999254bb1f0192c162fefe2641a1d79 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 11 May 2015 12:33:01 +0000 Subject: [PATCH 26/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index c5e08ec2..774eaba1 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -501,6 +501,16 @@ def git_install_requested(): requirements_dir = None +def _git_yaml_load(projects_yaml): + """ + Load the specified yaml into a dictionary. + """ + if not projects_yaml: + return None + + projects = yaml.load(projects_yaml) + + def git_clone_and_install(projects_yaml, core_project, depth=1): """ Clone/install all specified OpenStack repositories. @@ -523,10 +533,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): parent_dir = '/mnt/openstack-git' http_proxy = None - if not projects_yaml: - return - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) _git_validate_projects_yaml(projects, core_project) old_environ = dict(os.environ) @@ -644,10 +651,7 @@ def git_pip_venv_dir(projects_yaml): """ parent_dir = '/mnt/openstack-git' - if not projects_yaml: - return - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) if 'directory' in projects.keys(): parent_dir = projects['directory'] @@ -661,10 +665,7 @@ def git_src_dir(projects_yaml, project): """ parent_dir = '/mnt/openstack-git' - if not projects_yaml: - return - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) if 'directory' in projects.keys(): parent_dir = projects['directory'] @@ -680,10 +681,7 @@ def git_yaml_value(projects_yaml, key): """ Return the value in projects_yaml for the specified key. """ - if not projects_yaml: - return None - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) if key in projects.keys(): return projects[key] From 87de2f09433a928e8963d82111b4f3812b08a79b Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 11 May 2015 12:38:14 +0000 Subject: [PATCH 27/31] Add comment to fix bin symlinks --- hooks/quantum_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/quantum_utils.py b/hooks/quantum_utils.py index 44030cad..11786f9a 100644 --- a/hooks/quantum_utils.py +++ b/hooks/quantum_utils.py @@ -880,6 +880,7 @@ def git_post_install(projects_yaml): shutil.rmtree(c['dest']) shutil.copytree(c['src'], c['dest']) + # NOTE(coreycb): Need to find better solution than bin symlinks. symlinks = [ {'src': os.path.join(git_pip_venv_dir(projects_yaml), 'bin/neutron-rootwrap'), From 76541a1de2b35b983fc240ea59c7fd3ea9bd8e90 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 12 May 2015 12:14:40 +0000 Subject: [PATCH 28/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 774eaba1..d795a358 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -508,7 +508,7 @@ def _git_yaml_load(projects_yaml): if not projects_yaml: return None - projects = yaml.load(projects_yaml) + return yaml.load(projects_yaml) def git_clone_and_install(projects_yaml, core_project, depth=1): @@ -524,8 +524,8 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): repository: 'git://git.openstack.org/openstack/requirements.git', branch: 'stable/icehouse'} directory: /mnt/openstack-git - http_proxy: http://squid.internal:3128 - https_proxy: https://squid.internal:3128 + http_proxy: squid-proxy-url + https_proxy: squid-proxy-url The directory, http_proxy, and https_proxy keys are optional. """ From 26ce613bfbd21fd4e0517bdec9bba891ab42413f Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 12 May 2015 14:49:28 +0000 Subject: [PATCH 29/31] Clone from github in deploy from source amulet tests --- tests/basic_deployment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 8a59c510..3fc1fb2c 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -75,10 +75,10 @@ class QuantumGatewayBasicDeployment(OpenStackAmuletDeployment): openstack_origin_git = { 'repositories': [ {'name': 'requirements', - 'repository': 'git://git.openstack.org/openstack/requirements', + 'repository': 'git://github.com/openstack/requirements', 'branch': branch}, {'name': 'neutron', - 'repository': 'git://git.openstack.org/openstack/neutron', + 'repository': 'git://github.com/openstack/neutron', 'branch': branch}, ], 'directory': '/mnt/openstack-git', From f0c0fe63f400762397f07f016187c1a959e88742 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 12 May 2015 19:51:46 +0000 Subject: [PATCH 30/31] Add libyaml-dev as base git package --- hooks/neutron_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/neutron_utils.py b/hooks/neutron_utils.py index 961bf02d..9e106e9e 100644 --- a/hooks/neutron_utils.py +++ b/hooks/neutron_utils.py @@ -191,6 +191,7 @@ BASE_GIT_PACKAGES = [ 'libssl-dev', 'libxml2-dev', 'libxslt1-dev', + 'libyaml-dev', 'python-dev', 'python-pip', 'python-setuptools', From 6a0d635e80335dfbc314b644532cc3d1b128da18 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 27 May 2015 13:02:35 +0000 Subject: [PATCH 31/31] Sync charm-helpers --- hooks/charmhelpers/fetch/giturl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 7d842d49..ddc25b7e 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -45,7 +45,7 @@ class GitUrlFetchHandler(BaseFetchHandler): else: return True - def clone(self, source, dest, branch, depth): + def clone(self, source, dest, branch, depth=None): if not self.can_handle(source): raise UnhandledSource("Cannot handle {}".format(source))