From 38b3b4292d1579e197cc35ff0c3f358686202778 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 22 Apr 2015 18:39:49 +0000 Subject: [PATCH 01/35] Sync charm-helpers --- .../contrib/openstack/amulet/deployment.py | 2 +- hooks/charmhelpers/contrib/openstack/utils.py | 12 ++++++++++-- hooks/charmhelpers/contrib/python/packages.py | 7 +++++-- tests/charmhelpers/contrib/amulet/utils.py | 6 +++++- .../contrib/openstack/amulet/deployment.py | 2 +- 5 files changed, 22 insertions(+), 7 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 4538e961..461a702f 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -109,7 +109,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): # Must be ordered by OpenStack release (not by Ubuntu release): (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse, self.trusty_juno, self.utopic_juno, + self.trusty_icehouse, self.trusty_juno, self.utopic_juno, self.trusty_kilo, self.vivid_kilo) = range(10) releases = { diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index f90a0289..f4824f99 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -517,6 +517,7 @@ def git_clone_and_install(projects_yaml, core_project): """ global requirements_dir parent_dir = '/mnt/openstack-git' + http_proxy = None if not projects_yaml: return @@ -527,6 +528,7 @@ def git_clone_and_install(projects_yaml, core_project): old_environ = dict(os.environ) if 'http_proxy' in projects.keys(): + http_proxy = projects['http_proxy'] os.environ['http_proxy'] = projects['http_proxy'] if 'https_proxy' in projects.keys(): os.environ['https_proxy'] = projects['https_proxy'] @@ -539,10 +541,12 @@ def git_clone_and_install(projects_yaml, core_project): branch = p['branch'] if p['name'] == 'requirements': repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + http_proxy, update_requirements=False) requirements_dir = repo_dir else: repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + http_proxy, update_requirements=True) os.environ = old_environ @@ -574,7 +578,8 @@ def _git_ensure_key_exists(key, keys): error_out('openstack-origin-git key \'{}\' is missing'.format(key)) -def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements): +def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, + update_requirements): """ Clone and install a single git repository. """ @@ -598,7 +603,10 @@ def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements) _git_update_requirements(repo_dir, requirements_dir) juju_log('Installing git repo from dir: {}'.format(repo_dir)) - pip_install(repo_dir) + if http_proxy: + pip_install(repo_dir, ignore=True, proxy=http_proxy) + else: + pip_install(repo_dir, ignore=True) return repo_dir diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 8659516b..ceaa5971 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -51,17 +51,20 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, **options): +def pip_install(package, fatal=False, upgrade=False, ignore=False, **options): """Install a python package""" command = ["install"] - available_options = ('proxy', 'src', 'log', "index-url", ) + available_options = ('proxy', 'src', 'log', 'index-url', ) for option in parse_options(options, available_options): command.append(option) if upgrade: command.append('--upgrade') + if ignore: + command.append('--ignore-installed') + if isinstance(package, list): command.extend(package) else: diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 58233714..f61c2e8b 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -89,7 +89,11 @@ class AmuletUtils(object): def _get_config(self, unit, filename): """Get a ConfigParser object for parsing a unit's config file.""" file_contents = unit.file_contents(filename) - config = ConfigParser.ConfigParser() + + # NOTE(beisner): by default, ConfigParser does not handle options + # with no value, such as the flags used in the mysql my.cnf file. + # https://bugs.python.org/issue7005 + config = ConfigParser.ConfigParser(allow_no_value=True) config.readfp(io.StringIO(file_contents)) return config diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 4538e961..461a702f 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -109,7 +109,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): # Must be ordered by OpenStack release (not by Ubuntu release): (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse, self.trusty_juno, self.utopic_juno, + self.trusty_icehouse, self.trusty_juno, self.utopic_juno, self.trusty_kilo, self.vivid_kilo) = range(10) releases = { From 7819cf768789bc1987939c65a0dc8970bc60aedf Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 22 Apr 2015 18:40:12 +0000 Subject: [PATCH 02/35] Add libffi-dev, libssl-dev, and libyaml-dev to base git install packages --- hooks/neutron_api_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index 0739e33e..b0966c8d 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -66,8 +66,11 @@ KILO_PACKAGES = [ ] BASE_GIT_PACKAGES = [ + 'libffi-dev', + 'libssl-dev', 'libxml2-dev', 'libxslt1-dev', + 'libyaml-dev', 'python-dev', 'python-pip', 'python-setuptools', From 3472c5c8abeb8766992d32ac87508c4565e2a37a Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 11:55:32 +0000 Subject: [PATCH 03/35] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- hooks/charmhelpers/contrib/python/packages.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index f4824f99..0460e076 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -604,9 +604,9 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, ignore=True, proxy=http_proxy) + pip_install(repo_dir, proxy=http_proxy) else: - pip_install(repo_dir, ignore=True) + pip_install(repo_dir) return repo_dir diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index ceaa5971..3f1fb09d 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -51,7 +51,7 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, ignore=False, **options): +def pip_install(package, fatal=False, upgrade=False, **options): """Install a python package""" command = ["install"] @@ -62,9 +62,6 @@ def pip_install(package, fatal=False, upgrade=False, ignore=False, **options): if upgrade: command.append('--upgrade') - if ignore: - command.append('--ignore-installed') - if isinstance(package, list): command.extend(package) else: From 05902e855be88f403c301e802c13b06cc77c1a4f Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 13:01:33 +0000 Subject: [PATCH 04/35] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 0460e076..86bf91f5 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -604,9 +604,9 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, proxy=http_proxy) + pip_install(repo_dir, upgrade=True, proxy=http_proxy) else: - pip_install(repo_dir) + pip_install(repo_dir, upgrade=True) return repo_dir From 11ec41748030c2ea0ad41fb4769ebb9fc4acde06 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 14:43:27 +0000 Subject: [PATCH 05/35] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 86bf91f5..0460e076 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -604,9 +604,9 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, upgrade=True, proxy=http_proxy) + pip_install(repo_dir, proxy=http_proxy) else: - pip_install(repo_dir, upgrade=True) + pip_install(repo_dir) return repo_dir From b47dfc157be4c4ec83cfb89d14408eac6145fb83 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 16:05:54 +0000 Subject: [PATCH 06/35] Drop added packages --- hooks/neutron_api_utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index b0966c8d..0739e33e 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -66,11 +66,8 @@ KILO_PACKAGES = [ ] BASE_GIT_PACKAGES = [ - 'libffi-dev', - 'libssl-dev', 'libxml2-dev', 'libxslt1-dev', - 'libyaml-dev', 'python-dev', 'python-pip', 'python-setuptools', From e9c70fb71e315d6500d77ac52f7a66036e05cd9c Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 28 Apr 2015 19:05:05 +0000 Subject: [PATCH 07/35] Sync charm-helpers --- hooks/charmhelpers/fetch/__init__.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 792e629a..57a179b8 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -367,13 +367,15 @@ def install_remote(source, *args, **kwargs): # explaining why it can't handle a given source. handlers = [h for h in plugins() if h.can_handle(source) is True] installed_to = None - for handler in handlers: - try: - installed_to = handler.install(source, *args, **kwargs) - except UnhandledSource: - pass - if not installed_to: - raise UnhandledSource("No handler found for source {}".format(source)) + while not installed_to: + for handler in handlers: + try: + installed_to = handler.install(source, *args, **kwargs) + except UnhandledSource: + pass + log("CB: install loop - installed_to = |%s|".format(installed_to)) + #if not installed_to: + # raise UnhandledSource("No handler found for source {}".format(source)) return installed_to From abadc1caafebeb688ee657332239b92eb60d3173 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 1 May 2015 14:56:34 +0000 Subject: [PATCH 08/35] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 15 ++++++++------- hooks/charmhelpers/fetch/__init__.py | 16 +++++++--------- hooks/charmhelpers/fetch/giturl.py | 11 +++++++---- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 0460e076..b47ed937 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -497,7 +497,7 @@ def git_install_requested(): requirements_dir = None -def git_clone_and_install(projects_yaml, core_project): +def git_clone_and_install(projects_yaml, core_project, depth=1): """ Clone/install all specified OpenStack repositories. @@ -540,13 +540,13 @@ def git_clone_and_install(projects_yaml, core_project): repo = p['repository'] branch = p['branch'] if p['name'] == 'requirements': - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, - http_proxy, + repo_dir = _git_clone_and_install_single(repo, branch, depth, + parent_dir, http_proxy, update_requirements=False) requirements_dir = repo_dir else: - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, - http_proxy, + repo_dir = _git_clone_and_install_single(repo, branch, depth, + parent_dir, http_proxy, update_requirements=True) os.environ = old_environ @@ -578,7 +578,7 @@ def _git_ensure_key_exists(key, keys): error_out('openstack-origin-git key \'{}\' is missing'.format(key)) -def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, +def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, update_requirements): """ Clone and install a single git repository. @@ -592,7 +592,8 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, if not os.path.exists(dest_dir): juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) - repo_dir = install_remote(repo, dest=parent_dir, branch=branch) + repo_dir = install_remote(repo, dest=parent_dir, branch=branch, + depth=depth) else: repo_dir = dest_dir diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 57a179b8..792e629a 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -367,15 +367,13 @@ def install_remote(source, *args, **kwargs): # explaining why it can't handle a given source. handlers = [h for h in plugins() if h.can_handle(source) is True] installed_to = None - while not installed_to: - for handler in handlers: - try: - installed_to = handler.install(source, *args, **kwargs) - except UnhandledSource: - pass - log("CB: install loop - installed_to = |%s|".format(installed_to)) - #if not installed_to: - # raise UnhandledSource("No handler found for source {}".format(source)) + for handler in handlers: + try: + installed_to = handler.install(source, *args, **kwargs) + except UnhandledSource: + pass + if not installed_to: + raise UnhandledSource("No handler found for source {}".format(source)) return installed_to diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 93aae87b..06b0e767 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -45,14 +45,17 @@ class GitUrlFetchHandler(BaseFetchHandler): else: return True - def clone(self, source, dest, branch): + def clone(self, source, dest, branch, depth): if not self.can_handle(source): raise UnhandledSource("Cannot handle {}".format(source)) - repo = Repo.clone_from(source, dest) + if depth: + repo = Repo.clone_from(source, dest, depth) + else: + repo = Repo.clone_from(source, dest) repo.git.checkout(branch) - def install(self, source, branch="master", dest=None): + def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) branch_name = url_parts.path.strip("/").split("/")[-1] if dest: @@ -63,7 +66,7 @@ class GitUrlFetchHandler(BaseFetchHandler): if not os.path.exists(dest_dir): mkdir(dest_dir, perms=0o755) try: - self.clone(source, dest_dir, branch) + self.clone(source, dest_dir, branch, depth) except GitCommandError as e: raise UnhandledSource(e.message) except OSError as e: From ebb8fe70b62e9a3bac58dc2d7f06f60623627c83 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 2 May 2015 18:33:53 +0000 Subject: [PATCH 09/35] Sync charm-helpers --- hooks/charmhelpers/fetch/giturl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 06b0e767..742cf319 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -50,7 +50,7 @@ class GitUrlFetchHandler(BaseFetchHandler): raise UnhandledSource("Cannot handle {}".format(source)) if depth: - repo = Repo.clone_from(source, dest, depth) + repo = Repo.clone_from(source, dest, depth=depth) else: repo = Repo.clone_from(source, dest) repo.git.checkout(branch) From 7df6f3346b2e454c08c15b065affd4dec0e7d51e Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 2 May 2015 20:10:25 +0000 Subject: [PATCH 10/35] Sync charm-helpers --- hooks/charmhelpers/fetch/giturl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 742cf319..4e05c08a 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -50,10 +50,9 @@ class GitUrlFetchHandler(BaseFetchHandler): raise UnhandledSource("Cannot handle {}".format(source)) if depth: - repo = Repo.clone_from(source, dest, depth=depth) + repo = Repo.clone_from(source, dest, branch=branch, depth=depth) else: - repo = Repo.clone_from(source, dest) - repo.git.checkout(branch) + repo = Repo.clone_from(source, dest, branch=branch) def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) From 82152971095709f863dc0c08fff59023ed969348 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 5 May 2015 19:17:59 +0000 Subject: [PATCH 11/35] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 8 +++++++- hooks/charmhelpers/contrib/python/packages.py | 19 ++++++++++++++++++- hooks/charmhelpers/fetch/giturl.py | 4 ++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index b47ed937..ce8c8213 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -53,9 +53,13 @@ from charmhelpers.contrib.network.ip import ( get_ipv6_addr ) +from charmhelpers.contrib.python.packages import ( + pip_create_virtualenv, + pip_install, +) + from charmhelpers.core.host import lsb_release, mounts, umount from charmhelpers.fetch import apt_install, apt_cache, install_remote -from charmhelpers.contrib.python.packages import pip_install from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device @@ -536,6 +540,8 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): if 'directory' in projects.keys(): parent_dir = projects['directory'] + pip_create_virtualenv(proxy=http_proxy) + for p in projects['repositories']: repo = p['repository'] branch = p['branch'] diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 3f1fb09d..e62240b4 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -17,8 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . +import os +import subprocess + from charmhelpers.fetch import apt_install, apt_update -from charmhelpers.core.hookenv import log +from charmhelpers.core.hookenv import charm_dir, log try: from pip import main as pip_execute @@ -94,3 +97,17 @@ def pip_list(): """Returns the list of current python installed packages """ return pip_execute(["list"]) + + +def pip_create_virtualenv(parent_dir=charm_dir(), proxy=None): + """Create and activate an isolated Python environment (virtualenv).""" + if proxy: + pip_install('virtualenv', proxy=proxy) + else: + pip_install('virtualenv') + + venv_dir = os.path.join(parent_dir, 'venv') + subprocess.check_call(['virtualenv', '--no-site-packages', venv_dir]) + + venv_activate = os.path.join(venv_dir, 'bin/activate') + subprocess.check_call(['.', venv_activate], shell=True) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 4e05c08a..7d842d49 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -50,9 +50,9 @@ class GitUrlFetchHandler(BaseFetchHandler): raise UnhandledSource("Cannot handle {}".format(source)) if depth: - repo = Repo.clone_from(source, dest, branch=branch, depth=depth) + Repo.clone_from(source, dest, branch=branch, depth=depth) else: - repo = Repo.clone_from(source, dest, branch=branch) + Repo.clone_from(source, dest, branch=branch) def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) From e2615d975e06d7bbb45196bcbbcdfd8d8c643091 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 5 May 2015 19:48:05 +0000 Subject: [PATCH 12/35] Point upstart scripts at venv binaries --- hooks/neutron_api_utils.py | 3 +++ templates/git/upstart/neutron-server.upstart | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index 0739e33e..15a1f61b 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -20,6 +20,7 @@ from charmhelpers.contrib.openstack.utils import ( from charmhelpers.core.hookenv import ( config, + charm_dir, log, ) @@ -383,10 +384,12 @@ def git_post_install(projects_yaml): render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, perms=0o440) + bin_dir = os.path.join(charm_dir(), 'venv/bin') neutron_api_context = { 'service_description': 'Neutron API server', 'charm_name': 'neutron-api', 'process_name': 'neutron-server', + 'executable_name': os.path.join(bin_dir, 'neutron-server'), } # NOTE(coreycb): Needs systemd support diff --git a/templates/git/upstart/neutron-server.upstart b/templates/git/upstart/neutron-server.upstart index 7211e129..4bd8e268 100644 --- a/templates/git/upstart/neutron-server.upstart +++ b/templates/git/upstart/neutron-server.upstart @@ -16,7 +16,7 @@ end script script [ -r /etc/default/{{ process_name }} ] && . /etc/default/{{ process_name }} [ -r "$NEUTRON_PLUGIN_CONFIG" ] && CONF_ARG="--config-file $NEUTRON_PLUGIN_CONFIG" - exec start-stop-daemon --start --chuid neutron --exec /usr/local/bin/neutron-server -- \ + exec start-stop-daemon --start --chuid neutron --exec {{ executable_name }} -- \ --config-file /etc/neutron/neutron.conf \ --log-file /var/log/neutron/server.log $CONF_ARG end script From 3f9996a970d255bb247ed04ec8eb3bfe5dc4bedf Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 5 May 2015 19:55:25 +0000 Subject: [PATCH 13/35] Update unit tests --- unit_tests/test_neutron_api_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unit_tests/test_neutron_api_utils.py b/unit_tests/test_neutron_api_utils.py index b5760c0c..33879232 100644 --- a/unit_tests/test_neutron_api_utils.py +++ b/unit_tests/test_neutron_api_utils.py @@ -263,6 +263,7 @@ class TestNeutronAPIUtils(CharmTestCase): 'service_description': 'Neutron API server', 'charm_name': 'neutron-api', 'process_name': 'neutron-server', + 'executable_name': 'joined-string', } expected = [ call('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, From 6e5a2b3a02cefa125244dc1740d3c2fd29b081b1 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 6 May 2015 16:00:11 +0000 Subject: [PATCH 14/35] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 6 ++-- hooks/charmhelpers/contrib/python/packages.py | 29 +++++++++---------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index ce8c8213..5909b0f5 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -540,7 +540,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): if 'directory' in projects.keys(): parent_dir = projects['directory'] - pip_create_virtualenv(proxy=http_proxy) + pip_create_virtualenv() for p in projects['repositories']: repo = p['repository'] @@ -611,9 +611,9 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, proxy=http_proxy) + pip_install(repo_dir, proxy=http_proxy, venv=True) else: - pip_install(repo_dir) + pip_install(repo_dir, venv=True) return repo_dir diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index e62240b4..740eaa51 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -54,9 +54,13 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, **options): +def pip_install(package, fatal=False, upgrade=False, venv=False, **options): """Install a python package""" - command = ["install"] + if venv: + venv_python = os.path.join(charm_dir(), 'venv/bin/pip') + command = [venv_python, "install"] + else: + command = ["install"] available_options = ('proxy', 'src', 'log', 'index-url', ) for option in parse_options(options, available_options): @@ -72,7 +76,10 @@ def pip_install(package, fatal=False, upgrade=False, **options): log("Installing {} package with options: {}".format(package, command)) - pip_execute(command) + if venv: + subprocess.check_call(command) + else: + pip_execute(command) def pip_uninstall(package, **options): @@ -99,15 +106,7 @@ def pip_list(): return pip_execute(["list"]) -def pip_create_virtualenv(parent_dir=charm_dir(), proxy=None): - """Create and activate an isolated Python environment (virtualenv).""" - if proxy: - pip_install('virtualenv', proxy=proxy) - else: - pip_install('virtualenv') - - venv_dir = os.path.join(parent_dir, 'venv') - subprocess.check_call(['virtualenv', '--no-site-packages', venv_dir]) - - venv_activate = os.path.join(venv_dir, 'bin/activate') - subprocess.check_call(['.', venv_activate], shell=True) +def pip_create_virtualenv(): + """Create an isolated Python environment.""" + apt_install('python-virtualenv') + subprocess.check_call(['virtualenv', os.path.join(charm_dir(), 'venv')]) From 470d8dd388d55cba308ed7b2087bb0463a7cd850 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 12:44:23 +0000 Subject: [PATCH 15/35] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 17 +++++++++++++++- hooks/charmhelpers/contrib/python/packages.py | 20 ++++++++++++++++--- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 5909b0f5..29ac7b78 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -540,7 +540,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): if 'directory' in projects.keys(): parent_dir = projects['directory'] - pip_create_virtualenv() + pip_create_virtualenv(os.path.join(parent_dir, 'venv')) for p in projects['repositories']: repo = p['repository'] @@ -655,3 +655,18 @@ def git_src_dir(projects_yaml, project): return os.path.join(parent_dir, os.path.basename(p['repository'])) return None + + +def git_yaml_value(projects_yaml, key): + """ + Return the value in projects_yaml for the specified key. + """ + if not projects_yaml: + return None + + projects = yaml.load(projects_yaml) + + if key in projects.keys(): + return projects[key] + + return None diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 740eaa51..12838d24 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -30,6 +30,8 @@ except ImportError: apt_install('python-pip') from pip import main as pip_execute +pip_venv_path = None + __author__ = "Jorge Niedbalski " @@ -57,7 +59,7 @@ def pip_install_requirements(requirements, **options): def pip_install(package, fatal=False, upgrade=False, venv=False, **options): """Install a python package""" if venv: - venv_python = os.path.join(charm_dir(), 'venv/bin/pip') + venv_python = os.path.join(pip_venv_path, 'bin/pip') command = [venv_python, "install"] else: command = ["install"] @@ -106,7 +108,19 @@ def pip_list(): return pip_execute(["list"]) -def pip_create_virtualenv(): +def pip_create_virtualenv(path=None): """Create an isolated Python environment.""" + global pip_venv_path + apt_install('python-virtualenv') - subprocess.check_call(['virtualenv', os.path.join(charm_dir(), 'venv')]) + + if path: + pip_venv_path = path + else: + pip_venv_path = os.path.join(charm_dir(), 'venv') + + subprocess.check_call(['virtualenv', pip_venv_path]) + + +def pip_get_virtualenv_path(): + return pip_venv_path From c77ae94f69f86237d0fe6d3f0d912a460163744a Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 12:49:53 +0000 Subject: [PATCH 16/35] Use function to get pip venv path --- hooks/neutron_api_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index c8084861..80b8318a 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -10,6 +10,10 @@ from charmhelpers.contrib.openstack.neutron import ( neutron_plugin_attribute, ) +from charmhelpers.contrib.python.packages import ( + pip_get_virtualenv_path, +) + from charmhelpers.contrib.openstack.utils import ( os_release, get_os_codename_install_source, @@ -439,7 +443,7 @@ def git_post_install(projects_yaml): render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, perms=0o440) - bin_dir = os.path.join(charm_dir(), 'venv/bin') + bin_dir = os.path.join(pip_get_virtualenv_path(), 'bin') neutron_api_context = { 'service_description': 'Neutron API server', 'charm_name': 'neutron-api', From 10a4f86a992541f939f483af54ef52d454eaa76c Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 14:40:12 +0000 Subject: [PATCH 17/35] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 23 +++++++++++++++++-- hooks/charmhelpers/contrib/python/packages.py | 18 ++++----------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 29ac7b78..c5e08ec2 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -611,9 +611,11 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, proxy=http_proxy, venv=True) + pip_install(repo_dir, proxy=http_proxy, + venv=os.path.join(parent_dir, 'venv')) else: - pip_install(repo_dir, venv=True) + pip_install(repo_dir, + venv=os.path.join(parent_dir, 'venv')) return repo_dir @@ -636,6 +638,23 @@ def _git_update_requirements(package_dir, reqs_dir): os.chdir(orig_dir) +def git_pip_venv_dir(projects_yaml): + """ + Return the pip virtualenv path. + """ + parent_dir = '/mnt/openstack-git' + + if not projects_yaml: + return + + projects = yaml.load(projects_yaml) + + if 'directory' in projects.keys(): + parent_dir = projects['directory'] + + return os.path.join(parent_dir, 'venv') + + def git_src_dir(projects_yaml, project): """ Return the directory where the specified project's source is located. diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 12838d24..88c8887d 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -30,8 +30,6 @@ except ImportError: apt_install('python-pip') from pip import main as pip_execute -pip_venv_path = None - __author__ = "Jorge Niedbalski " @@ -56,10 +54,10 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, venv=False, **options): +def pip_install(package, fatal=False, upgrade=False, venv=None, **options): """Install a python package""" if venv: - venv_python = os.path.join(pip_venv_path, 'bin/pip') + venv_python = os.path.join(venv, 'bin/pip') command = [venv_python, "install"] else: command = ["install"] @@ -110,17 +108,11 @@ def pip_list(): def pip_create_virtualenv(path=None): """Create an isolated Python environment.""" - global pip_venv_path - apt_install('python-virtualenv') if path: - pip_venv_path = path + venv_path = path else: - pip_venv_path = os.path.join(charm_dir(), 'venv') + venv_path = os.path.join(charm_dir(), 'venv') - subprocess.check_call(['virtualenv', pip_venv_path]) - - -def pip_get_virtualenv_path(): - return pip_venv_path + subprocess.check_call(['virtualenv', venv_path]) From 3eef78431f5f906cebd0b38cc05283f3ccd3bf32 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 14:53:48 +0000 Subject: [PATCH 18/35] Use git_pip_venv_dir to get venv path --- hooks/neutron_api_utils.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index 80b8318a..90d21f96 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -10,16 +10,13 @@ from charmhelpers.contrib.openstack.neutron import ( neutron_plugin_attribute, ) -from charmhelpers.contrib.python.packages import ( - pip_get_virtualenv_path, -) - from charmhelpers.contrib.openstack.utils import ( os_release, get_os_codename_install_source, git_install_requested, git_clone_and_install, git_src_dir, + git_pip_venv_dir, configure_installation_source, ) @@ -443,7 +440,7 @@ def git_post_install(projects_yaml): render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, perms=0o440) - bin_dir = os.path.join(pip_get_virtualenv_path(), 'bin') + bin_dir = os.path.join(git_pip_venv_dir(), 'bin') neutron_api_context = { 'service_description': 'Neutron API server', 'charm_name': 'neutron-api', From 461a7fa736513fefbb8e98da467910bbf90a18a0 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 15:11:07 +0000 Subject: [PATCH 19/35] Add missing params --- hooks/neutron_api_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index 90d21f96..787fdf8a 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -440,7 +440,7 @@ def git_post_install(projects_yaml): render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, perms=0o440) - bin_dir = os.path.join(git_pip_venv_dir(), 'bin') + bin_dir = os.path.join(git_pip_venv_dir(projects_yaml), 'bin') neutron_api_context = { 'service_description': 'Neutron API server', 'charm_name': 'neutron-api', From 49e437f88008ef4d4b51e924fbd04f8bf5abb7ac Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 16:42:06 +0000 Subject: [PATCH 20/35] Add rootwrap symlink --- hooks/neutron_api_utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index 787fdf8a..14cad323 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -437,6 +437,17 @@ def git_post_install(projects_yaml): shutil.rmtree(c['dest']) shutil.copytree(c['src'], c['dest']) + symlinks = [ + {'src': os.path.join(git_pip_venv_dir(projects_yaml), + 'bin/neutron-rootwrap'), + 'link': '/usr/local/bin/neutron-rootwrap'}, + ] + + for s in symlinks: + if os.path.lexists(s['link']): + os.remove(s['link']) + os.symlink(s['src'], s['link']) + render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, perms=0o440) From 76eaa4c3972b582dfe384decdddd47c979abeecb Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 17:11:14 +0000 Subject: [PATCH 21/35] Unit test updates --- hooks/neutron_api_utils.py | 1 - unit_tests/test_neutron_api_utils.py | 9 +++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index 14cad323..b0389166 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -22,7 +22,6 @@ from charmhelpers.contrib.openstack.utils import ( from charmhelpers.core.hookenv import ( config, - charm_dir, log, ) diff --git a/unit_tests/test_neutron_api_utils.py b/unit_tests/test_neutron_api_utils.py index 3880ca30..9993153f 100644 --- a/unit_tests/test_neutron_api_utils.py +++ b/unit_tests/test_neutron_api_utils.py @@ -417,10 +417,11 @@ class TestNeutronAPIUtils(CharmTestCase): @patch.object(nutils, 'render') @patch('os.path.join') @patch('os.path.exists') + @patch('os.symlink') @patch('shutil.copytree') @patch('shutil.rmtree') - def test_git_post_install(self, rmtree, copytree, exists, join, render, - service_restart, git_src_dir): + def test_git_post_install(self, rmtree, copytree, symlink, exists, join, + render, service_restart, git_src_dir): projects_yaml = openstack_origin_git join.return_value = 'joined-string' nutils.git_post_install(projects_yaml) @@ -430,6 +431,10 @@ class TestNeutronAPIUtils(CharmTestCase): call('joined-string', '/etc/neutron/rootwrap.d'), ] copytree.assert_has_calls(expected) + expected = [ + call('joined-string', '/usr/local/bin/neutron-rootwrap'), + ] + symlink.assert_has_calls(expected, any_order=True) neutron_api_context = { 'service_description': 'Neutron API server', 'charm_name': 'neutron-api', From 6b0acbe459d029ff17d2dbcef33b0fb3b37398d4 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 18:42:45 +0000 Subject: [PATCH 22/35] Add base mysql support --- hooks/neutron_api_utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index b0389166..448802dc 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -17,9 +17,14 @@ from charmhelpers.contrib.openstack.utils import ( git_clone_and_install, git_src_dir, git_pip_venv_dir, + git_yaml_value, configure_installation_source, ) +from charmhelpers.contrib.python.packages import ( + pip_install, +) + from charmhelpers.core.hookenv import ( config, log, @@ -68,6 +73,7 @@ KILO_PACKAGES = [ ] BASE_GIT_PACKAGES = [ + 'libmysqlclient-dev', 'libxml2-dev', 'libxslt1-dev', 'python-dev', @@ -421,6 +427,14 @@ def git_pre_install(): def git_post_install(projects_yaml): """Perform post-install setup.""" + http_proxy = git_yaml_value(projects_yaml, 'http_proxy') + if http_proxy: + pip_install('mysql-python', proxy=http_proxy, + venv=git_pip_venv_dir(projects_yaml)) + else: + pip_install('mysql-python', + venv=git_pip_venv_dir(projects_yaml)) + src_etc = os.path.join(git_src_dir(projects_yaml, 'neutron'), 'etc') configs = [ {'src': src_etc, From 722ccb5a3a93a5be09caf57fa550a951b1530d33 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 18:47:03 +0000 Subject: [PATCH 23/35] Unit test updates --- unit_tests/test_neutron_api_utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/unit_tests/test_neutron_api_utils.py b/unit_tests/test_neutron_api_utils.py index 9993153f..0eff01c8 100644 --- a/unit_tests/test_neutron_api_utils.py +++ b/unit_tests/test_neutron_api_utils.py @@ -415,15 +415,19 @@ class TestNeutronAPIUtils(CharmTestCase): @patch.object(nutils, 'git_src_dir') @patch.object(nutils, 'service_restart') @patch.object(nutils, 'render') + @patch.object(nutils, 'git_pip_venv_dir') @patch('os.path.join') @patch('os.path.exists') @patch('os.symlink') @patch('shutil.copytree') @patch('shutil.rmtree') - def test_git_post_install(self, rmtree, copytree, symlink, exists, join, - render, service_restart, git_src_dir): + @patch('subprocess.check_call') + def test_git_post_install(self, check_call, rmtree, copytree, symlink, + exists, join, venv, render, service_restart, + git_src_dir): projects_yaml = openstack_origin_git join.return_value = 'joined-string' + venv.return_value = '/mnt/openstack-git/venv' nutils.git_post_install(projects_yaml) expected = [ call('joined-string', '/etc/neutron'), From 073dc6fd15d7ebf7bf17e8a69e776b4d50fbbe1f Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 18:54:24 +0000 Subject: [PATCH 24/35] Sync charm-helpers --- hooks/charmhelpers/contrib/python/packages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 88c8887d..07b0c1d7 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -115,4 +115,5 @@ def pip_create_virtualenv(path=None): else: venv_path = os.path.join(charm_dir(), 'venv') - subprocess.check_call(['virtualenv', venv_path]) + if not os.path.exists(venv_path): + subprocess.check_call(['virtualenv', venv_path]) From 59fcf046c45c12f25633813aa2af6d7bb4ad4919 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 8 May 2015 12:38:19 +0000 Subject: [PATCH 25/35] Patch out pip_install --- unit_tests/test_neutron_api_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unit_tests/test_neutron_api_utils.py b/unit_tests/test_neutron_api_utils.py index 0eff01c8..a1e518ef 100644 --- a/unit_tests/test_neutron_api_utils.py +++ b/unit_tests/test_neutron_api_utils.py @@ -31,6 +31,7 @@ TO_PATCH = [ 'log', 'neutron_plugin_attribute', 'os_release', + 'pip_install', 'subprocess', ] From 156f969c314b9a4f88c65714c7c92d2ebdd1a64f Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 8 May 2015 15:31:08 +0000 Subject: [PATCH 26/35] Add libffi-dev to base git pkgs --- hooks/neutron_api_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index 448802dc..a9216a44 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -73,6 +73,7 @@ KILO_PACKAGES = [ ] BASE_GIT_PACKAGES = [ + 'libffi-dev', 'libmysqlclient-dev', 'libxml2-dev', 'libxslt1-dev', From 73d89aca8210a5a28cbb0e180e3827fe3856cdb1 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 8 May 2015 15:43:50 +0000 Subject: [PATCH 27/35] Add libssl-dev to base git pkgs --- hooks/neutron_api_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index a9216a44..93c9f315 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -75,6 +75,7 @@ KILO_PACKAGES = [ BASE_GIT_PACKAGES = [ 'libffi-dev', 'libmysqlclient-dev', + 'libssl-dev', 'libxml2-dev', 'libxslt1-dev', 'python-dev', From 8fe5cb7ed02f5c054553361369f1bd289465503e Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 8 May 2015 16:46:38 +0000 Subject: [PATCH 28/35] Add symlink for neutron-db-manage --- hooks/neutron_api_utils.py | 3 +++ unit_tests/test_neutron_api_utils.py | 1 + 2 files changed, 4 insertions(+) diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index 93c9f315..43b80b75 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -456,6 +456,9 @@ def git_post_install(projects_yaml): {'src': os.path.join(git_pip_venv_dir(projects_yaml), 'bin/neutron-rootwrap'), 'link': '/usr/local/bin/neutron-rootwrap'}, + {'src': os.path.join(git_pip_venv_dir(projects_yaml), + 'bin/neutron-db-manage'), + 'link': '/usr/local/bin/neutron-db-manage'}, ] for s in symlinks: diff --git a/unit_tests/test_neutron_api_utils.py b/unit_tests/test_neutron_api_utils.py index a1e518ef..7de08226 100644 --- a/unit_tests/test_neutron_api_utils.py +++ b/unit_tests/test_neutron_api_utils.py @@ -438,6 +438,7 @@ class TestNeutronAPIUtils(CharmTestCase): copytree.assert_has_calls(expected) expected = [ call('joined-string', '/usr/local/bin/neutron-rootwrap'), + call('joined-string', '/usr/local/bin/neutron-db-manage'), ] symlink.assert_has_calls(expected, any_order=True) neutron_api_context = { From 4115ab1f68f5b130643ec380b4b43664eb34ec01 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 11 May 2015 12:32:50 +0000 Subject: [PATCH 29/35] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index c5e08ec2..774eaba1 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -501,6 +501,16 @@ def git_install_requested(): requirements_dir = None +def _git_yaml_load(projects_yaml): + """ + Load the specified yaml into a dictionary. + """ + if not projects_yaml: + return None + + projects = yaml.load(projects_yaml) + + def git_clone_and_install(projects_yaml, core_project, depth=1): """ Clone/install all specified OpenStack repositories. @@ -523,10 +533,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): parent_dir = '/mnt/openstack-git' http_proxy = None - if not projects_yaml: - return - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) _git_validate_projects_yaml(projects, core_project) old_environ = dict(os.environ) @@ -644,10 +651,7 @@ def git_pip_venv_dir(projects_yaml): """ parent_dir = '/mnt/openstack-git' - if not projects_yaml: - return - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) if 'directory' in projects.keys(): parent_dir = projects['directory'] @@ -661,10 +665,7 @@ def git_src_dir(projects_yaml, project): """ parent_dir = '/mnt/openstack-git' - if not projects_yaml: - return - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) if 'directory' in projects.keys(): parent_dir = projects['directory'] @@ -680,10 +681,7 @@ def git_yaml_value(projects_yaml, key): """ Return the value in projects_yaml for the specified key. """ - if not projects_yaml: - return None - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) if key in projects.keys(): return projects[key] From 3c674e7fbd5150f5ed28604ff001ccaeba6a4341 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 11 May 2015 12:37:15 +0000 Subject: [PATCH 30/35] Add comment to fix bin symlinks --- hooks/neutron_api_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index 43b80b75..71450bff 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -452,6 +452,7 @@ def git_post_install(projects_yaml): shutil.rmtree(c['dest']) shutil.copytree(c['src'], c['dest']) + # NOTE(coreycb): Need to find better solution than bin symlinks. symlinks = [ {'src': os.path.join(git_pip_venv_dir(projects_yaml), 'bin/neutron-rootwrap'), From 2891bfdb2b9eb01adc8f8aeedbef9418d2b3d39a Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 11 May 2015 12:46:05 +0000 Subject: [PATCH 31/35] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 774eaba1..8bd2356e 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -508,7 +508,7 @@ def _git_yaml_load(projects_yaml): if not projects_yaml: return None - projects = yaml.load(projects_yaml) + return yaml.load(projects_yaml) def git_clone_and_install(projects_yaml, core_project, depth=1): From 574cf2f106b1185b67b177fc49fef62465a4109c Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 12 May 2015 14:24:44 +0000 Subject: [PATCH 32/35] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 8bd2356e..d795a358 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -524,8 +524,8 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): repository: 'git://git.openstack.org/openstack/requirements.git', branch: 'stable/icehouse'} directory: /mnt/openstack-git - http_proxy: http://squid.internal:3128 - https_proxy: https://squid.internal:3128 + http_proxy: squid-proxy-url + https_proxy: squid-proxy-url The directory, http_proxy, and https_proxy keys are optional. """ From 11145b85b708c8de2910bb42ff507058012e4028 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 12 May 2015 14:49:28 +0000 Subject: [PATCH 33/35] Clone from github in deploy from source amulet tests --- tests/basic_deployment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index f4617162..2428557d 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -76,10 +76,10 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment): openstack_origin_git = { 'repositories': [ {'name': 'requirements', - 'repository': 'git://git.openstack.org/openstack/requirements', + 'repository': 'git://github.com/openstack/requirements', 'branch': branch}, {'name': 'neutron', - 'repository': 'git://git.openstack.org/openstack/neutron', + 'repository': 'git://github.com/openstack/neutron', 'branch': branch}, ], 'directory': '/mnt/openstack-git', From 84204aba62319d58fa0ad153ecd1fada8c07b445 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 12 May 2015 19:50:14 +0000 Subject: [PATCH 34/35] Add libyaml-dev as base git package --- hooks/neutron_api_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index 71450bff..9f3724aa 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -78,6 +78,7 @@ BASE_GIT_PACKAGES = [ 'libssl-dev', 'libxml2-dev', 'libxslt1-dev', + 'libyaml-dev', 'python-dev', 'python-pip', 'python-setuptools', From 728f829e4043682c02b31fea229dc0b647811250 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 27 May 2015 13:02:18 +0000 Subject: [PATCH 35/35] Sync charm-helpers --- hooks/charmhelpers/fetch/giturl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 7d842d49..ddc25b7e 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -45,7 +45,7 @@ class GitUrlFetchHandler(BaseFetchHandler): else: return True - def clone(self, source, dest, branch, depth): + def clone(self, source, dest, branch, depth=None): if not self.can_handle(source): raise UnhandledSource("Cannot handle {}".format(source))