From 0fe6d3ae11c242862aa28a31ba2f1c0587e0ced8 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 22 Apr 2015 18:48:17 +0000 Subject: [PATCH 01/40] Sync charm-helpers --- .../contrib/openstack/amulet/deployment.py | 21 +++++++++++++------ hooks/charmhelpers/contrib/openstack/utils.py | 12 +++++++++-- hooks/charmhelpers/contrib/python/packages.py | 7 +++++-- tests/charmhelpers/contrib/amulet/utils.py | 9 +++++++- .../contrib/openstack/amulet/deployment.py | 21 +++++++++++++------ 5 files changed, 53 insertions(+), 17 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 11d49a7c..461a702f 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -46,15 +46,22 @@ class OpenStackAmuletDeployment(AmuletDeployment): stable or next branches for the other_services.""" base_charms = ['mysql', 'mongodb'] + if self.series in ['precise', 'trusty']: + base_series = self.series + else: + base_series = self.current_next + if self.stable: for svc in other_services: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: for svc in other_services: if svc['name'] in base_charms: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: temp = 'lp:~openstack-charmers/charms/{}/{}/next' svc['location'] = temp.format(self.current_next, @@ -99,10 +106,12 @@ class OpenStackAmuletDeployment(AmuletDeployment): Return an integer representing the enum value of the openstack release. """ + # Must be ordered by OpenStack release (not by Ubuntu release): (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse, self.trusty_juno, self.trusty_kilo, - self.utopic_juno, self.vivid_kilo) = range(10) + self.trusty_icehouse, self.trusty_juno, self.utopic_juno, + self.trusty_kilo, self.vivid_kilo) = range(10) + releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index f90a0289..f4824f99 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -517,6 +517,7 @@ def git_clone_and_install(projects_yaml, core_project): """ global requirements_dir parent_dir = '/mnt/openstack-git' + http_proxy = None if not projects_yaml: return @@ -527,6 +528,7 @@ def git_clone_and_install(projects_yaml, core_project): old_environ = dict(os.environ) if 'http_proxy' in projects.keys(): + http_proxy = projects['http_proxy'] os.environ['http_proxy'] = projects['http_proxy'] if 'https_proxy' in projects.keys(): os.environ['https_proxy'] = projects['https_proxy'] @@ -539,10 +541,12 @@ def git_clone_and_install(projects_yaml, core_project): branch = p['branch'] if p['name'] == 'requirements': repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + http_proxy, update_requirements=False) requirements_dir = repo_dir else: repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + http_proxy, update_requirements=True) os.environ = old_environ @@ -574,7 +578,8 @@ def _git_ensure_key_exists(key, keys): error_out('openstack-origin-git key \'{}\' is missing'.format(key)) -def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements): +def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, + update_requirements): """ Clone and install a single git repository. """ @@ -598,7 +603,10 @@ def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements) _git_update_requirements(repo_dir, requirements_dir) juju_log('Installing git repo from dir: {}'.format(repo_dir)) - pip_install(repo_dir) + if http_proxy: + pip_install(repo_dir, ignore=True, proxy=http_proxy) + else: + pip_install(repo_dir, ignore=True) return repo_dir diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 8659516b..ceaa5971 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -51,17 +51,20 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, **options): +def pip_install(package, fatal=False, upgrade=False, ignore=False, **options): """Install a python package""" command = ["install"] - available_options = ('proxy', 'src', 'log', "index-url", ) + available_options = ('proxy', 'src', 'log', 'index-url', ) for option in parse_options(options, available_options): command.append(option) if upgrade: command.append('--upgrade') + if ignore: + command.append('--ignore-installed') + if isinstance(package, list): command.extend(package) else: diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 5088b1d1..f61c2e8b 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -79,6 +79,9 @@ class AmuletUtils(object): for k, v in six.iteritems(commands): for cmd in v: output, code = k.run(cmd) + self.log.debug('{} `{}` returned ' + '{}'.format(k.info['unit_name'], + cmd, code)) if code != 0: return "command `{}` returned {}".format(cmd, str(code)) return None @@ -86,7 +89,11 @@ class AmuletUtils(object): def _get_config(self, unit, filename): """Get a ConfigParser object for parsing a unit's config file.""" file_contents = unit.file_contents(filename) - config = ConfigParser.ConfigParser() + + # NOTE(beisner): by default, ConfigParser does not handle options + # with no value, such as the flags used in the mysql my.cnf file. + # https://bugs.python.org/issue7005 + config = ConfigParser.ConfigParser(allow_no_value=True) config.readfp(io.StringIO(file_contents)) return config diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 11d49a7c..461a702f 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -46,15 +46,22 @@ class OpenStackAmuletDeployment(AmuletDeployment): stable or next branches for the other_services.""" base_charms = ['mysql', 'mongodb'] + if self.series in ['precise', 'trusty']: + base_series = self.series + else: + base_series = self.current_next + if self.stable: for svc in other_services: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: for svc in other_services: if svc['name'] in base_charms: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: temp = 'lp:~openstack-charmers/charms/{}/{}/next' svc['location'] = temp.format(self.current_next, @@ -99,10 +106,12 @@ class OpenStackAmuletDeployment(AmuletDeployment): Return an integer representing the enum value of the openstack release. """ + # Must be ordered by OpenStack release (not by Ubuntu release): (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse, self.trusty_juno, self.trusty_kilo, - self.utopic_juno, self.vivid_kilo) = range(10) + self.trusty_icehouse, self.trusty_juno, self.utopic_juno, + self.trusty_kilo, self.vivid_kilo) = range(10) + releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, From 9347b098dccf4cb38ba36e103dbc0fd7c57e8d7e Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 22 Apr 2015 19:33:19 +0000 Subject: [PATCH 02/40] Add libyaml-dev to base git install packages --- hooks/keystone_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index c6bd3835..615c1998 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -126,6 +126,7 @@ BASE_GIT_PACKAGES = [ 'libssl-dev', 'libxml2-dev', 'libxslt1-dev', + 'libyaml-dev', 'python-dev', 'python-pip', 'python-setuptools', From d29d61b7f0a910bd35a49730a6d7870b50673436 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 01:19:23 +0000 Subject: [PATCH 03/40] Unit test update --- unit_tests/test_keystone_hooks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index 50f0adcd..27670ae5 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -120,7 +120,8 @@ class KeystoneRelationTests(CharmTestCase): ['haproxy', 'unison', 'python-setuptools', 'python-six', 'uuid', 'python-mysqldb', 'python-pip', 'libssl-dev', 'openssl', 'libffi-dev', 'apache2', 'pwgen', 'libxslt1-dev', - 'python-psycopg2', 'zlib1g-dev', 'python-dev', 'libxml2-dev'], + 'python-psycopg2', 'libyaml-dev', 'zlib1g-dev', 'python-dev', + 'libxml2-dev'], fatal=True) self.git_install.assert_called_with(projects_yaml) From d552055ec2f506c89d620bddd9be2c109db94830 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 11:55:31 +0000 Subject: [PATCH 04/40] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- hooks/charmhelpers/contrib/python/packages.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index f4824f99..0460e076 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -604,9 +604,9 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, ignore=True, proxy=http_proxy) + pip_install(repo_dir, proxy=http_proxy) else: - pip_install(repo_dir, ignore=True) + pip_install(repo_dir) return repo_dir diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index ceaa5971..3f1fb09d 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -51,7 +51,7 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, ignore=False, **options): +def pip_install(package, fatal=False, upgrade=False, **options): """Install a python package""" command = ["install"] @@ -62,9 +62,6 @@ def pip_install(package, fatal=False, upgrade=False, ignore=False, **options): if upgrade: command.append('--upgrade') - if ignore: - command.append('--ignore-installed') - if isinstance(package, list): command.extend(package) else: From 3a3ec5a17abd47bfe6d1c45fbd09424a33c9b527 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 13:01:33 +0000 Subject: [PATCH 05/40] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 0460e076..86bf91f5 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -604,9 +604,9 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, proxy=http_proxy) + pip_install(repo_dir, upgrade=True, proxy=http_proxy) else: - pip_install(repo_dir) + pip_install(repo_dir, upgrade=True) return repo_dir From 74d8b51df7a78e05195bc9758350afa1e1a647db Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 14:43:27 +0000 Subject: [PATCH 06/40] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 86bf91f5..0460e076 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -604,9 +604,9 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, upgrade=True, proxy=http_proxy) + pip_install(repo_dir, proxy=http_proxy) else: - pip_install(repo_dir, upgrade=True) + pip_install(repo_dir) return repo_dir From f6414e3a908a9fd0f414698435cd47a8e86de327 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 16:01:40 +0000 Subject: [PATCH 07/40] Drop added packages --- hooks/keystone_utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 615c1998..05d0a96a 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -122,11 +122,8 @@ BASE_PACKAGES = [ ] BASE_GIT_PACKAGES = [ - 'libffi-dev', - 'libssl-dev', 'libxml2-dev', 'libxslt1-dev', - 'libyaml-dev', 'python-dev', 'python-pip', 'python-setuptools', From f7f29385e28be7c266edb8c3dfb7ef71ba60b265 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 16:03:49 +0000 Subject: [PATCH 08/40] Fix unit test --- unit_tests/test_keystone_hooks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index 27670ae5..c5a2d33b 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -118,9 +118,9 @@ class KeystoneRelationTests(CharmTestCase): self.assertTrue(self.apt_update.called) self.apt_install.assert_called_with( ['haproxy', 'unison', 'python-setuptools', 'python-six', 'uuid', - 'python-mysqldb', 'python-pip', 'libssl-dev', 'openssl', - 'libffi-dev', 'apache2', 'pwgen', 'libxslt1-dev', - 'python-psycopg2', 'libyaml-dev', 'zlib1g-dev', 'python-dev', + 'python-mysqldb', 'python-pip', 'openssl', + 'apache2', 'pwgen', 'libxslt1-dev', + 'python-psycopg2', 'zlib1g-dev', 'python-dev', 'libxml2-dev'], fatal=True) self.git_install.assert_called_with(projects_yaml) From 30f5a891b559dfcaa33d46434d1c1cd1c3da7fc5 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 28 Apr 2015 19:04:45 +0000 Subject: [PATCH 09/40] Sync charm-helpers --- hooks/charmhelpers/fetch/__init__.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 792e629a..57a179b8 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -367,13 +367,15 @@ def install_remote(source, *args, **kwargs): # explaining why it can't handle a given source. handlers = [h for h in plugins() if h.can_handle(source) is True] installed_to = None - for handler in handlers: - try: - installed_to = handler.install(source, *args, **kwargs) - except UnhandledSource: - pass - if not installed_to: - raise UnhandledSource("No handler found for source {}".format(source)) + while not installed_to: + for handler in handlers: + try: + installed_to = handler.install(source, *args, **kwargs) + except UnhandledSource: + pass + log("CB: install loop - installed_to = |%s|".format(installed_to)) + #if not installed_to: + # raise UnhandledSource("No handler found for source {}".format(source)) return installed_to From 5a301434ae14cc6cafb2f7f10cd6a58e2963d2c7 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 1 May 2015 14:56:20 +0000 Subject: [PATCH 10/40] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 15 ++++++++------- hooks/charmhelpers/fetch/__init__.py | 16 +++++++--------- hooks/charmhelpers/fetch/giturl.py | 11 +++++++---- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 0460e076..b47ed937 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -497,7 +497,7 @@ def git_install_requested(): requirements_dir = None -def git_clone_and_install(projects_yaml, core_project): +def git_clone_and_install(projects_yaml, core_project, depth=1): """ Clone/install all specified OpenStack repositories. @@ -540,13 +540,13 @@ def git_clone_and_install(projects_yaml, core_project): repo = p['repository'] branch = p['branch'] if p['name'] == 'requirements': - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, - http_proxy, + repo_dir = _git_clone_and_install_single(repo, branch, depth, + parent_dir, http_proxy, update_requirements=False) requirements_dir = repo_dir else: - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, - http_proxy, + repo_dir = _git_clone_and_install_single(repo, branch, depth, + parent_dir, http_proxy, update_requirements=True) os.environ = old_environ @@ -578,7 +578,7 @@ def _git_ensure_key_exists(key, keys): error_out('openstack-origin-git key \'{}\' is missing'.format(key)) -def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, +def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, update_requirements): """ Clone and install a single git repository. @@ -592,7 +592,8 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, if not os.path.exists(dest_dir): juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) - repo_dir = install_remote(repo, dest=parent_dir, branch=branch) + repo_dir = install_remote(repo, dest=parent_dir, branch=branch, + depth=depth) else: repo_dir = dest_dir diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 57a179b8..792e629a 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -367,15 +367,13 @@ def install_remote(source, *args, **kwargs): # explaining why it can't handle a given source. handlers = [h for h in plugins() if h.can_handle(source) is True] installed_to = None - while not installed_to: - for handler in handlers: - try: - installed_to = handler.install(source, *args, **kwargs) - except UnhandledSource: - pass - log("CB: install loop - installed_to = |%s|".format(installed_to)) - #if not installed_to: - # raise UnhandledSource("No handler found for source {}".format(source)) + for handler in handlers: + try: + installed_to = handler.install(source, *args, **kwargs) + except UnhandledSource: + pass + if not installed_to: + raise UnhandledSource("No handler found for source {}".format(source)) return installed_to diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 93aae87b..06b0e767 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -45,14 +45,17 @@ class GitUrlFetchHandler(BaseFetchHandler): else: return True - def clone(self, source, dest, branch): + def clone(self, source, dest, branch, depth): if not self.can_handle(source): raise UnhandledSource("Cannot handle {}".format(source)) - repo = Repo.clone_from(source, dest) + if depth: + repo = Repo.clone_from(source, dest, depth) + else: + repo = Repo.clone_from(source, dest) repo.git.checkout(branch) - def install(self, source, branch="master", dest=None): + def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) branch_name = url_parts.path.strip("/").split("/")[-1] if dest: @@ -63,7 +66,7 @@ class GitUrlFetchHandler(BaseFetchHandler): if not os.path.exists(dest_dir): mkdir(dest_dir, perms=0o755) try: - self.clone(source, dest_dir, branch) + self.clone(source, dest_dir, branch, depth) except GitCommandError as e: raise UnhandledSource(e.message) except OSError as e: From 8dc701ff6f5e02c5e8f8adf9414026e8eb3c62ca Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 2 May 2015 18:33:30 +0000 Subject: [PATCH 11/40] Sync charm-helpers --- hooks/charmhelpers/fetch/giturl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 06b0e767..742cf319 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -50,7 +50,7 @@ class GitUrlFetchHandler(BaseFetchHandler): raise UnhandledSource("Cannot handle {}".format(source)) if depth: - repo = Repo.clone_from(source, dest, depth) + repo = Repo.clone_from(source, dest, depth=depth) else: repo = Repo.clone_from(source, dest) repo.git.checkout(branch) From 49336f4824ac22ac86593da9277552eef06908ad Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 2 May 2015 20:10:04 +0000 Subject: [PATCH 12/40] Sync charm-helpers --- charm-helpers-tests.yaml | 2 +- hooks/charmhelpers/fetch/giturl.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/charm-helpers-tests.yaml b/charm-helpers-tests.yaml index 48b12f6f..8b9615f7 100644 --- a/charm-helpers-tests.yaml +++ b/charm-helpers-tests.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-helpers +branch: lp:charm-aelpers destination: tests/charmhelpers include: - contrib.amulet diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 742cf319..4e05c08a 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -50,10 +50,9 @@ class GitUrlFetchHandler(BaseFetchHandler): raise UnhandledSource("Cannot handle {}".format(source)) if depth: - repo = Repo.clone_from(source, dest, depth=depth) + repo = Repo.clone_from(source, dest, branch=branch, depth=depth) else: - repo = Repo.clone_from(source, dest) - repo.git.checkout(branch) + repo = Repo.clone_from(source, dest, branch=branch) def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) From 1508f122396498c863d54b4938f5f02f48fc3c41 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 5 May 2015 19:17:46 +0000 Subject: [PATCH 13/40] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 8 +++++++- hooks/charmhelpers/contrib/python/packages.py | 19 ++++++++++++++++++- hooks/charmhelpers/fetch/giturl.py | 4 ++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index b47ed937..ce8c8213 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -53,9 +53,13 @@ from charmhelpers.contrib.network.ip import ( get_ipv6_addr ) +from charmhelpers.contrib.python.packages import ( + pip_create_virtualenv, + pip_install, +) + from charmhelpers.core.host import lsb_release, mounts, umount from charmhelpers.fetch import apt_install, apt_cache, install_remote -from charmhelpers.contrib.python.packages import pip_install from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device @@ -536,6 +540,8 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): if 'directory' in projects.keys(): parent_dir = projects['directory'] + pip_create_virtualenv(proxy=http_proxy) + for p in projects['repositories']: repo = p['repository'] branch = p['branch'] diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 3f1fb09d..e62240b4 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -17,8 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . +import os +import subprocess + from charmhelpers.fetch import apt_install, apt_update -from charmhelpers.core.hookenv import log +from charmhelpers.core.hookenv import charm_dir, log try: from pip import main as pip_execute @@ -94,3 +97,17 @@ def pip_list(): """Returns the list of current python installed packages """ return pip_execute(["list"]) + + +def pip_create_virtualenv(parent_dir=charm_dir(), proxy=None): + """Create and activate an isolated Python environment (virtualenv).""" + if proxy: + pip_install('virtualenv', proxy=proxy) + else: + pip_install('virtualenv') + + venv_dir = os.path.join(parent_dir, 'venv') + subprocess.check_call(['virtualenv', '--no-site-packages', venv_dir]) + + venv_activate = os.path.join(venv_dir, 'bin/activate') + subprocess.check_call(['.', venv_activate], shell=True) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 4e05c08a..7d842d49 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -50,9 +50,9 @@ class GitUrlFetchHandler(BaseFetchHandler): raise UnhandledSource("Cannot handle {}".format(source)) if depth: - repo = Repo.clone_from(source, dest, branch=branch, depth=depth) + Repo.clone_from(source, dest, branch=branch, depth=depth) else: - repo = Repo.clone_from(source, dest, branch=branch) + Repo.clone_from(source, dest, branch=branch) def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) From ecefb064f0a5f413225a84606a68ae6d6b2bf509 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 5 May 2015 19:21:02 +0000 Subject: [PATCH 14/40] Point upstart scripts at venv binaries --- hooks/keystone_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 05d0a96a..85c42270 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -1706,13 +1706,14 @@ def git_post_install(projects_yaml): render('git/logging.conf', '/etc/keystone/logging.conf', {}, perms=0o644) + bin_dir = os.path.join(charm_dir(), 'venv/bin') keystone_context = { 'service_description': 'Keystone API server', 'service_name': 'Keystone', 'user_name': 'keystone', 'start_dir': '/var/lib/keystone', 'process_name': 'keystone', - 'executable_name': '/usr/local/bin/keystone-all', + 'executable_name': os.path.join(bin_dir, 'keystone-all'), 'config_files': ['/etc/keystone/keystone.conf'], 'log_file': '/var/log/keystone/keystone.log', } From fa6a9f6ec35c759f5bc6e5d2f2f84512f9a3c418 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 5 May 2015 19:54:36 +0000 Subject: [PATCH 15/40] Update unit tests --- unit_tests/test_keystone_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/test_keystone_utils.py b/unit_tests/test_keystone_utils.py index d32e2ab4..7d918cf1 100644 --- a/unit_tests/test_keystone_utils.py +++ b/unit_tests/test_keystone_utils.py @@ -675,7 +675,7 @@ class TestKeystoneUtils(CharmTestCase): 'user_name': 'keystone', 'start_dir': '/var/lib/keystone', 'process_name': 'keystone', - 'executable_name': '/usr/local/bin/keystone-all', + 'executable_name': 'joined-string', 'config_files': ['/etc/keystone/keystone.conf'], 'log_file': '/var/log/keystone/keystone.log', } From e9ad6f1ead762e92d93356f54fdd6a8df4bc708c Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 6 May 2015 15:53:57 +0000 Subject: [PATCH 16/40] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 6 ++-- hooks/charmhelpers/contrib/python/packages.py | 29 +++++++++---------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index ce8c8213..5909b0f5 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -540,7 +540,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): if 'directory' in projects.keys(): parent_dir = projects['directory'] - pip_create_virtualenv(proxy=http_proxy) + pip_create_virtualenv() for p in projects['repositories']: repo = p['repository'] @@ -611,9 +611,9 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, proxy=http_proxy) + pip_install(repo_dir, proxy=http_proxy, venv=True) else: - pip_install(repo_dir) + pip_install(repo_dir, venv=True) return repo_dir diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index e62240b4..740eaa51 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -54,9 +54,13 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, **options): +def pip_install(package, fatal=False, upgrade=False, venv=False, **options): """Install a python package""" - command = ["install"] + if venv: + venv_python = os.path.join(charm_dir(), 'venv/bin/pip') + command = [venv_python, "install"] + else: + command = ["install"] available_options = ('proxy', 'src', 'log', 'index-url', ) for option in parse_options(options, available_options): @@ -72,7 +76,10 @@ def pip_install(package, fatal=False, upgrade=False, **options): log("Installing {} package with options: {}".format(package, command)) - pip_execute(command) + if venv: + subprocess.check_call(command) + else: + pip_execute(command) def pip_uninstall(package, **options): @@ -99,15 +106,7 @@ def pip_list(): return pip_execute(["list"]) -def pip_create_virtualenv(parent_dir=charm_dir(), proxy=None): - """Create and activate an isolated Python environment (virtualenv).""" - if proxy: - pip_install('virtualenv', proxy=proxy) - else: - pip_install('virtualenv') - - venv_dir = os.path.join(parent_dir, 'venv') - subprocess.check_call(['virtualenv', '--no-site-packages', venv_dir]) - - venv_activate = os.path.join(venv_dir, 'bin/activate') - subprocess.check_call(['.', venv_activate], shell=True) +def pip_create_virtualenv(): + """Create an isolated Python environment.""" + apt_install('python-virtualenv') + subprocess.check_call(['virtualenv', os.path.join(charm_dir(), 'venv')]) From 705bba9358740065845daec93b604158cc2e9c79 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 6 May 2015 19:12:31 +0000 Subject: [PATCH 17/40] Symlink keystone binaries to /usr/local/bin/* for 'keystone keystone-manage db_sync' --- hooks/keystone_utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 85c42270..9e06630e 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -1704,6 +1704,18 @@ def git_post_install(projects_yaml): shutil.rmtree(configs['dest']) shutil.copytree(configs['src'], configs['dest']) + symlinks = [ + {'src': os.path.join(charm_dir(), 'venv/bin/keystone'), + 'link': '/usr/local/bin/keystone'}, + {'src': os.path.join(charm_dir(), 'venv/bin/keystone-manage'), + 'link': '/usr/local/bin/keystone-manage'}, + ] + + for s in symlinks: + if os.path.lexists(s['link']): + os.remove(s['link']) + os.symlink(s['src'], s['link']) + render('git/logging.conf', '/etc/keystone/logging.conf', {}, perms=0o644) bin_dir = os.path.join(charm_dir(), 'venv/bin') From 79a7bb8f07f7b8afb71726aa6e99274476550314 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 6 May 2015 21:11:46 +0000 Subject: [PATCH 18/40] Install mysql client/interface packages --- hooks/keystone_utils.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 9e06630e..9d4f19db 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -44,8 +44,14 @@ from charmhelpers.contrib.openstack.utils import ( git_install_requested, git_clone_and_install, git_src_dir, + git_http_proxy, os_release, - save_script_rc as _save_script_rc) + save_script_rc as _save_script_rc +) + +from charmhelpers.contrib.python.packages import ( + pip_install, +) from charmhelpers.core.strutils import ( bool_from_string, @@ -122,6 +128,7 @@ BASE_PACKAGES = [ ] BASE_GIT_PACKAGES = [ + 'libmysqlclient-dev', 'libxml2-dev', 'libxslt1-dev', 'python-dev', @@ -1694,6 +1701,12 @@ def git_pre_install(): def git_post_install(projects_yaml): """Perform keystone post-install setup.""" + http_proxy = git_http_proxy(projects_yaml) + if http_proxy: + pip_install('mysql-python', proxy=http_proxy, venv=True) + else: + pip_install('mysql-python', venv=True) + src_etc = os.path.join(git_src_dir(projects_yaml, 'keystone'), 'etc') configs = { 'src': src_etc, From 8699fa70c9c412b37e2826f19cc32c23b9f786d7 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 6 May 2015 21:23:33 +0000 Subject: [PATCH 19/40] Sync charm-helpers --- charm-helpers-tests.yaml | 2 +- hooks/charmhelpers/contrib/openstack/utils.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/charm-helpers-tests.yaml b/charm-helpers-tests.yaml index 8b9615f7..48b12f6f 100644 --- a/charm-helpers-tests.yaml +++ b/charm-helpers-tests.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-aelpers +branch: lp:charm-helpers destination: tests/charmhelpers include: - contrib.amulet diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 5909b0f5..6d91490e 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -655,3 +655,18 @@ def git_src_dir(projects_yaml, project): return os.path.join(parent_dir, os.path.basename(p['repository'])) return None + + +def git_http_proxy(projects_yaml): + """ + Return the http_proxy value if it is specified in projects_yaml. + """ + if not projects_yaml: + return + + projects = yaml.load(projects_yaml) + + if 'http_proxy' in projects.keys(): + return projects['http_proxy'] + + return None From c20287a4f841bd6719e316d453f0fc75651b5b9f Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 6 May 2015 23:37:13 +0000 Subject: [PATCH 20/40] Allow python-keystoneclient to be installed as keystone_utils imports from it. Charm uses base system python environment while deployed openstack services use virtualenv python environment. --- hooks/keystone_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 9d4f19db..c10cc11b 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -144,7 +144,6 @@ BASE_SERVICES = [ # ubuntu packages that should not be installed when deploying from git GIT_PACKAGE_BLACKLIST = [ 'keystone', - 'python-keystoneclient', ] API_PORTS = { From 511744c4caa76ab6e7d784ee58f02bc32ff6f54d Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 01:12:22 +0000 Subject: [PATCH 21/40] Update symlinks --- hooks/keystone_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index c10cc11b..6e731394 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -1717,8 +1717,6 @@ def git_post_install(projects_yaml): shutil.copytree(configs['src'], configs['dest']) symlinks = [ - {'src': os.path.join(charm_dir(), 'venv/bin/keystone'), - 'link': '/usr/local/bin/keystone'}, {'src': os.path.join(charm_dir(), 'venv/bin/keystone-manage'), 'link': '/usr/local/bin/keystone-manage'}, ] From ea8bde8b75cd8990467745f492798ffa6beaf2b5 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 12:44:13 +0000 Subject: [PATCH 22/40] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 12 +++++------ hooks/charmhelpers/contrib/python/packages.py | 20 ++++++++++++++++--- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 6d91490e..29ac7b78 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -540,7 +540,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): if 'directory' in projects.keys(): parent_dir = projects['directory'] - pip_create_virtualenv() + pip_create_virtualenv(os.path.join(parent_dir, 'venv')) for p in projects['repositories']: repo = p['repository'] @@ -657,16 +657,16 @@ def git_src_dir(projects_yaml, project): return None -def git_http_proxy(projects_yaml): +def git_yaml_value(projects_yaml, key): """ - Return the http_proxy value if it is specified in projects_yaml. + Return the value in projects_yaml for the specified key. """ if not projects_yaml: - return + return None projects = yaml.load(projects_yaml) - if 'http_proxy' in projects.keys(): - return projects['http_proxy'] + if key in projects.keys(): + return projects[key] return None diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 740eaa51..12838d24 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -30,6 +30,8 @@ except ImportError: apt_install('python-pip') from pip import main as pip_execute +pip_venv_path = None + __author__ = "Jorge Niedbalski " @@ -57,7 +59,7 @@ def pip_install_requirements(requirements, **options): def pip_install(package, fatal=False, upgrade=False, venv=False, **options): """Install a python package""" if venv: - venv_python = os.path.join(charm_dir(), 'venv/bin/pip') + venv_python = os.path.join(pip_venv_path, 'bin/pip') command = [venv_python, "install"] else: command = ["install"] @@ -106,7 +108,19 @@ def pip_list(): return pip_execute(["list"]) -def pip_create_virtualenv(): +def pip_create_virtualenv(path=None): """Create an isolated Python environment.""" + global pip_venv_path + apt_install('python-virtualenv') - subprocess.check_call(['virtualenv', os.path.join(charm_dir(), 'venv')]) + + if path: + pip_venv_path = path + else: + pip_venv_path = os.path.join(charm_dir(), 'venv') + + subprocess.check_call(['virtualenv', pip_venv_path]) + + +def pip_get_virtualenv_path(): + return pip_venv_path From 835ab1377563ff4ef160b3b8b71e18916386c968 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 12:48:14 +0000 Subject: [PATCH 23/40] Use function to get pip venv path --- hooks/keystone_utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 6e731394..af8db35f 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -44,13 +44,14 @@ from charmhelpers.contrib.openstack.utils import ( git_install_requested, git_clone_and_install, git_src_dir, - git_http_proxy, + git_yaml_value, os_release, save_script_rc as _save_script_rc ) from charmhelpers.contrib.python.packages import ( pip_install, + pip_get_virtualenv_path, ) from charmhelpers.core.strutils import ( @@ -1700,7 +1701,7 @@ def git_pre_install(): def git_post_install(projects_yaml): """Perform keystone post-install setup.""" - http_proxy = git_http_proxy(projects_yaml) + http_proxy = git_yaml_value(projects_yaml, 'http_proxy') if http_proxy: pip_install('mysql-python', proxy=http_proxy, venv=True) else: @@ -1717,7 +1718,7 @@ def git_post_install(projects_yaml): shutil.copytree(configs['src'], configs['dest']) symlinks = [ - {'src': os.path.join(charm_dir(), 'venv/bin/keystone-manage'), + {'src': os.path.join(pip_get_virtualenv_path(), 'bin/keystone-manage') 'link': '/usr/local/bin/keystone-manage'}, ] @@ -1728,7 +1729,7 @@ def git_post_install(projects_yaml): render('git/logging.conf', '/etc/keystone/logging.conf', {}, perms=0o644) - bin_dir = os.path.join(charm_dir(), 'venv/bin') + bin_dir = os.path.join(pip_get_virtualenv_path(), 'bin') keystone_context = { 'service_description': 'Keystone API server', 'service_name': 'Keystone', From d114b0d9772be6f6dfe41da6c5f7b7f9622ad4d5 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 13:04:07 +0000 Subject: [PATCH 24/40] Add missing comma --- hooks/keystone_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index af8db35f..85fc2093 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -1718,7 +1718,7 @@ def git_post_install(projects_yaml): shutil.copytree(configs['src'], configs['dest']) symlinks = [ - {'src': os.path.join(pip_get_virtualenv_path(), 'bin/keystone-manage') + {'src': os.path.join(pip_get_virtualenv_path(), 'bin/keystone-manage'), 'link': '/usr/local/bin/keystone-manage'}, ] From 15de703b502fd234d7ff208588a408eace6f7e7c Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 14:40:01 +0000 Subject: [PATCH 25/40] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 23 +++++++++++++++++-- hooks/charmhelpers/contrib/python/packages.py | 18 ++++----------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 29ac7b78..c5e08ec2 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -611,9 +611,11 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, proxy=http_proxy, venv=True) + pip_install(repo_dir, proxy=http_proxy, + venv=os.path.join(parent_dir, 'venv')) else: - pip_install(repo_dir, venv=True) + pip_install(repo_dir, + venv=os.path.join(parent_dir, 'venv')) return repo_dir @@ -636,6 +638,23 @@ def _git_update_requirements(package_dir, reqs_dir): os.chdir(orig_dir) +def git_pip_venv_dir(projects_yaml): + """ + Return the pip virtualenv path. + """ + parent_dir = '/mnt/openstack-git' + + if not projects_yaml: + return + + projects = yaml.load(projects_yaml) + + if 'directory' in projects.keys(): + parent_dir = projects['directory'] + + return os.path.join(parent_dir, 'venv') + + def git_src_dir(projects_yaml, project): """ Return the directory where the specified project's source is located. diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 12838d24..88c8887d 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -30,8 +30,6 @@ except ImportError: apt_install('python-pip') from pip import main as pip_execute -pip_venv_path = None - __author__ = "Jorge Niedbalski " @@ -56,10 +54,10 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, venv=False, **options): +def pip_install(package, fatal=False, upgrade=False, venv=None, **options): """Install a python package""" if venv: - venv_python = os.path.join(pip_venv_path, 'bin/pip') + venv_python = os.path.join(venv, 'bin/pip') command = [venv_python, "install"] else: command = ["install"] @@ -110,17 +108,11 @@ def pip_list(): def pip_create_virtualenv(path=None): """Create an isolated Python environment.""" - global pip_venv_path - apt_install('python-virtualenv') if path: - pip_venv_path = path + venv_path = path else: - pip_venv_path = os.path.join(charm_dir(), 'venv') + venv_path = os.path.join(charm_dir(), 'venv') - subprocess.check_call(['virtualenv', pip_venv_path]) - - -def pip_get_virtualenv_path(): - return pip_venv_path + subprocess.check_call(['virtualenv', venv_path]) From 2612b3aa0e6a202caafab41491853ed9395bb0ac Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 14:53:05 +0000 Subject: [PATCH 26/40] Use git_pip_venv_dir to get venv path --- hooks/keystone_utils.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 85fc2093..ad81bde8 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -45,13 +45,13 @@ from charmhelpers.contrib.openstack.utils import ( git_clone_and_install, git_src_dir, git_yaml_value, + git_pip_venv_dir os_release, save_script_rc as _save_script_rc ) from charmhelpers.contrib.python.packages import ( pip_install, - pip_get_virtualenv_path, ) from charmhelpers.core.strutils import ( @@ -1703,9 +1703,11 @@ def git_post_install(projects_yaml): """Perform keystone post-install setup.""" http_proxy = git_yaml_value(projects_yaml, 'http_proxy') if http_proxy: - pip_install('mysql-python', proxy=http_proxy, venv=True) + pip_install('mysql-python', proxy=http_proxy, + venv=git_pip_venv_dir()) else: - pip_install('mysql-python', venv=True) + pip_install('mysql-python', + venv=git_pip_venv_dir()) src_etc = os.path.join(git_src_dir(projects_yaml, 'keystone'), 'etc') configs = { @@ -1718,7 +1720,7 @@ def git_post_install(projects_yaml): shutil.copytree(configs['src'], configs['dest']) symlinks = [ - {'src': os.path.join(pip_get_virtualenv_path(), 'bin/keystone-manage'), + {'src': os.path.join(git_pip_venv_dir(), 'bin/keystone-manage'), 'link': '/usr/local/bin/keystone-manage'}, ] @@ -1729,7 +1731,7 @@ def git_post_install(projects_yaml): render('git/logging.conf', '/etc/keystone/logging.conf', {}, perms=0o644) - bin_dir = os.path.join(pip_get_virtualenv_path(), 'bin') + bin_dir = os.path.join(git_pip_venv_dir(), 'bin') keystone_context = { 'service_description': 'Keystone API server', 'service_name': 'Keystone', From f238fd57d298ed788d790ca718e72f8b588802de Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 15:10:49 +0000 Subject: [PATCH 27/40] Add missing params --- hooks/keystone_utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index ad81bde8..b73b376d 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -1704,10 +1704,10 @@ def git_post_install(projects_yaml): http_proxy = git_yaml_value(projects_yaml, 'http_proxy') if http_proxy: pip_install('mysql-python', proxy=http_proxy, - venv=git_pip_venv_dir()) + venv=git_pip_venv_dir(projects_yaml)) else: pip_install('mysql-python', - venv=git_pip_venv_dir()) + venv=git_pip_venv_dir(projects_yaml)) src_etc = os.path.join(git_src_dir(projects_yaml, 'keystone'), 'etc') configs = { @@ -1720,7 +1720,8 @@ def git_post_install(projects_yaml): shutil.copytree(configs['src'], configs['dest']) symlinks = [ - {'src': os.path.join(git_pip_venv_dir(), 'bin/keystone-manage'), + {'src': os.path.join(git_pip_venv_dir(projects_yaml), + 'bin/keystone-manage'), 'link': '/usr/local/bin/keystone-manage'}, ] @@ -1731,7 +1732,7 @@ def git_post_install(projects_yaml): render('git/logging.conf', '/etc/keystone/logging.conf', {}, perms=0o644) - bin_dir = os.path.join(git_pip_venv_dir(), 'bin') + bin_dir = os.path.join(git_pip_venv_dir(projects_yaml), 'bin') keystone_context = { 'service_description': 'Keystone API server', 'service_name': 'Keystone', From 0fe51a65741347f8d8147a0074b980fb776093d7 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 15:20:51 +0000 Subject: [PATCH 28/40] Add missing comma --- hooks/keystone_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index b73b376d..dddc6f8c 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -45,7 +45,7 @@ from charmhelpers.contrib.openstack.utils import ( git_clone_and_install, git_src_dir, git_yaml_value, - git_pip_venv_dir + git_pip_venv_dir, os_release, save_script_rc as _save_script_rc ) From d7d2fd47d16e62dc3eb43e40f15316c1439de4b8 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 17:05:18 +0000 Subject: [PATCH 29/40] Unit test updates --- unit_tests/test_keystone_hooks.py | 9 ++++----- unit_tests/test_keystone_utils.py | 13 +++++++++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index c5a2d33b..941f1ae8 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -117,11 +117,10 @@ class KeystoneRelationTests(CharmTestCase): self.configure_installation_source.assert_called_with(repo) self.assertTrue(self.apt_update.called) self.apt_install.assert_called_with( - ['haproxy', 'unison', 'python-setuptools', 'python-six', 'uuid', - 'python-mysqldb', 'python-pip', 'openssl', - 'apache2', 'pwgen', 'libxslt1-dev', - 'python-psycopg2', 'zlib1g-dev', 'python-dev', - 'libxml2-dev'], + ['haproxy', 'unison', 'python-setuptools', 'python-keystoneclient', + 'uuid', 'python-mysqldb', 'libmysqlclient-dev', 'openssl', + 'apache2', 'python-pip', 'pwgen', 'python-six', 'libxslt1-dev', + 'python-psycopg2', 'zlib1g-dev', 'python-dev', 'libxml2-dev'], fatal=True) self.git_install.assert_called_with(projects_yaml) diff --git a/unit_tests/test_keystone_utils.py b/unit_tests/test_keystone_utils.py index 7d918cf1..7bfd330b 100644 --- a/unit_tests/test_keystone_utils.py +++ b/unit_tests/test_keystone_utils.py @@ -656,19 +656,28 @@ class TestKeystoneUtils(CharmTestCase): @patch.object(utils, 'git_src_dir') @patch.object(utils, 'service_restart') @patch.object(utils, 'render') + @patch.object(utils, 'git_pip_venv_dir') @patch('os.path.join') @patch('os.path.exists') + @patch('os.symlink') @patch('shutil.copytree') @patch('shutil.rmtree') - def test_git_post_install(self, rmtree, copytree, exists, join, render, - service_restart, git_src_dir): + @patch('subprocess.check_call') + def test_git_post_install(self, check_call, rmtree, copytree, symlink, + exists, join, venv, render, service_restart, + git_src_dir): projects_yaml = openstack_origin_git join.return_value = 'joined-string' + venv.return_value = '/mnt/openstack-git/venv' utils.git_post_install(projects_yaml) expected = [ call('joined-string', '/etc/keystone'), ] copytree.assert_has_calls(expected) + expected = [ + call('joined-string', '/usr/local/bin/keystone-manage'), + ] + symlink.assert_has_calls(expected, any_order=True) keystone_context = { 'service_description': 'Keystone API server', 'service_name': 'Keystone', From 4289274404b92b4c66bfb81144afb931bdd25fc5 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 18:54:11 +0000 Subject: [PATCH 30/40] Sync charm-helpers --- hooks/charmhelpers/contrib/python/packages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 88c8887d..07b0c1d7 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -115,4 +115,5 @@ def pip_create_virtualenv(path=None): else: venv_path = os.path.join(charm_dir(), 'venv') - subprocess.check_call(['virtualenv', venv_path]) + if not os.path.exists(venv_path): + subprocess.check_call(['virtualenv', venv_path]) From 37e4ccc5eb18a373df47f607853916b08b74cca2 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 8 May 2015 12:36:59 +0000 Subject: [PATCH 31/40] Patch out pip_install --- unit_tests/test_keystone_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unit_tests/test_keystone_utils.py b/unit_tests/test_keystone_utils.py index 7bfd330b..0344a385 100644 --- a/unit_tests/test_keystone_utils.py +++ b/unit_tests/test_keystone_utils.py @@ -42,6 +42,7 @@ TO_PATCH = [ 'https', 'is_relation_made', 'peer_store', + 'pip_install', # generic 'apt_update', 'apt_upgrade', From ad501bcc9b7cb663ae23d45a817f9ba913e6a7b5 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 8 May 2015 15:30:22 +0000 Subject: [PATCH 32/40] Add libffi-dev to base git pkgs --- hooks/keystone_utils.py | 1 + unit_tests/test_keystone_hooks.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index dddc6f8c..6d83526e 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -129,6 +129,7 @@ BASE_PACKAGES = [ ] BASE_GIT_PACKAGES = [ + 'libffi-dev', 'libmysqlclient-dev', 'libxml2-dev', 'libxslt1-dev', diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index 941f1ae8..e10bd616 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -119,8 +119,9 @@ class KeystoneRelationTests(CharmTestCase): self.apt_install.assert_called_with( ['haproxy', 'unison', 'python-setuptools', 'python-keystoneclient', 'uuid', 'python-mysqldb', 'libmysqlclient-dev', 'openssl', - 'apache2', 'python-pip', 'pwgen', 'python-six', 'libxslt1-dev', - 'python-psycopg2', 'zlib1g-dev', 'python-dev', 'libxml2-dev'], + 'libffi-dev', 'apache2', 'python-pip', 'pwgen', 'python-six', + 'libxslt1-dev', 'python-psycopg2', 'zlib1g-dev', 'python-dev', + 'libxml2-dev'], fatal=True) self.git_install.assert_called_with(projects_yaml) From f4a152e9f3bcd221da125aadd467445b4c8ca722 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 8 May 2015 15:33:39 +0000 Subject: [PATCH 33/40] Add libssl-dev to base git pkgs --- hooks/keystone_utils.py | 1 + unit_tests/test_keystone_hooks.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 6d83526e..ea38cb72 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -131,6 +131,7 @@ BASE_PACKAGES = [ BASE_GIT_PACKAGES = [ 'libffi-dev', 'libmysqlclient-dev', + 'libssl-dev', 'libxml2-dev', 'libxslt1-dev', 'python-dev', diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index e10bd616..b9efb895 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -118,10 +118,10 @@ class KeystoneRelationTests(CharmTestCase): self.assertTrue(self.apt_update.called) self.apt_install.assert_called_with( ['haproxy', 'unison', 'python-setuptools', 'python-keystoneclient', - 'uuid', 'python-mysqldb', 'libmysqlclient-dev', 'openssl', - 'libffi-dev', 'apache2', 'python-pip', 'pwgen', 'python-six', - 'libxslt1-dev', 'python-psycopg2', 'zlib1g-dev', 'python-dev', - 'libxml2-dev'], + 'uuid', 'python-mysqldb', 'libmysqlclient-dev', 'libssl-dev', + 'openssl', 'libffi-dev', 'apache2', 'python-pip', 'pwgen', + 'python-six', 'libxslt1-dev', 'python-psycopg2', 'zlib1g-dev', + 'python-dev', 'libxml2-dev'], fatal=True) self.git_install.assert_called_with(projects_yaml) From 9b4fb260260eb6c9826536243c8340e35e112773 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 11 May 2015 12:32:39 +0000 Subject: [PATCH 34/40] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index c5e08ec2..774eaba1 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -501,6 +501,16 @@ def git_install_requested(): requirements_dir = None +def _git_yaml_load(projects_yaml): + """ + Load the specified yaml into a dictionary. + """ + if not projects_yaml: + return None + + projects = yaml.load(projects_yaml) + + def git_clone_and_install(projects_yaml, core_project, depth=1): """ Clone/install all specified OpenStack repositories. @@ -523,10 +533,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): parent_dir = '/mnt/openstack-git' http_proxy = None - if not projects_yaml: - return - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) _git_validate_projects_yaml(projects, core_project) old_environ = dict(os.environ) @@ -644,10 +651,7 @@ def git_pip_venv_dir(projects_yaml): """ parent_dir = '/mnt/openstack-git' - if not projects_yaml: - return - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) if 'directory' in projects.keys(): parent_dir = projects['directory'] @@ -661,10 +665,7 @@ def git_src_dir(projects_yaml, project): """ parent_dir = '/mnt/openstack-git' - if not projects_yaml: - return - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) if 'directory' in projects.keys(): parent_dir = projects['directory'] @@ -680,10 +681,7 @@ def git_yaml_value(projects_yaml, key): """ Return the value in projects_yaml for the specified key. """ - if not projects_yaml: - return None - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) if key in projects.keys(): return projects[key] From f24c091e49314246ff5a99247951d7232b07fe9a Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 11 May 2015 12:36:44 +0000 Subject: [PATCH 35/40] Add comment to fix bin symlinks --- hooks/keystone_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index ea38cb72..a724136b 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -1721,6 +1721,7 @@ def git_post_install(projects_yaml): shutil.rmtree(configs['dest']) shutil.copytree(configs['src'], configs['dest']) + # NOTE(coreycb): Need to find better solution than bin symlinks. symlinks = [ {'src': os.path.join(git_pip_venv_dir(projects_yaml), 'bin/keystone-manage'), From 0ed9fb4abf3c49777180aefb665f29e541c0eb26 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 11 May 2015 12:45:53 +0000 Subject: [PATCH 36/40] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 774eaba1..8bd2356e 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -508,7 +508,7 @@ def _git_yaml_load(projects_yaml): if not projects_yaml: return None - projects = yaml.load(projects_yaml) + return yaml.load(projects_yaml) def git_clone_and_install(projects_yaml, core_project, depth=1): From c93ba8bfb6923062d31d33f560a983dc3def40fe Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 12 May 2015 14:24:29 +0000 Subject: [PATCH 37/40] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 8bd2356e..d795a358 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -524,8 +524,8 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): repository: 'git://git.openstack.org/openstack/requirements.git', branch: 'stable/icehouse'} directory: /mnt/openstack-git - http_proxy: http://squid.internal:3128 - https_proxy: https://squid.internal:3128 + http_proxy: squid-proxy-url + https_proxy: squid-proxy-url The directory, http_proxy, and https_proxy keys are optional. """ From 42ad495cce956e57e1a8957de73bbc0b685bc5e0 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 12 May 2015 14:49:27 +0000 Subject: [PATCH 38/40] Clone from github in deploy from source amulet tests --- tests/basic_deployment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 76cc8328..efd5e3ae 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -59,10 +59,10 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): openstack_origin_git = { 'repositories': [ {'name': 'requirements', - 'repository': 'git://git.openstack.org/openstack/requirements', + 'repository': 'git://github.com/openstack/requirements', 'branch': branch}, {'name': 'keystone', - 'repository': 'git://git.openstack.org/openstack/keystone', + 'repository': 'git://github.com/openstack/keystone', 'branch': branch}, ], 'directory': '/mnt/openstack-git', From a29ba80b15ce6e3b1b8aa463a0723fe39d6c8834 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 12 May 2015 19:49:40 +0000 Subject: [PATCH 39/40] Add libyaml-dev as base git package --- hooks/keystone_utils.py | 1 + unit_tests/test_keystone_hooks.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index a724136b..a8c40be0 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -134,6 +134,7 @@ BASE_GIT_PACKAGES = [ 'libssl-dev', 'libxml2-dev', 'libxslt1-dev', + 'libyaml-dev', 'python-dev', 'python-pip', 'python-setuptools', diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index b9efb895..366bbc68 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -120,8 +120,8 @@ class KeystoneRelationTests(CharmTestCase): ['haproxy', 'unison', 'python-setuptools', 'python-keystoneclient', 'uuid', 'python-mysqldb', 'libmysqlclient-dev', 'libssl-dev', 'openssl', 'libffi-dev', 'apache2', 'python-pip', 'pwgen', - 'python-six', 'libxslt1-dev', 'python-psycopg2', 'zlib1g-dev', - 'python-dev', 'libxml2-dev'], + 'python-six', 'libxslt1-dev', 'python-psycopg2', 'libyaml-dev', + 'zlib1g-dev', 'python-dev', 'libxml2-dev'], fatal=True) self.git_install.assert_called_with(projects_yaml) From aed7a422161b0263656f84752d609f5acd5b31a1 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 27 May 2015 13:02:00 +0000 Subject: [PATCH 40/40] Sync charm-helpers --- hooks/charmhelpers/fetch/giturl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 7d842d49..ddc25b7e 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -45,7 +45,7 @@ class GitUrlFetchHandler(BaseFetchHandler): else: return True - def clone(self, source, dest, branch, depth): + def clone(self, source, dest, branch, depth=None): if not self.can_handle(source): raise UnhandledSource("Cannot handle {}".format(source))