From faaee49a6f0fbd152a63b1b97a6cebb9926929eb Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 20 Jan 2015 15:11:51 +0000 Subject: [PATCH 01/33] Initial support for deploying from git. --- config.yaml | 16 ++++ config/git-juno-minimal.yaml | 6 ++ config/git-tip-all.yaml | 57 +++++++++++ config/git-tip-minimal.yaml | 6 ++ hooks/glance_relations.py | 32 ++++--- hooks/glance_utils.py | 160 +++++++++++++++++++++++++++++-- templates/upstart/glance.upstart | 11 +++ unit_tests/test_glance_utils.py | 6 +- 8 files changed, 275 insertions(+), 19 deletions(-) create mode 100644 config/git-juno-minimal.yaml create mode 100644 config/git-tip-all.yaml create mode 100644 config/git-tip-minimal.yaml create mode 100644 templates/upstart/glance.upstart diff --git a/config.yaml b/config.yaml index 87ec9bf9..1dccec21 100644 --- a/config.yaml +++ b/config.yaml @@ -14,6 +14,22 @@ options: Note that updating this setting to a source that is known to provide a later version of OpenStack will trigger a software upgrade. + + Note that when openstack-origin-git is specified, the packages + configured for that option will be installed from git rather + than from the openstack-origin repository. + openstack-origin-git: + default: None + type: "string" + description: | + Specifies a YAML file which lists the git repositories and branches + from which to install OpenStack. See config/git-*.yaml for a + starting point. + + Note that the config files that are installed will be determined + based on the OpenStack release in the openstack-origin option. + + Note that updating this setting after deployment will do nothing. database-user: default: glance type: string diff --git a/config/git-juno-minimal.yaml b/config/git-juno-minimal.yaml new file mode 100644 index 00000000..d0998bd0 --- /dev/null +++ b/config/git-juno-minimal.yaml @@ -0,0 +1,6 @@ +glance: + repository: git://git.openstack.org/openstack/glance.git + branch: stable/juno +requirements: + repository: git://git.openstack.org/openstack/requirements.git + branch: stable/juno diff --git a/config/git-tip-all.yaml b/config/git-tip-all.yaml new file mode 100644 index 00000000..1350ecd0 --- /dev/null +++ b/config/git-tip-all.yaml @@ -0,0 +1,57 @@ +glance: + repository: git://git.openstack.org/openstack/glance.git + branch: master +glance-store: + repository: git://git.openstack.org/openstack/glance_store.git + branch: master +keystonemiddleware: + repository: git://git.openstack.org/openstack/keystonemiddleware.git + branch: master +oslo-config: + repository: git://git.openstack.org/openstack/oslo.config.git + branch: master +oslo-concurrency: + repository: git://git.openstack.org/openstack/oslo.concurrency.git + branch: master +oslo-db: + repository: git://git.openstack.org/openstack/oslo.db.git + branch: master +oslo-i18n: + repository: git://git.openstack.org/openstack/oslo.i18n.git + branch: master +oslo-messaging: + repository: git://git.openstack.org/openstack/oslo.messaging.git + branch: master +oslo-serialization: + repository: git://git.openstack.org/openstack/oslo.serialization.git + branch: master +oslo-utils: + repository: git://git.openstack.org/openstack/oslo.utils.git + branch: master +oslo-vmware: + repository: git://git.openstack.org/openstack/oslo.vmware.git + branch: master +osprofiler: + repository: git://git.openstack.org/stackforge/osprofiler.git + branch: master +pbr: + repository: git://git.openstack.org/openstack-dev/pbr.git + branch: master +python-keystoneclient: + repository: git://git.openstack.org/openstack/python-keystoneclient.git + branch: master +python-swiftclient: + repository: git://git.openstack.org/openstack/python-swiftclient.git + branch: master +requirements: + repository: git://git.openstack.org/openstack/requirements.git + branch: master +stevedore: + repository: git://git.openstack.org/openstack/stevedore.git + branch: master +sqlalchemy-migrate: + repository: git://git.openstack.org/stackforge/sqlalchemy-migrate.git + branch: master +wsme: + repository: git://git.openstack.org/stackforge/wsme.git + branch: master diff --git a/config/git-tip-minimal.yaml b/config/git-tip-minimal.yaml new file mode 100644 index 00000000..486c5306 --- /dev/null +++ b/config/git-tip-minimal.yaml @@ -0,0 +1,6 @@ +glance: + repository: git://git.openstack.org/openstack/glance.git + branch: master +requirements: + repository: git://git.openstack.org/openstack/requirements.git + branch: master diff --git a/hooks/glance_relations.py b/hooks/glance_relations.py index 04b2dc83..c21dedc1 100755 --- a/hooks/glance_relations.py +++ b/hooks/glance_relations.py @@ -7,12 +7,13 @@ import sys from glance_utils import ( do_openstack_upgrade, + git_install, migrate_database, register_configs, restart_map, services, CLUSTER_RES, - PACKAGES, + determine_packages, SERVICES, CHARM, GLANCE_REGISTRY_CONF, @@ -54,10 +55,11 @@ from charmhelpers.contrib.hahelpers.cluster import ( ) from charmhelpers.contrib.openstack.utils import ( configure_installation_source, - get_os_codename_package, - openstack_upgrade_available, + git_install_requested, lsb_release, - sync_db_with_multi_ipv6_addresses + openstack_upgrade_available, + os_release, + sync_db_with_multi_ipv6_addresses, ) from charmhelpers.contrib.storage.linux.ceph import ( ensure_ceph_keyring, @@ -100,7 +102,8 @@ def install_hook(): configure_installation_source(src) apt_update(fatal=True) - apt_install(PACKAGES, fatal=True) + apt_install(determine_packages(), fatal=True) + git_install(config('openstack-origin-git')) for service in SERVICES: service_stop(service) @@ -140,7 +143,7 @@ def pgsql_db_joined(): @hooks.hook('shared-db-relation-changed') @restart_on_change(restart_map()) def db_changed(): - rel = get_os_codename_package("glance-common") + rel = os_release('glance-common') if 'shared-db' not in CONFIGS.complete_contexts(): juju_log('shared-db relation incomplete. Peer not ready?') @@ -172,7 +175,7 @@ def db_changed(): @hooks.hook('pgsql-db-relation-changed') @restart_on_change(restart_map()) def pgsql_db_changed(): - rel = get_os_codename_package("glance-common") + rel = os_release('glance-common') if 'pgsql-db' not in CONFIGS.complete_contexts(): juju_log('pgsql-db relation incomplete. Peer not ready?') @@ -308,9 +311,10 @@ def config_changed(): sync_db_with_multi_ipv6_addresses(config('database'), config('database-user')) - if openstack_upgrade_available('glance-common'): - juju_log('Upgrading OpenStack release') - do_openstack_upgrade(CONFIGS) + if not git_install_requested(): + if openstack_upgrade_available('glance-common'): + juju_log('Upgrading OpenStack release') + do_openstack_upgrade(CONFIGS) open_port(9292) configure_https() @@ -324,6 +328,12 @@ def config_changed(): [cluster_joined(rid) for rid in relation_ids('cluster')] +#TODO(coreycb): For deploy from git support, need to implement action-set +# and action-get to trigger re-install of git-installed +# services. IIUC they'd be triggered via: +# juju do + + @hooks.hook('cluster-relation-joined') def cluster_joined(relation_id=None): for addr_type in ADDRESS_TYPES: @@ -352,7 +362,7 @@ def cluster_changed(): @hooks.hook('upgrade-charm') @restart_on_change(restart_map(), stopstart=True) def upgrade_charm(): - apt_install(filter_installed_packages(PACKAGES), fatal=True) + apt_install(filter_installed_packages(determine_packages()), fatal=True) configure_https() update_nrpe_config() CONFIGS.write_all() diff --git a/hooks/glance_utils.py b/hooks/glance_utils.py index f1d6139c..f101d9d4 100755 --- a/hooks/glance_utils.py +++ b/hooks/glance_utils.py @@ -1,6 +1,7 @@ #!/usr/bin/python import os +import shutil import subprocess import glance_contexts @@ -14,21 +15,26 @@ from charmhelpers.fetch import ( add_source) from charmhelpers.core.hookenv import ( + charm_dir, config, log, relation_ids, service_name) from charmhelpers.core.host import ( + adduser, + add_group, + add_user_to_group, mkdir, service_stop, service_start, - lsb_release + lsb_release, + write_file, ) from charmhelpers.contrib.openstack import ( templating, - context, ) + context,) from charmhelpers.contrib.hahelpers.cluster import ( eligible_leader, @@ -38,7 +44,13 @@ from charmhelpers.contrib.openstack.alternatives import install_alternative from charmhelpers.contrib.openstack.utils import ( get_os_codename_install_source, get_os_codename_package, - configure_installation_source) + git_install_requested, + git_clone_and_install, + configure_installation_source, + os_release, +) + +from charmhelpers.core.templating import render CLUSTER_RES = "grp_glance_vips" @@ -46,8 +58,27 @@ PACKAGES = [ "apache2", "glance", "python-mysqldb", "python-swiftclient", "python-psycopg2", "python-keystone", "python-six", "uuid", "haproxy", ] +BASE_GIT_PACKAGES = [ + 'libxml2-dev', + 'libxslt1-dev', + 'python-dev', + 'python-pip', + 'python-setuptools', + 'zlib1g-dev', +] + SERVICES = [ - "glance-api", "glance-registry", ] + "glance-api", + "glance-registry", +] + +# ubuntu packages that should not be installed when deploying from git +GIT_PACKAGE_BLACKLIST = [ + 'glance', + 'python-swiftclient', + 'python-keystone', +] + CHARM = "glance" @@ -136,7 +167,7 @@ def register_configs(): # Register config files with their respective contexts. # Regstration of some configs may not be required depending on # existing of certain relations. - release = get_os_codename_package('glance-common', fatal=False) or 'essex' + release = os_release('glance-common') configs = templating.OSConfigRenderer(templates_dir=TEMPLATES, openstack_release=release) @@ -173,6 +204,18 @@ def register_configs(): return configs +def determine_packages(): + packages = [] + PACKAGES + + if git_install_requested(): + packages.extend(BASE_GIT_PACKAGES) + # don't include packages that will be installed from git + for p in GIT_PACKAGE_BLACKLIST: + packages.remove(p) + + return list(set(packages)) + + def migrate_database(): '''Runs glance-manage to initialize a new database or migrate existing @@ -201,7 +244,7 @@ def do_openstack_upgrade(configs): ] apt_update() apt_upgrade(options=dpkg_opts, fatal=True, dist=True) - apt_install(PACKAGES, fatal=True) + apt_install(packages=determine_packages(), fatal=True) # set CONFIGS to load templates from new release and regenerate config configs.set_release(openstack_release=new_os_rel) @@ -252,3 +295,108 @@ def setup_ipv6(): ' main') apt_update() apt_install('haproxy/trusty-backports', fatal=True) + + +def git_install(file_name): + """Perform setup, and install git repos specified in yaml config file.""" + if git_install_requested(): + git_pre_install() + git_clone_and_install(file_name, core_project='glance') + git_post_install() + + +def git_pre_install(): + """Perform pre glance installation setup.""" + dirs = [ + '/var/lib/glance', + '/var/lib/glance/images', + '/var/lib/glance/image-cache', + '/var/lib/glance/image-cache/incomplete', + '/var/lib/glance/image-cache/invalid', + '/var/lib/glance/image-cache/queue', + '/var/log/glance', + '/etc/glance', + ] + + logs = [ + '/var/log/glance/glance-api.log', + '/var/log/glance/glance-registry.log', + ] + + adduser('glance', shell='/bin/bash', system_user=True) + add_group('glance', system_group=True) + add_user_to_group('glance', 'glance') + + for d in dirs: + mkdir(d, owner='glance', group='glance', perms=0700, force=False) + + for l in logs: + write_file(l, '', owner='glance', group='glance', perms=0600) + + +def git_post_install(): + """Perform post glance installation setup.""" + src_etc = os.path.join(charm_dir(), '/mnt/openstack-git/glance.git/etc/') + configs = { + 'glance-api-paste': { + 'src': os.path.join(src_etc, 'glance-api-paste.ini'), + 'dest': '/etc/glance/glance-api-paste.ini', + }, + 'glance-api': { + 'src': os.path.join(src_etc, 'glance-api.conf'), + 'dest': '/etc/glance/glance-api.conf', + }, + 'glance-registry-paste': { + 'src': os.path.join(src_etc, 'glance-registry-paste.ini'), + 'dest': '/etc/glance/glance-registry-paste.ini', + }, + 'glance-registry': { + 'src': os.path.join(src_etc, 'glance-registry.conf'), + 'dest': '/etc/glance/glance-registry.conf', + }, + 'glance-cache': { + 'src': os.path.join(src_etc, 'glance-cache.conf'), + 'dest': '/etc/glance/glance-cache.conf', + }, + 'glance-scrubber': { + 'src': os.path.join(src_etc, 'glance-scrubber.conf'), + 'dest': '/etc/glance/glance-scrubber.conf', + }, + 'policy': { + 'src': os.path.join(src_etc, 'policy.json'), + 'dest': '/etc/glance/policy.json', + }, + 'schema-image': { + 'src': os.path.join(src_etc, 'schema-image.json'), + 'dest': '/etc/glance/schema-image.json', + }, + } + + for conf, files in configs.iteritems(): + shutil.copyfile(files['src'], files['dest']) + + glance_api_context = { + 'service_description': 'Glance API server', + 'service_name': 'Glance', + 'user_name': 'glance', + 'start_dir': '/var/lib/glance', + 'process_name': 'glance-api', + 'executable_name': '/usr/local/bin/glance-api', + } + + glance_registry_context = { + 'service_description': 'Glance registry server', + 'service_name': 'Glance', + 'user_name': 'glance', + 'start_dir': '/var/lib/glance', + 'process_name': 'glance-registry', + 'executable_name': '/usr/local/bin/glance-registry', + } + + render('upstart/glance.upstart', '/etc/init/glance-api.conf', + glance_api_context, perms=0o644) + render('upstart/glance.upstart', '/etc/init/glance-registry.conf', + glance_registry_context, perms=0o644) + + service_start('glance-api') + service_start('glance-registry') diff --git a/templates/upstart/glance.upstart b/templates/upstart/glance.upstart new file mode 100644 index 00000000..e5f39741 --- /dev/null +++ b/templates/upstart/glance.upstart @@ -0,0 +1,11 @@ +description "{{ service_description }}" +author "Juju {{ service_name }} Charm " + +start on runlevel [2345] +stop on runlevel [!2345] + +respawn + +exec start-stop-daemon --start --chuid {{ user_name }} \ + --chdir {{ start_dir }} --name {{ process_name }} \ + --exec {{ executable_name }} diff --git a/unit_tests/test_glance_utils.py b/unit_tests/test_glance_utils.py index 67d1105d..dcb3be03 100644 --- a/unit_tests/test_glance_utils.py +++ b/unit_tests/test_glance_utils.py @@ -130,7 +130,8 @@ class TestGlanceUtils(CharmTestCase): configs = MagicMock() utils.do_openstack_upgrade(configs) self.assertTrue(configs.write_all.called) - self.apt_install.assert_called_with(utils.PACKAGES, fatal=True) + self.apt_install.assert_called_with(utils.determine_packages(), + fatal=True) self.apt_upgrade.assert_called_with(options=DPKG_OPTS, fatal=True, dist=True) configs.set_release.assert_called_with(openstack_release='havana') @@ -145,7 +146,8 @@ class TestGlanceUtils(CharmTestCase): configs = MagicMock() utils.do_openstack_upgrade(configs) self.assertTrue(configs.write_all.called) - self.apt_install.assert_called_with(utils.PACKAGES, fatal=True) + self.apt_install.assert_called_with(utils.determine_packages(), + fatal=True) self.apt_upgrade.assert_called_with(options=DPKG_OPTS, fatal=True, dist=True) configs.set_release.assert_called_with(openstack_release='havana') From 884d0948b0641e81d1a3eaf1aa989a8de0c17844 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 7 Mar 2015 21:28:53 -0500 Subject: [PATCH 02/33] Sync charm-helpers --- hooks/charmhelpers/__init__.py | 16 ++ hooks/charmhelpers/contrib/__init__.py | 15 ++ .../contrib/charmsupport/__init__.py | 15 ++ .../charmhelpers/contrib/charmsupport/nrpe.py | 64 ++++- .../contrib/charmsupport/volumes.py | 16 ++ .../contrib/hahelpers/__init__.py | 15 ++ .../charmhelpers/contrib/hahelpers/apache.py | 16 ++ .../charmhelpers/contrib/hahelpers/cluster.py | 28 ++- .../charmhelpers/contrib/network/__init__.py | 15 ++ hooks/charmhelpers/contrib/network/ip.py | 101 +++++++- .../contrib/openstack/__init__.py | 15 ++ .../contrib/openstack/alternatives.py | 16 ++ .../contrib/openstack/amulet/__init__.py | 15 ++ .../contrib/openstack/amulet/deployment.py | 23 +- .../contrib/openstack/amulet/utils.py | 16 ++ .../charmhelpers/contrib/openstack/context.py | 88 ++++++- hooks/charmhelpers/contrib/openstack/ip.py | 53 +++++ .../charmhelpers/contrib/openstack/neutron.py | 16 ++ .../contrib/openstack/templates/__init__.py | 16 ++ .../contrib/openstack/templating.py | 16 ++ hooks/charmhelpers/contrib/openstack/utils.py | 225 ++++++++---------- hooks/charmhelpers/contrib/python/__init__.py | 15 ++ hooks/charmhelpers/contrib/python/packages.py | 23 +- .../charmhelpers/contrib/storage/__init__.py | 15 ++ .../contrib/storage/linux/__init__.py | 15 ++ .../contrib/storage/linux/ceph.py | 16 ++ hooks/charmhelpers/core/__init__.py | 15 ++ hooks/charmhelpers/core/decorators.py | 16 ++ hooks/charmhelpers/core/fstab.py | 22 +- hooks/charmhelpers/core/hookenv.py | 16 ++ hooks/charmhelpers/core/host.py | 45 +++- hooks/charmhelpers/core/services/__init__.py | 16 ++ hooks/charmhelpers/core/services/base.py | 16 ++ hooks/charmhelpers/core/services/helpers.py | 32 ++- hooks/charmhelpers/core/sysctl.py | 34 ++- hooks/charmhelpers/core/templating.py | 22 +- hooks/charmhelpers/fetch/__init__.py | 16 ++ hooks/charmhelpers/fetch/archiveurl.py | 36 ++- hooks/charmhelpers/fetch/bzrurl.py | 26 +- hooks/charmhelpers/fetch/giturl.py | 20 ++ hooks/charmhelpers/payload/__init__.py | 16 ++ hooks/charmhelpers/payload/execd.py | 16 ++ tests/charmhelpers/__init__.py | 16 ++ tests/charmhelpers/contrib/__init__.py | 15 ++ tests/charmhelpers/contrib/amulet/__init__.py | 15 ++ .../charmhelpers/contrib/amulet/deployment.py | 16 ++ tests/charmhelpers/contrib/amulet/utils.py | 140 ++++++++++- .../contrib/openstack/__init__.py | 15 ++ .../contrib/openstack/amulet/__init__.py | 15 ++ .../contrib/openstack/amulet/deployment.py | 23 +- .../contrib/openstack/amulet/utils.py | 16 ++ 51 files changed, 1332 insertions(+), 187 deletions(-) diff --git a/hooks/charmhelpers/__init__.py b/hooks/charmhelpers/__init__.py index b46e2e23..f72e7f84 100644 --- a/hooks/charmhelpers/__init__.py +++ b/hooks/charmhelpers/__init__.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # Bootstrap charm-helpers, installing its dependencies if necessary using # only standard libraries. import subprocess diff --git a/hooks/charmhelpers/contrib/__init__.py b/hooks/charmhelpers/contrib/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/__init__.py +++ b/hooks/charmhelpers/contrib/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/charmsupport/__init__.py b/hooks/charmhelpers/contrib/charmsupport/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/charmsupport/__init__.py +++ b/hooks/charmhelpers/contrib/charmsupport/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py index f3a936d0..9d961cfb 100644 --- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py +++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + """Compatibility with the nrpe-external-master charm""" # Copyright 2012 Canonical Ltd. # @@ -8,6 +24,8 @@ import subprocess import pwd import grp import os +import glob +import shutil import re import shlex import yaml @@ -145,7 +163,7 @@ define service {{ log('Check command not found: {}'.format(parts[0])) return '' - def write(self, nagios_context, hostname, nagios_servicegroups=None): + def write(self, nagios_context, hostname, nagios_servicegroups): nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( self.command) with open(nrpe_check_file, 'w') as nrpe_check_config: @@ -161,14 +179,11 @@ define service {{ nagios_servicegroups) def write_service_config(self, nagios_context, hostname, - nagios_servicegroups=None): + nagios_servicegroups): for f in os.listdir(NRPE.nagios_exportdir): if re.search('.*{}.cfg'.format(self.command), f): os.remove(os.path.join(NRPE.nagios_exportdir, f)) - if not nagios_servicegroups: - nagios_servicegroups = nagios_context - templ_vars = { 'nagios_hostname': hostname, 'nagios_servicegroup': nagios_servicegroups, @@ -195,10 +210,10 @@ class NRPE(object): super(NRPE, self).__init__() self.config = config() self.nagios_context = self.config['nagios_context'] - if 'nagios_servicegroups' in self.config: + if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']: self.nagios_servicegroups = self.config['nagios_servicegroups'] else: - self.nagios_servicegroups = 'juju' + self.nagios_servicegroups = self.nagios_context self.unit_name = local_unit().replace('/', '-') if hostname: self.hostname = hostname @@ -306,3 +321,38 @@ def add_init_service_checks(nrpe, services, unit_name): check_cmd='check_status_file.py -f ' '/var/lib/nagios/service-check-%s.txt' % svc, ) + + +def copy_nrpe_checks(): + """ + Copy the nrpe checks into place + + """ + NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins' + nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks', + 'charmhelpers', 'contrib', 'openstack', + 'files') + + if not os.path.exists(NAGIOS_PLUGINS): + os.makedirs(NAGIOS_PLUGINS) + for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")): + if os.path.isfile(fname): + shutil.copy2(fname, + os.path.join(NAGIOS_PLUGINS, os.path.basename(fname))) + + +def add_haproxy_checks(nrpe, unit_name): + """ + Add checks for each service in list + + :param NRPE nrpe: NRPE object to add check to + :param str unit_name: Unit name to use in check description + """ + nrpe.add_check( + shortname='haproxy_servers', + description='Check HAProxy {%s}' % unit_name, + check_cmd='check_haproxy.sh') + nrpe.add_check( + shortname='haproxy_queue', + description='Check HAProxy queue depth {%s}' % unit_name, + check_cmd='check_haproxy_queue_depth.sh') diff --git a/hooks/charmhelpers/contrib/charmsupport/volumes.py b/hooks/charmhelpers/contrib/charmsupport/volumes.py index d61aa47f..320961b9 100644 --- a/hooks/charmhelpers/contrib/charmsupport/volumes.py +++ b/hooks/charmhelpers/contrib/charmsupport/volumes.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + ''' Functions for managing volumes in juju units. One volume is supported per unit. Subordinates may have their own storage, provided it is on its own partition. diff --git a/hooks/charmhelpers/contrib/hahelpers/__init__.py b/hooks/charmhelpers/contrib/hahelpers/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/hahelpers/__init__.py +++ b/hooks/charmhelpers/contrib/hahelpers/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/hahelpers/apache.py b/hooks/charmhelpers/contrib/hahelpers/apache.py index 6616ffff..00917195 100644 --- a/hooks/charmhelpers/contrib/hahelpers/apache.py +++ b/hooks/charmhelpers/contrib/hahelpers/apache.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # # Copyright 2012 Canonical Ltd. # diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 912b2fe3..9333efc3 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # # Copyright 2012 Canonical Ltd. # @@ -32,6 +48,9 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.decorators import ( retry_on_exception, ) +from charmhelpers.core.strutils import ( + bool_from_string, +) class HAIncompleteConfig(Exception): @@ -148,7 +167,8 @@ def https(): . returns: boolean ''' - if config_get('use-https') == "yes": + use_https = config_get('use-https') + if use_https and bool_from_string(use_https): return True if config_get('ssl_cert') and config_get('ssl_key'): return True @@ -205,19 +225,23 @@ def determine_apache_port(public_port, singlenode_mode=False): return public_port - (i * 10) -def get_hacluster_config(): +def get_hacluster_config(exclude_keys=None): ''' Obtains all relevant configuration from charm configuration required for initiating a relation to hacluster: ha-bindiface, ha-mcastport, vip + param: exclude_keys: list of setting key(s) to be excluded. returns: dict: A dict containing settings keyed by setting name. raises: HAIncompleteConfig if settings are missing. ''' settings = ['ha-bindiface', 'ha-mcastport', 'vip'] conf = {} for setting in settings: + if exclude_keys and setting in exclude_keys: + continue + conf[setting] = config_get(setting) missing = [] [missing.append(s) for s, v in six.iteritems(conf) if v is None] diff --git a/hooks/charmhelpers/contrib/network/__init__.py b/hooks/charmhelpers/contrib/network/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/network/__init__.py +++ b/hooks/charmhelpers/contrib/network/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 8dc83165..fff6d5ca 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -1,13 +1,32 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import glob import re import subprocess +import six +import socket from functools import partial from charmhelpers.core.hookenv import unit_get from charmhelpers.fetch import apt_install from charmhelpers.core.hookenv import ( - log + log, + WARNING, ) try: @@ -349,3 +368,83 @@ def is_bridge_member(nic): return True return False + + +def is_ip(address): + """ + Returns True if address is a valid IP address. + """ + try: + # Test to see if already an IPv4 address + socket.inet_aton(address) + return True + except socket.error: + return False + + +def ns_query(address): + try: + import dns.resolver + except ImportError: + apt_install('python-dnspython') + import dns.resolver + + if isinstance(address, dns.name.Name): + rtype = 'PTR' + elif isinstance(address, six.string_types): + rtype = 'A' + else: + return None + + answers = dns.resolver.query(address, rtype) + if answers: + return str(answers[0]) + return None + + +def get_host_ip(hostname, fallback=None): + """ + Resolves the IP for a given hostname, or returns + the input if it is already an IP. + """ + if is_ip(hostname): + return hostname + + ip_addr = ns_query(hostname) + if not ip_addr: + try: + ip_addr = socket.gethostbyname(hostname) + except: + log("Failed to resolve hostname '%s'" % (hostname), + level=WARNING) + return fallback + return ip_addr + + +def get_hostname(address, fqdn=True): + """ + Resolves hostname for given IP, or returns the input + if it is already a hostname. + """ + if is_ip(address): + try: + import dns.reversename + except ImportError: + apt_install("python-dnspython") + import dns.reversename + + rev = dns.reversename.from_address(address) + result = ns_query(rev) + if not result: + return None + else: + result = address + + if fqdn: + # strip trailing . + if result.endswith('.'): + return result[:-1] + else: + return result + else: + return result.split('.')[0] diff --git a/hooks/charmhelpers/contrib/openstack/__init__.py b/hooks/charmhelpers/contrib/openstack/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/openstack/__init__.py +++ b/hooks/charmhelpers/contrib/openstack/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/openstack/alternatives.py b/hooks/charmhelpers/contrib/openstack/alternatives.py index b413259c..ef77caf3 100644 --- a/hooks/charmhelpers/contrib/openstack/alternatives.py +++ b/hooks/charmhelpers/contrib/openstack/alternatives.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + ''' Helper for managing alternatives for file conflict resolution ''' import subprocess diff --git a/hooks/charmhelpers/contrib/openstack/amulet/__init__.py b/hooks/charmhelpers/contrib/openstack/amulet/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/__init__.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index f3fee074..0cfeaa4c 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import six from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment @@ -55,16 +71,19 @@ class OpenStackAmuletDeployment(AmuletDeployment): services.append(this_service) use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', 'ceph-osd', 'ceph-radosgw'] + # Openstack subordinate charms do not expose an origin option as that + # is controlled by the principle + ignore = ['neutron-openvswitch'] if self.openstack: for svc in services: - if svc['name'] not in use_source: + if svc['name'] not in use_source + ignore: config = {'openstack-origin': self.openstack} self.d.configure(svc['name'], config) if self.source: for svc in services: - if svc['name'] in use_source: + if svc['name'] in use_source and svc['name'] not in ignore: config = {'source': self.source} self.d.configure(svc['name'], config) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index 3e0cc61c..9c3d918a 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import logging import os import time diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index eaa89a67..2d9a95cd 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import json import os import time @@ -5,6 +21,7 @@ from base64 import b64decode from subprocess import check_call import six +import yaml from charmhelpers.fetch import ( apt_install, @@ -88,9 +105,41 @@ def context_complete(ctxt): def config_flags_parser(config_flags): """Parses config flags string into dict. + This parsing method supports a few different formats for the config + flag values to be parsed: + + 1. A string in the simple format of key=value pairs, with the possibility + of specifying multiple key value pairs within the same string. For + example, a string in the format of 'key1=value1, key2=value2' will + return a dict of: + {'key1': 'value1', + 'key2': 'value2'}. + + 2. A string in the above format, but supporting a comma-delimited list + of values for the same key. For example, a string in the format of + 'key1=value1, key2=value3,value4,value5' will return a dict of: + {'key1', 'value1', + 'key2', 'value2,value3,value4'} + + 3. A string containing a colon character (:) prior to an equal + character (=) will be treated as yaml and parsed as such. This can be + used to specify more complex key value pairs. For example, + a string in the format of 'key1: subkey1=value1, subkey2=value2' will + return a dict of: + {'key1', 'subkey1=value1, subkey2=value2'} + The provided config_flags string may be a list of comma-separated values which themselves may be comma-separated list of values. """ + # If we find a colon before an equals sign then treat it as yaml. + # Note: limit it to finding the colon first since this indicates assignment + # for inline yaml. + colon = config_flags.find(':') + equals = config_flags.find('=') + if colon > 0: + if colon < equals or equals < 0: + return yaml.safe_load(config_flags) + if config_flags.find('==') >= 0: log("config_flags is not in expected format (key=value)", level=ERROR) raise OSContextError @@ -175,7 +224,7 @@ class SharedDBContext(OSContextGenerator): unit=local_unit()) if set_hostname != access_hostname: relation_set(relation_settings={hostname_key: access_hostname}) - return ctxt # Defer any further hook execution for now.... + return None # Defer any further hook execution for now.... password_setting = 'password' if self.relation_prefix: @@ -263,9 +312,25 @@ def db_ssl(rdata, ctxt, ssl_dir): class IdentityServiceContext(OSContextGenerator): interfaces = ['identity-service'] + def __init__(self, service=None, service_user=None): + self.service = service + self.service_user = service_user + def __call__(self): log('Generating template context for identity-service', level=DEBUG) ctxt = {} + + if self.service and self.service_user: + # This is required for pki token signing if we don't want /tmp to + # be used. + cachedir = '/var/cache/%s' % (self.service) + if not os.path.isdir(cachedir): + log("Creating service cache dir %s" % (cachedir), level=DEBUG) + mkdir(path=cachedir, owner=self.service_user, + group=self.service_user, perms=0o700) + + ctxt['signing_dir'] = cachedir + for rid in relation_ids('identity-service'): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) @@ -275,15 +340,16 @@ class IdentityServiceContext(OSContextGenerator): auth_host = format_ipv6_addr(auth_host) or auth_host svc_protocol = rdata.get('service_protocol') or 'http' auth_protocol = rdata.get('auth_protocol') or 'http' - ctxt = {'service_port': rdata.get('service_port'), - 'service_host': serv_host, - 'auth_host': auth_host, - 'auth_port': rdata.get('auth_port'), - 'admin_tenant_name': rdata.get('service_tenant'), - 'admin_user': rdata.get('service_username'), - 'admin_password': rdata.get('service_password'), - 'service_protocol': svc_protocol, - 'auth_protocol': auth_protocol} + ctxt.update({'service_port': rdata.get('service_port'), + 'service_host': serv_host, + 'auth_host': auth_host, + 'auth_port': rdata.get('auth_port'), + 'admin_tenant_name': rdata.get('service_tenant'), + 'admin_user': rdata.get('service_username'), + 'admin_password': rdata.get('service_password'), + 'service_protocol': svc_protocol, + 'auth_protocol': auth_protocol}) + if context_complete(ctxt): # NOTE(jamespage) this is required for >= icehouse # so a missing value just indicates keystone needs @@ -1005,6 +1071,8 @@ class ZeroMQContext(OSContextGenerator): for unit in related_units(rid): ctxt['zmq_nonce'] = relation_get('nonce', unit, rid) ctxt['zmq_host'] = relation_get('host', unit, rid) + ctxt['zmq_redis_address'] = relation_get( + 'zmq_redis_address', unit, rid) return ctxt diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py index f062c807..29bbddcb 100644 --- a/hooks/charmhelpers/contrib/openstack/ip.py +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + from charmhelpers.core.hookenv import ( config, unit_get, @@ -10,6 +26,8 @@ from charmhelpers.contrib.network.ip import ( ) from charmhelpers.contrib.hahelpers.cluster import is_clustered +from functools import partial + PUBLIC = 'public' INTERNAL = 'int' ADMIN = 'admin' @@ -91,3 +109,38 @@ def resolve_address(endpoint_type=PUBLIC): "clustered=%s)" % (net_type, clustered)) return resolved_address + + +def endpoint_url(configs, url_template, port, endpoint_type=PUBLIC, + override=None): + """Returns the correct endpoint URL to advertise to Keystone. + + This method provides the correct endpoint URL which should be advertised to + the keystone charm for endpoint creation. This method allows for the url to + be overridden to force a keystone endpoint to have specific URL for any of + the defined scopes (admin, internal, public). + + :param configs: OSTemplateRenderer config templating object to inspect + for a complete https context. + :param url_template: str format string for creating the url template. Only + two values will be passed - the scheme+hostname + returned by the canonical_url and the port. + :param endpoint_type: str endpoint type to resolve. + :param override: str the name of the config option which overrides the + endpoint URL defined by the charm itself. None will + disable any overrides (default). + """ + if override: + # Return any user-defined overrides for the keystone endpoint URL. + user_value = config(override) + if user_value: + return user_value.strip() + + return url_template % (canonical_url(configs, endpoint_type), port) + + +public_endpoint = partial(endpoint_url, endpoint_type=PUBLIC) + +internal_endpoint = partial(endpoint_url, endpoint_type=INTERNAL) + +admin_endpoint = partial(endpoint_url, endpoint_type=ADMIN) diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 095cc24b..902757fe 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # Various utilies for dealing with Neutron and the renaming from Quantum. from subprocess import check_output diff --git a/hooks/charmhelpers/contrib/openstack/templates/__init__.py b/hooks/charmhelpers/contrib/openstack/templates/__init__.py index 0b49ad28..75876796 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/__init__.py +++ b/hooks/charmhelpers/contrib/openstack/templates/__init__.py @@ -1,2 +1,18 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # dummy __init__.py to fool syncer into thinking this is a syncable python # module diff --git a/hooks/charmhelpers/contrib/openstack/templating.py b/hooks/charmhelpers/contrib/openstack/templating.py index 33df0675..24cb272b 100644 --- a/hooks/charmhelpers/contrib/openstack/templating.py +++ b/hooks/charmhelpers/contrib/openstack/templating.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import six diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index ddd40ce5..0293c7d7 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -1,18 +1,37 @@ #!/usr/bin/python +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # Common python helper functions used for OpenStack charms. from collections import OrderedDict from functools import wraps +import errno import subprocess import json import os -import socket import sys +import time import six import yaml +from charmhelpers.contrib.network import ip + from charmhelpers.core.hookenv import ( config, log as juju_log, @@ -87,6 +106,7 @@ SWIFT_CODENAMES = OrderedDict([ ('2.1.0', 'juno'), ('2.2.0', 'juno'), ('2.2.1', 'kilo'), + ('2.2.2', 'kilo'), ]) DEFAULT_LOOPBACK_SIZE = '5G' @@ -404,77 +424,10 @@ def clean_storage(block_device): else: zap_disk(block_device) - -def is_ip(address): - """ - Returns True if address is a valid IP address. - """ - try: - # Test to see if already an IPv4 address - socket.inet_aton(address) - return True - except socket.error: - return False - - -def ns_query(address): - try: - import dns.resolver - except ImportError: - apt_install('python-dnspython') - import dns.resolver - - if isinstance(address, dns.name.Name): - rtype = 'PTR' - elif isinstance(address, six.string_types): - rtype = 'A' - else: - return None - - answers = dns.resolver.query(address, rtype) - if answers: - return str(answers[0]) - return None - - -def get_host_ip(hostname): - """ - Resolves the IP for a given hostname, or returns - the input if it is already an IP. - """ - if is_ip(hostname): - return hostname - - return ns_query(hostname) - - -def get_hostname(address, fqdn=True): - """ - Resolves hostname for given IP, or returns the input - if it is already a hostname. - """ - if is_ip(address): - try: - import dns.reversename - except ImportError: - apt_install('python-dnspython') - import dns.reversename - - rev = dns.reversename.from_address(address) - result = ns_query(rev) - if not result: - return None - else: - result = address - - if fqdn: - # strip trailing . - if result.endswith('.'): - return result[:-1] - else: - return result - else: - return result.split('.')[0] +is_ip = ip.is_ip +ns_query = ip.ns_query +get_host_ip = ip.get_host_ip +get_hostname = ip.get_hostname def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'): @@ -519,89 +472,115 @@ def os_requires_version(ostack_release, pkg): def git_install_requested(): """Returns true if openstack-origin-git is specified.""" - return config('openstack-origin-git') != "None" + return config('openstack-origin-git') != None requirements_dir = None -def git_clone_and_install(file_name, core_project): - """Clone/install all OpenStack repos specified in yaml config file.""" +def git_clone_and_install(projects, core_project, + parent_dir='/mnt/openstack-git'): + """Clone/install all OpenStack repos specified in projects dictionary.""" global requirements_dir + update_reqs = True - if file_name == "None": + if not projects: return - yaml_file = os.path.join(charm_dir(), file_name) - # clone/install the requirements project first - installed = _git_clone_and_install_subset(yaml_file, + installed = _git_clone_and_install_subset(projects, parent_dir, whitelist=['requirements']) if 'requirements' not in installed: - error_out('requirements git repository must be specified') + update_reqs = False # clone/install all other projects except requirements and the core project blacklist = ['requirements', core_project] - _git_clone_and_install_subset(yaml_file, blacklist=blacklist, - update_requirements=True) + _git_clone_and_install_subset(projects, parent_dir, blacklist=blacklist, + update_requirements=update_reqs) # clone/install the core project whitelist = [core_project] - installed = _git_clone_and_install_subset(yaml_file, whitelist=whitelist, - update_requirements=True) + installed = _git_clone_and_install_subset(projects, parent_dir, + whitelist=whitelist, + update_requirements=update_reqs) if core_project not in installed: error_out('{} git repository must be specified'.format(core_project)) -def _git_clone_and_install_subset(yaml_file, whitelist=[], blacklist=[], - update_requirements=False): - """Clone/install subset of OpenStack repos specified in yaml config file.""" +def _git_clone_and_install_subset(projects, parent_dir, whitelist=[], + blacklist=[], update_requirements=False): + """Clone/install subset of OpenStack repos specified in projects dict.""" global requirements_dir installed = [] - with open(yaml_file, 'r') as fd: - projects = yaml.load(fd) - for proj, val in projects.items(): - # The project subset is chosen based on the following 3 rules: - # 1) If project is in blacklist, we don't clone/install it, period. - # 2) If whitelist is empty, we clone/install everything else. - # 3) If whitelist is not empty, we clone/install everything in the - # whitelist. - if proj in blacklist: - continue - if whitelist and proj not in whitelist: - continue - repo = val['repository'] - branch = val['branch'] - repo_dir = _git_clone_and_install_single(repo, branch, - update_requirements) - if proj == 'requirements': - requirements_dir = repo_dir - installed.append(proj) + for proj, val in projects.items(): + # The project subset is chosen based on the following 3 rules: + # 1) If project is in blacklist, we don't clone/install it, period. + # 2) If whitelist is empty, we clone/install everything else. + # 3) If whitelist is not empty, we clone/install everything in the + # whitelist. + if proj in blacklist: + continue + if whitelist and proj not in whitelist: + continue + repo = val['repository'] + branch = val['branch'] + repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + update_requirements) + if proj == 'requirements': + requirements_dir = repo_dir + installed.append(proj) return installed -def _git_clone_and_install_single(repo, branch, update_requirements=False): +def _git_clone_and_install_single(repo, branch, parent_dir, + update_requirements=False): """Clone and install a single git repository.""" - dest_parent_dir = "/mnt/openstack-git/" - dest_dir = os.path.join(dest_parent_dir, os.path.basename(repo)) + dest_dir = os.path.join(parent_dir, os.path.basename(repo)) + lock_dir = os.path.join(parent_dir, os.path.basename(repo) + '.lock') - if not os.path.exists(dest_parent_dir): - juju_log('Host dir not mounted at {}. ' - 'Creating directory there instead.'.format(dest_parent_dir)) - os.mkdir(dest_parent_dir) - - if not os.path.exists(dest_dir): - juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) - repo_dir = install_remote(repo, dest=dest_parent_dir, branch=branch) + # Note(coreycb): The parent directory for storing git repositories can be + # shared by multiple charms via bind mount, etc, so we use exception + # handling to ensure the test for existence and mkdir are atomic. + try: + os.mkdir(parent_dir) + except OSError as e: + if e.errno == errno.EEXIST: + juju_log('Directory already exists at {}. ' + 'No need to create directory.'.format(parent_dir)) + pass else: - repo_dir = dest_dir + juju_log('Host directory not mounted at {}. ' + 'Directory created.'.format(parent_dir)) - if update_requirements: - if not requirements_dir: - error_out('requirements repo must be cloned before ' - 'updating from global requirements.') - _git_update_requirements(repo_dir, requirements_dir) + # Note(coreycb): Similar to above, the cloned git repositories can be shared + # by multiple charms via bind mount, etc, so we use exception handling and + # special lock directories to ensure that a repository clone is only + # attempted once. + try: + os.mkdir(lock_dir) + except OSError as e: + if e.errno == errno.EEXIST: + juju_log('Lock directory exists at {}. Skip git clone and wait ' + 'for lock removal before installing.'.format(lock_dir)) + while os.path.exists(lock_dir): + juju_log('Waiting for git clone to complete before installing.') + time.sleep(1) + pass + else: + if not os.path.exists(dest_dir): + juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) + repo_dir = install_remote(repo, dest=parent_dir, branch=branch) + else: + repo_dir = dest_dir + + if update_requirements: + if not requirements_dir: + error_out('requirements repo must be cloned before ' + 'updating from global requirements.') + _git_update_requirements(repo_dir, requirements_dir) + + os.rmdir(lock_dir) juju_log('Installing git repo from dir: {}'.format(repo_dir)) pip_install(repo_dir) diff --git a/hooks/charmhelpers/contrib/python/__init__.py b/hooks/charmhelpers/contrib/python/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/python/__init__.py +++ b/hooks/charmhelpers/contrib/python/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 78162b1b..8659516b 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -1,7 +1,21 @@ #!/usr/bin/env python # coding: utf-8 -__author__ = "Jorge Niedbalski " +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . from charmhelpers.fetch import apt_install, apt_update from charmhelpers.core.hookenv import log @@ -13,6 +27,8 @@ except ImportError: apt_install('python-pip') from pip import main as pip_execute +__author__ = "Jorge Niedbalski " + def parse_options(given, available): """Given a set of options, check if available""" @@ -35,7 +51,7 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, **options): +def pip_install(package, fatal=False, upgrade=False, **options): """Install a python package""" command = ["install"] @@ -43,6 +59,9 @@ def pip_install(package, fatal=False, **options): for option in parse_options(options, available_options): command.append(option) + if upgrade: + command.append('--upgrade') + if isinstance(package, list): command.extend(package) else: diff --git a/hooks/charmhelpers/contrib/storage/__init__.py b/hooks/charmhelpers/contrib/storage/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/storage/__init__.py +++ b/hooks/charmhelpers/contrib/storage/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/storage/linux/__init__.py b/hooks/charmhelpers/contrib/storage/linux/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/contrib/storage/linux/__init__.py +++ b/hooks/charmhelpers/contrib/storage/linux/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 6ebeab5c..31ea7f9e 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # # Copyright 2012 Canonical Ltd. # diff --git a/hooks/charmhelpers/core/__init__.py b/hooks/charmhelpers/core/__init__.py index e69de29b..d1400a02 100644 --- a/hooks/charmhelpers/core/__init__.py +++ b/hooks/charmhelpers/core/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/hooks/charmhelpers/core/decorators.py b/hooks/charmhelpers/core/decorators.py index 029a4ef4..bb05620b 100644 --- a/hooks/charmhelpers/core/decorators.py +++ b/hooks/charmhelpers/core/decorators.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # # Copyright 2014 Canonical Ltd. # diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py index 0adf0db3..3056fbac 100644 --- a/hooks/charmhelpers/core/fstab.py +++ b/hooks/charmhelpers/core/fstab.py @@ -1,11 +1,27 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -__author__ = 'Jorge Niedbalski R. ' +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . import io import os +__author__ = 'Jorge Niedbalski R. ' + class Fstab(io.FileIO): """This class extends file in order to implement a file reader/writer @@ -61,7 +77,7 @@ class Fstab(io.FileIO): for line in self.readlines(): line = line.decode('us-ascii') try: - if line.strip() and not line.startswith("#"): + if line.strip() and not line.strip().startswith("#"): yield self._hydrate_entry(line) except ValueError: pass @@ -88,7 +104,7 @@ class Fstab(io.FileIO): found = False for index, line in enumerate(lines): - if not line.startswith("#"): + if line.strip() and not line.strip().startswith("#"): if self._hydrate_entry(line) == entry: found = True break diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index 69ae4564..cf552b39 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + "Interactions with the Juju environment" # Copyright 2013 Canonical Ltd. # diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 5221120c..b771c611 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + """Tools for working with the host system""" # Copyright 2012 Canonical Ltd. # @@ -168,18 +184,18 @@ def mkdir(path, owner='root', group='root', perms=0o555, force=False): log("Removing non-directory file {} prior to mkdir()".format(path)) os.unlink(realpath) os.makedirs(realpath, perms) - os.chown(realpath, uid, gid) elif not path_exists: os.makedirs(realpath, perms) - os.chown(realpath, uid, gid) + os.chown(realpath, uid, gid) + os.chmod(realpath, perms) def write_file(path, content, owner='root', group='root', perms=0o444): - """Create or overwrite a file with the contents of a string""" + """Create or overwrite a file with the contents of a byte string.""" log("Writing file {} {}:{} {:o}".format(path, owner, group, perms)) uid = pwd.getpwnam(owner).pw_uid gid = grp.getgrnam(group).gr_gid - with open(path, 'w') as target: + with open(path, 'wb') as target: os.fchown(target.fileno(), uid, gid) os.fchmod(target.fileno(), perms) target.write(content) @@ -289,11 +305,11 @@ def restart_on_change(restart_map, stopstart=False): ceph_client_changed function. """ def wrap(f): - def wrapped_f(*args): + def wrapped_f(*args, **kwargs): checksums = {} for path in restart_map: checksums[path] = file_hash(path) - f(*args) + f(*args, **kwargs) restarts = [] for path in restart_map: if checksums[path] != file_hash(path): @@ -345,7 +361,7 @@ def list_nics(nic_type): ip_output = (line for line in ip_output if line) for line in ip_output: if line.split()[1].startswith(int_type): - matched = re.search('.*: (bond[0-9]+\.[0-9]+)@.*', line) + matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line) if matched: interface = matched.groups()[0] else: @@ -389,6 +405,9 @@ def cmp_pkgrevno(package, revno, pkgcache=None): * 0 => Installed revno is the same as supplied arg * -1 => Installed revno is less than supplied arg + This function imports apt_cache function from charmhelpers.fetch if + the pkgcache argument is None. Be sure to add charmhelpers.fetch if + you call this function, or pass an apt_pkg.Cache() instance. ''' import apt_pkg if not pkgcache: @@ -407,13 +426,21 @@ def chdir(d): os.chdir(cur) -def chownr(path, owner, group): +def chownr(path, owner, group, follow_links=True): uid = pwd.getpwnam(owner).pw_uid gid = grp.getgrnam(group).gr_gid + if follow_links: + chown = os.chown + else: + chown = os.lchown for root, dirs, files in os.walk(path): for name in dirs + files: full = os.path.join(root, name) broken_symlink = os.path.lexists(full) and not os.path.exists(full) if not broken_symlink: - os.chown(full, uid, gid) + chown(full, uid, gid) + + +def lchownr(path, owner, group): + chownr(path, owner, group, follow_links=False) diff --git a/hooks/charmhelpers/core/services/__init__.py b/hooks/charmhelpers/core/services/__init__.py index 69dde79a..0928158b 100644 --- a/hooks/charmhelpers/core/services/__init__.py +++ b/hooks/charmhelpers/core/services/__init__.py @@ -1,2 +1,18 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + from .base import * # NOQA from .helpers import * # NOQA diff --git a/hooks/charmhelpers/core/services/base.py b/hooks/charmhelpers/core/services/base.py index 87ecb130..c5534e4c 100644 --- a/hooks/charmhelpers/core/services/base.py +++ b/hooks/charmhelpers/core/services/base.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import re import json diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py index 163a7932..15b21664 100644 --- a/hooks/charmhelpers/core/services/helpers.py +++ b/hooks/charmhelpers/core/services/helpers.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import yaml from charmhelpers.core import hookenv @@ -29,12 +45,14 @@ class RelationContext(dict): """ name = None interface = None - required_keys = [] def __init__(self, name=None, additional_required_keys=None): + if not hasattr(self, 'required_keys'): + self.required_keys = [] + if name is not None: self.name = name - if additional_required_keys is not None: + if additional_required_keys: self.required_keys.extend(additional_required_keys) self.get_data() @@ -118,7 +136,10 @@ class MysqlRelation(RelationContext): """ name = 'db' interface = 'mysql' - required_keys = ['host', 'user', 'password', 'database'] + + def __init__(self, *args, **kwargs): + self.required_keys = ['host', 'user', 'password', 'database'] + super(HttpRelation).__init__(self, *args, **kwargs) class HttpRelation(RelationContext): @@ -130,7 +151,10 @@ class HttpRelation(RelationContext): """ name = 'website' interface = 'http' - required_keys = ['host', 'port'] + + def __init__(self, *args, **kwargs): + self.required_keys = ['host', 'port'] + super(HttpRelation).__init__(self, *args, **kwargs) def provide_data(self): return { diff --git a/hooks/charmhelpers/core/sysctl.py b/hooks/charmhelpers/core/sysctl.py index 0f299630..21cc8ab2 100644 --- a/hooks/charmhelpers/core/sysctl.py +++ b/hooks/charmhelpers/core/sysctl.py @@ -1,7 +1,21 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -__author__ = 'Jorge Niedbalski R. ' +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . import yaml @@ -10,25 +24,33 @@ from subprocess import check_call from charmhelpers.core.hookenv import ( log, DEBUG, + ERROR, ) +__author__ = 'Jorge Niedbalski R. ' + def create(sysctl_dict, sysctl_file): """Creates a sysctl.conf file from a YAML associative array - :param sysctl_dict: a dict of sysctl options eg { 'kernel.max_pid': 1337 } - :type sysctl_dict: dict + :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }" + :type sysctl_dict: str :param sysctl_file: path to the sysctl file to be saved :type sysctl_file: str or unicode :returns: None """ - sysctl_dict = yaml.load(sysctl_dict) + try: + sysctl_dict_parsed = yaml.safe_load(sysctl_dict) + except yaml.YAMLError: + log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), + level=ERROR) + return with open(sysctl_file, "w") as fd: - for key, value in sysctl_dict.items(): + for key, value in sysctl_dict_parsed.items(): fd.write("{}={}\n".format(key, value)) - log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict), + log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict_parsed), level=DEBUG) check_call(["sysctl", "-p", sysctl_file]) diff --git a/hooks/charmhelpers/core/templating.py b/hooks/charmhelpers/core/templating.py index 569eaed6..45319998 100644 --- a/hooks/charmhelpers/core/templating.py +++ b/hooks/charmhelpers/core/templating.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os from charmhelpers.core import host @@ -5,7 +21,7 @@ from charmhelpers.core import hookenv def render(source, target, context, owner='root', group='root', - perms=0o444, templates_dir=None): + perms=0o444, templates_dir=None, encoding='UTF-8'): """ Render a template. @@ -48,5 +64,5 @@ def render(source, target, context, owner='root', group='root', level=hookenv.ERROR) raise e content = template.render(context) - host.mkdir(os.path.dirname(target), owner, group) - host.write_file(target, content, owner, group, perms) + host.mkdir(os.path.dirname(target), owner, group, perms=0o755) + host.write_file(target, content.encode(encoding), owner, group, perms) diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index aceadea4..792e629a 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import importlib from tempfile import NamedTemporaryFile import time diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index 8a4624b2..8dfce505 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -1,7 +1,33 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import hashlib import re +from charmhelpers.fetch import ( + BaseFetchHandler, + UnhandledSource +) +from charmhelpers.payload.archive import ( + get_archive_handler, + extract, +) +from charmhelpers.core.host import mkdir, check_hash + import six if six.PY3: from urllib.request import ( @@ -19,16 +45,6 @@ else: ) from urlparse import urlparse, urlunparse, parse_qs -from charmhelpers.fetch import ( - BaseFetchHandler, - UnhandledSource -) -from charmhelpers.payload.archive import ( - get_archive_handler, - extract, -) -from charmhelpers.core.host import mkdir, check_hash - def splituser(host): '''urllib.splituser(), but six's support of this seems broken''' diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py index 8ef48f30..3531315a 100644 --- a/hooks/charmhelpers/fetch/bzrurl.py +++ b/hooks/charmhelpers/fetch/bzrurl.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os from charmhelpers.fetch import ( BaseFetchHandler, @@ -11,10 +27,12 @@ if six.PY3: try: from bzrlib.branch import Branch + from bzrlib import bzrdir, workingtree, errors except ImportError: from charmhelpers.fetch import apt_install apt_install("python-bzrlib") from bzrlib.branch import Branch + from bzrlib import bzrdir, workingtree, errors class BzrUrlFetchHandler(BaseFetchHandler): @@ -34,9 +52,15 @@ class BzrUrlFetchHandler(BaseFetchHandler): if url_parts.scheme == "lp": from bzrlib.plugin import load_plugins load_plugins() + try: + local_branch = bzrdir.BzrDir.create_branch_convenience(dest) + except errors.AlreadyControlDirError: + local_branch = Branch.open(dest) try: remote_branch = Branch.open(source) - remote_branch.bzrdir.sprout(dest).open_branch() + remote_branch.push(local_branch) + tree = workingtree.WorkingTree.open(dest) + tree.update() except Exception as e: raise e diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index f3aa2821..93aae87b 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os from charmhelpers.fetch import ( BaseFetchHandler, @@ -16,6 +32,8 @@ except ImportError: apt_install("python-git") from git import Repo +from git.exc import GitCommandError # noqa E402 + class GitUrlFetchHandler(BaseFetchHandler): """Handler for git branches via generic and github URLs""" @@ -46,6 +64,8 @@ class GitUrlFetchHandler(BaseFetchHandler): mkdir(dest_dir, perms=0o755) try: self.clone(source, dest_dir, branch) + except GitCommandError as e: + raise UnhandledSource(e.message) except OSError as e: raise UnhandledSource(e.strerror) return dest_dir diff --git a/hooks/charmhelpers/payload/__init__.py b/hooks/charmhelpers/payload/__init__.py index fc9fbc08..e6f42497 100644 --- a/hooks/charmhelpers/payload/__init__.py +++ b/hooks/charmhelpers/payload/__init__.py @@ -1 +1,17 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + "Tools for working with files injected into a charm just before deployment." diff --git a/hooks/charmhelpers/payload/execd.py b/hooks/charmhelpers/payload/execd.py index 6476a75f..4d4d81a6 100644 --- a/hooks/charmhelpers/payload/execd.py +++ b/hooks/charmhelpers/payload/execd.py @@ -1,5 +1,21 @@ #!/usr/bin/env python +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import os import sys import subprocess diff --git a/tests/charmhelpers/__init__.py b/tests/charmhelpers/__init__.py index b46e2e23..f72e7f84 100644 --- a/tests/charmhelpers/__init__.py +++ b/tests/charmhelpers/__init__.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + # Bootstrap charm-helpers, installing its dependencies if necessary using # only standard libraries. import subprocess diff --git a/tests/charmhelpers/contrib/__init__.py b/tests/charmhelpers/contrib/__init__.py index e69de29b..d1400a02 100644 --- a/tests/charmhelpers/contrib/__init__.py +++ b/tests/charmhelpers/contrib/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/amulet/__init__.py b/tests/charmhelpers/contrib/amulet/__init__.py index e69de29b..d1400a02 100644 --- a/tests/charmhelpers/contrib/amulet/__init__.py +++ b/tests/charmhelpers/contrib/amulet/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/amulet/deployment.py b/tests/charmhelpers/contrib/amulet/deployment.py index 3d3ef339..367d6b47 100644 --- a/tests/charmhelpers/contrib/amulet/deployment.py +++ b/tests/charmhelpers/contrib/amulet/deployment.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import amulet import os import six diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index d333e63b..65219d33 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import ConfigParser import io import logging @@ -153,8 +169,13 @@ class AmuletUtils(object): cmd = 'pgrep -o -f {}'.format(service) else: cmd = 'pgrep -o {}'.format(service) - proc_dir = '/proc/{}'.format(sentry_unit.run(cmd)[0].strip()) - return self._get_dir_mtime(sentry_unit, proc_dir) + cmd = cmd + ' | grep -v pgrep || exit 0' + cmd_out = sentry_unit.run(cmd) + self.log.debug('CMDout: ' + str(cmd_out)) + if cmd_out[0]: + self.log.debug('Pid for %s %s' % (service, str(cmd_out[0]))) + proc_dir = '/proc/{}'.format(cmd_out[0].strip()) + return self._get_dir_mtime(sentry_unit, proc_dir) def service_restarted(self, sentry_unit, service, filename, pgrep_full=False, sleep_time=20): @@ -171,6 +192,121 @@ class AmuletUtils(object): else: return False + def service_restarted_since(self, sentry_unit, mtime, service, + pgrep_full=False, sleep_time=20, + retry_count=2): + """Check if service was been started after a given time. + + Args: + sentry_unit (sentry): The sentry unit to check for the service on + mtime (float): The epoch time to check against + service (string): service name to look for in process table + pgrep_full (boolean): Use full command line search mode with pgrep + sleep_time (int): Seconds to sleep before looking for process + retry_count (int): If service is not found, how many times to retry + + Returns: + bool: True if service found and its start time it newer than mtime, + False if service is older than mtime or if service was + not found. + """ + self.log.debug('Checking %s restarted since %s' % (service, mtime)) + time.sleep(sleep_time) + proc_start_time = self._get_proc_start_time(sentry_unit, service, + pgrep_full) + while retry_count > 0 and not proc_start_time: + self.log.debug('No pid file found for service %s, will retry %i ' + 'more times' % (service, retry_count)) + time.sleep(30) + proc_start_time = self._get_proc_start_time(sentry_unit, service, + pgrep_full) + retry_count = retry_count - 1 + + if not proc_start_time: + self.log.warn('No proc start time found, assuming service did ' + 'not start') + return False + if proc_start_time >= mtime: + self.log.debug('proc start time is newer than provided mtime' + '(%s >= %s)' % (proc_start_time, mtime)) + return True + else: + self.log.warn('proc start time (%s) is older than provided mtime ' + '(%s), service did not restart' % (proc_start_time, + mtime)) + return False + + def config_updated_since(self, sentry_unit, filename, mtime, + sleep_time=20): + """Check if file was modified after a given time. + + Args: + sentry_unit (sentry): The sentry unit to check the file mtime on + filename (string): The file to check mtime of + mtime (float): The epoch time to check against + sleep_time (int): Seconds to sleep before looking for process + + Returns: + bool: True if file was modified more recently than mtime, False if + file was modified before mtime, + """ + self.log.debug('Checking %s updated since %s' % (filename, mtime)) + time.sleep(sleep_time) + file_mtime = self._get_file_mtime(sentry_unit, filename) + if file_mtime >= mtime: + self.log.debug('File mtime is newer than provided mtime ' + '(%s >= %s)' % (file_mtime, mtime)) + return True + else: + self.log.warn('File mtime %s is older than provided mtime %s' + % (file_mtime, mtime)) + return False + + def validate_service_config_changed(self, sentry_unit, mtime, service, + filename, pgrep_full=False, + sleep_time=20, retry_count=2): + """Check service and file were updated after mtime + + Args: + sentry_unit (sentry): The sentry unit to check for the service on + mtime (float): The epoch time to check against + service (string): service name to look for in process table + filename (string): The file to check mtime of + pgrep_full (boolean): Use full command line search mode with pgrep + sleep_time (int): Seconds to sleep before looking for process + retry_count (int): If service is not found, how many times to retry + + Typical Usage: + u = OpenStackAmuletUtils(ERROR) + ... + mtime = u.get_sentry_time(self.cinder_sentry) + self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'}) + if not u.validate_service_config_changed(self.cinder_sentry, + mtime, + 'cinder-api', + '/etc/cinder/cinder.conf') + amulet.raise_status(amulet.FAIL, msg='update failed') + Returns: + bool: True if both service and file where updated/restarted after + mtime, False if service is older than mtime or if service was + not found or if filename was modified before mtime. + """ + self.log.debug('Checking %s restarted since %s' % (service, mtime)) + time.sleep(sleep_time) + service_restart = self.service_restarted_since(sentry_unit, mtime, + service, + pgrep_full=pgrep_full, + sleep_time=0, + retry_count=retry_count) + config_update = self.config_updated_since(sentry_unit, filename, mtime, + sleep_time=0) + return service_restart and config_update + + def get_sentry_time(self, sentry_unit): + """Return current epoch time on a sentry""" + cmd = "date +'%s'" + return float(sentry_unit.run(cmd)[0]) + def relation_error(self, name, data): return 'unexpected relation data in {} - {}'.format(name, data) diff --git a/tests/charmhelpers/contrib/openstack/__init__.py b/tests/charmhelpers/contrib/openstack/__init__.py index e69de29b..d1400a02 100644 --- a/tests/charmhelpers/contrib/openstack/__init__.py +++ b/tests/charmhelpers/contrib/openstack/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/openstack/amulet/__init__.py b/tests/charmhelpers/contrib/openstack/amulet/__init__.py index e69de29b..d1400a02 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/__init__.py +++ b/tests/charmhelpers/contrib/openstack/amulet/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index f3fee074..0cfeaa4c 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import six from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment @@ -55,16 +71,19 @@ class OpenStackAmuletDeployment(AmuletDeployment): services.append(this_service) use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', 'ceph-osd', 'ceph-radosgw'] + # Openstack subordinate charms do not expose an origin option as that + # is controlled by the principle + ignore = ['neutron-openvswitch'] if self.openstack: for svc in services: - if svc['name'] not in use_source: + if svc['name'] not in use_source + ignore: config = {'openstack-origin': self.openstack} self.d.configure(svc['name'], config) if self.source: for svc in services: - if svc['name'] in use_source: + if svc['name'] in use_source and svc['name'] not in ignore: config = {'source': self.source} self.d.configure(svc['name'], config) diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index 3e0cc61c..9c3d918a 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -1,3 +1,19 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + import logging import os import time From 6986c6fafdb59ab4fc55658000f024b7668f75a3 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 7 Mar 2015 21:29:39 -0500 Subject: [PATCH 03/33] Update openstack-origin-git to take a YAML-formatted two-dimensional array --- README.md | 90 ++++++++++++++++++++++++++++++++++++ charm-helpers-hooks.yaml | 3 +- charm-helpers-tests.yaml | 3 +- config.yaml | 23 +++++---- config/git-juno-minimal.yaml | 6 --- config/git-tip-all.yaml | 57 ----------------------- config/git-tip-minimal.yaml | 6 --- hooks/glance_relations.py | 6 +++ hooks/glance_utils.py | 7 +-- tests/basic_deployment.py | 8 +++- 10 files changed, 124 insertions(+), 85 deletions(-) delete mode 100644 config/git-juno-minimal.yaml delete mode 100644 config/git-tip-all.yaml delete mode 100644 config/git-tip-minimal.yaml diff --git a/README.md b/README.md index 1a619639..6645ecaa 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,96 @@ as the endpoint for Glance). Note that Glance in this configuration must be used with either Ceph or Swift providing backing image storage. +Deploying from source +--------------------- + +The minimal openstack-origin-git config required to deploy from source is: + + openstack-origin-git: + "{'glance': + {'repository': 'git://git.openstack.org/openstack/glance.git', + 'branch': 'stable/icehouse'}}" + +If you specify a 'requirements' repository, it will be used to update the +requirements.txt files of all other git repos that it applies to, before +they are installed: + + openstack-origin-git: + "{'requirements': + {'repository': 'git://git.openstack.org/openstack/requirements.git', + 'branch': 'master'}, + 'glance': + {'repository': 'git://git.openstack.org/openstack/glance.git', + 'branch': 'master'}}" + +Note that there are only two key values the charm knows about for the outermost +dictionary: 'glance' and 'requirements'. These repositories must correspond to +these keys. If the requirements repository is specified, it will be installed +first. The glance repository is always installed last. All other repostories +will be installed in between. + +NOTE(coreycb): The following is temporary to keep track of the full list of +current tip repos (may not be up-to-date). + + openstack-origin-git: + "{'requirements': + {'repository': 'git://git.openstack.org/openstack/requirements.git', + 'branch': 'master'}, + 'glance-store': + {'repository': 'git://git.openstack.org/openstack/glance_store.git', + 'branch': 'master'}, + 'keystonemiddleware: + {'repository': 'git://git.openstack.org/openstack/keystonemiddleware.git', + 'branch: 'master'}, + 'oslo-concurrency': + {'repository': 'git://git.openstack.org/openstack/oslo.concurrency.git', + 'branch: 'master'}, + 'oslo-config': + {'repository': 'git://git.openstack.org/openstack/oslo.config.git', + 'branch: 'master'}, + 'oslo-db': + {'repository': 'git://git.openstack.org/openstack/oslo.db.git', + 'branch: 'master'}, + 'oslo-i18n': + {'repository': 'git://git.openstack.org/openstack/oslo.i18n.git', + 'branch: 'master'}, + 'oslo-messaging': + {'repository': 'git://git.openstack.org/openstack/oslo.messaging.git', + 'branch: 'master'}, + 'oslo-serialization': + {'repository': 'git://git.openstack.org/openstack/oslo.serialization.git', + 'branch: 'master'}, + 'oslo-utils': + {'repository': 'git://git.openstack.org/openstack/oslo.utils.git', + 'branch: 'master'}, + 'oslo-vmware': + {'repository': 'git://git.openstack.org/openstack/oslo.vmware.git', + 'branch: 'master'}, + 'osprofiler': + {'repository': 'git://git.openstack.org/stackforge/osprofiler.git', + 'branch: 'master'}, + 'pbr': + {'repository': 'git://git.openstack.org/openstack-dev/pbr.git', + 'branch: 'master'}, + 'python-keystoneclient': + {'repository': 'git://git.openstack.org/openstack/python-keystoneclient.git', + 'branch: 'master'}, + 'python-swiftclient': + {'repository': 'git://git.openstack.org/openstack/python-swiftclient.git', + 'branch: 'master'}, + 'stevedore': + {'repository': 'git://git.openstack.org/openstack/stevedore.git', + 'branch: 'master'}, + 'sqlalchemy-migrate': + {'repository': 'git://git.openstack.org/stackforge/sqlalchemy-migrate.git', + 'branch: 'master'}, + 'wsme': + {'repository': 'git://git.openstack.org/stackforge/wsme.git', + 'branch': 'master'}, + 'glance': + {'repository': 'git://git.openstack.org/openstack/glance.git', + 'branch': 'master'}}" + Contact Information ------------------- diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index c73186f1..1053fa5f 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -1,4 +1,5 @@ -branch: lp:charm-helpers +#branch: lp:charm-helpers +branch: /home/corey/src/charms/git/charm-helpers destination: hooks/charmhelpers include: - core diff --git a/charm-helpers-tests.yaml b/charm-helpers-tests.yaml index 48b12f6f..aaa21c31 100644 --- a/charm-helpers-tests.yaml +++ b/charm-helpers-tests.yaml @@ -1,4 +1,5 @@ -branch: lp:charm-helpers +#branch: lp:charm-helpers +branch: /home/corey/src/charms/git/charm-helpers destination: tests/charmhelpers include: - contrib.amulet diff --git a/config.yaml b/config.yaml index 1dccec21..27a7264b 100644 --- a/config.yaml +++ b/config.yaml @@ -15,21 +15,24 @@ options: provide a later version of OpenStack will trigger a software upgrade. - Note that when openstack-origin-git is specified, the packages - configured for that option will be installed from git rather - than from the openstack-origin repository. + Note that when openstack-origin-git is specified, openstack-specific + packages will be installed from source rather than from the + openstack-origin repository. openstack-origin-git: default: None - type: "string" + type: string description: | - Specifies a YAML file which lists the git repositories and branches - from which to install OpenStack. See config/git-*.yaml for a - starting point. + Specifies a YAML-formatted two-dimensional array listing the git + repositories and branches from which to install OpenStack and its + dependencies. - Note that the config files that are installed will be determined - based on the OpenStack release in the openstack-origin option. + Note that the installed config files will be determined based on + the OpenStack release of the openstack-origin option. - Note that updating this setting after deployment will do nothing. + Note also that this option is processed for the initial install + only. Setting this option after deployment is not supported. + + For more details see README.md. database-user: default: glance type: string diff --git a/config/git-juno-minimal.yaml b/config/git-juno-minimal.yaml deleted file mode 100644 index d0998bd0..00000000 --- a/config/git-juno-minimal.yaml +++ /dev/null @@ -1,6 +0,0 @@ -glance: - repository: git://git.openstack.org/openstack/glance.git - branch: stable/juno -requirements: - repository: git://git.openstack.org/openstack/requirements.git - branch: stable/juno diff --git a/config/git-tip-all.yaml b/config/git-tip-all.yaml deleted file mode 100644 index 1350ecd0..00000000 --- a/config/git-tip-all.yaml +++ /dev/null @@ -1,57 +0,0 @@ -glance: - repository: git://git.openstack.org/openstack/glance.git - branch: master -glance-store: - repository: git://git.openstack.org/openstack/glance_store.git - branch: master -keystonemiddleware: - repository: git://git.openstack.org/openstack/keystonemiddleware.git - branch: master -oslo-config: - repository: git://git.openstack.org/openstack/oslo.config.git - branch: master -oslo-concurrency: - repository: git://git.openstack.org/openstack/oslo.concurrency.git - branch: master -oslo-db: - repository: git://git.openstack.org/openstack/oslo.db.git - branch: master -oslo-i18n: - repository: git://git.openstack.org/openstack/oslo.i18n.git - branch: master -oslo-messaging: - repository: git://git.openstack.org/openstack/oslo.messaging.git - branch: master -oslo-serialization: - repository: git://git.openstack.org/openstack/oslo.serialization.git - branch: master -oslo-utils: - repository: git://git.openstack.org/openstack/oslo.utils.git - branch: master -oslo-vmware: - repository: git://git.openstack.org/openstack/oslo.vmware.git - branch: master -osprofiler: - repository: git://git.openstack.org/stackforge/osprofiler.git - branch: master -pbr: - repository: git://git.openstack.org/openstack-dev/pbr.git - branch: master -python-keystoneclient: - repository: git://git.openstack.org/openstack/python-keystoneclient.git - branch: master -python-swiftclient: - repository: git://git.openstack.org/openstack/python-swiftclient.git - branch: master -requirements: - repository: git://git.openstack.org/openstack/requirements.git - branch: master -stevedore: - repository: git://git.openstack.org/openstack/stevedore.git - branch: master -sqlalchemy-migrate: - repository: git://git.openstack.org/stackforge/sqlalchemy-migrate.git - branch: master -wsme: - repository: git://git.openstack.org/stackforge/wsme.git - branch: master diff --git a/config/git-tip-minimal.yaml b/config/git-tip-minimal.yaml deleted file mode 100644 index 486c5306..00000000 --- a/config/git-tip-minimal.yaml +++ /dev/null @@ -1,6 +0,0 @@ -glance: - repository: git://git.openstack.org/openstack/glance.git - branch: master -requirements: - repository: git://git.openstack.org/openstack/requirements.git - branch: master diff --git a/hooks/glance_relations.py b/hooks/glance_relations.py index c21dedc1..b4fe760e 100755 --- a/hooks/glance_relations.py +++ b/hooks/glance_relations.py @@ -103,6 +103,12 @@ def install_hook(): apt_update(fatal=True) apt_install(determine_packages(), fatal=True) + + # NOTE(coreycb): This is temporary for sstack proxy, unless we decide + # we need to code proxy support into the charms. + os.environ["http_proxy"] = "http://squid.internal:3128" + os.environ["https_proxy"] = "https://squid.internal:3128" + git_install(config('openstack-origin-git')) for service in SERVICES: diff --git a/hooks/glance_utils.py b/hooks/glance_utils.py index f101d9d4..62ae4a07 100755 --- a/hooks/glance_utils.py +++ b/hooks/glance_utils.py @@ -3,6 +3,7 @@ import os import shutil import subprocess +import yaml import glance_contexts @@ -297,11 +298,11 @@ def setup_ipv6(): apt_install('haproxy/trusty-backports', fatal=True) -def git_install(file_name): - """Perform setup, and install git repos specified in yaml config file.""" +def git_install(projects): + """Perform setup, and install git repos specified in yaml parameter.""" if git_install_requested(): git_pre_install() - git_clone_and_install(file_name, core_project='glance') + git_clone_and_install(yaml.load(projects), core_project='glance') git_post_install() diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 34142de8..427bb665 100755 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -55,11 +55,17 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment): def _configure_services(self): '''Configure all of the services.''' + # NOTE(coreycb): Added the following temporarily to test deploy from source + glance_config = {'openstack-origin-git': + "{'glance':" + " {'repository': 'git://git.openstack.org/openstack/glance.git'," + " 'branch': 'stable/icehouse'}}"} keystone_config = {'admin-password': 'openstack', 'admin-token': 'ubuntutesting'} mysql_config = {'dataset-size': '50%'} - configs = {'keystone': keystone_config, + configs = {'glance': glance_config, + 'keystone': keystone_config, 'mysql': mysql_config} super(GlanceBasicDeployment, self)._configure_services(configs) From d2e7ca3025aa667e065717a8fb22ac11efe29630 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 20 Mar 2015 17:34:54 +0000 Subject: [PATCH 04/33] Sync charm-helpers --- .../contrib/openstack/amulet/deployment.py | 27 +- hooks/charmhelpers/contrib/openstack/utils.py | 234 ++++++++++-------- hooks/charmhelpers/core/services/helpers.py | 4 +- hooks/charmhelpers/core/unitdata.py | 2 +- .../contrib/openstack/amulet/deployment.py | 27 +- 5 files changed, 184 insertions(+), 110 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 0cfeaa4c..0e0db566 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -15,6 +15,7 @@ # along with charm-helpers. If not, see . import six +from collections import OrderedDict from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment ) @@ -100,12 +101,34 @@ class OpenStackAmuletDeployment(AmuletDeployment): """ (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse) = range(6) + self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8) releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, ('precise', 'cloud:precise-grizzly'): self.precise_grizzly, ('precise', 'cloud:precise-havana'): self.precise_havana, ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, - ('trusty', None): self.trusty_icehouse} + ('trusty', None): self.trusty_icehouse, + ('trusty', 'cloud:trusty-juno'): self.trusty_juno, + ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo} return releases[(self.series, self.openstack)] + + def _get_openstack_release_string(self): + """Get openstack release string. + + Return a string representing the openstack release. + """ + releases = OrderedDict([ + ('precise', 'essex'), + ('quantal', 'folsom'), + ('raring', 'grizzly'), + ('saucy', 'havana'), + ('trusty', 'icehouse'), + ('utopic', 'juno'), + ('vivid', 'kilo'), + ]) + if self.openstack: + os_origin = self.openstack.split(':')[1] + return os_origin.split('%s-' % self.series)[1].split('/')[0] + else: + return releases[self.series] diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 0293c7d7..da65f6d3 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -20,18 +20,21 @@ from collections import OrderedDict from functools import wraps -import errno import subprocess import json import os import sys -import time import six import yaml from charmhelpers.contrib.network import ip +from charmhelpers.core import ( + hookenv, + unitdata, +) + from charmhelpers.core.hookenv import ( config, log as juju_log, @@ -332,6 +335,21 @@ def configure_installation_source(rel): error_out("Invalid openstack-release specified: %s" % rel) +def config_value_changed(option): + """ + Determine if config value changed since last call to this function. + """ + hook_data = unitdata.HookData() + with hook_data(): + db = unitdata.kv() + current = hookenv.execution_environment()['conf'][option] + saved = db.get(option) + db.set(option, current) + if saved is None: + return False + return current != saved + + def save_script_rc(script_path="scripts/scriptrc", **env_vars): """ Write an rc file in the charm-delivered directory containing @@ -471,116 +489,103 @@ def os_requires_version(ostack_release, pkg): def git_install_requested(): - """Returns true if openstack-origin-git is specified.""" - return config('openstack-origin-git') != None + """ + Returns true if openstack-origin-git is specified. + """ + return config('openstack-origin-git').lower() != "none" requirements_dir = None -def git_clone_and_install(projects, core_project, - parent_dir='/mnt/openstack-git'): - """Clone/install all OpenStack repos specified in projects dictionary.""" - global requirements_dir - update_reqs = True +def git_clone_and_install(projects_yaml, core_project): + """ + Clone/install all specified OpenStack repositories. - if not projects: + The expected format of projects_yaml is: + repositories: + - {name: keystone, + repository: 'git://git.openstack.org/openstack/keystone.git', + branch: 'stable/icehouse'} + - {name: requirements, + repository: 'git://git.openstack.org/openstack/requirements.git', + branch: 'stable/icehouse'} + directory: /mnt/openstack-git + + The directory key is optional. + """ + global requirements_dir + parent_dir = '/mnt/openstack-git' + + if not projects_yaml: return - # clone/install the requirements project first - installed = _git_clone_and_install_subset(projects, parent_dir, - whitelist=['requirements']) - if 'requirements' not in installed: - update_reqs = False + projects = yaml.load(projects_yaml) + _git_validate_projects_yaml(projects, core_project) - # clone/install all other projects except requirements and the core project - blacklist = ['requirements', core_project] - _git_clone_and_install_subset(projects, parent_dir, blacklist=blacklist, - update_requirements=update_reqs) + if 'directory' in projects.keys(): + parent_dir = projects['directory'] - # clone/install the core project - whitelist = [core_project] - installed = _git_clone_and_install_subset(projects, parent_dir, - whitelist=whitelist, - update_requirements=update_reqs) - if core_project not in installed: - error_out('{} git repository must be specified'.format(core_project)) - - -def _git_clone_and_install_subset(projects, parent_dir, whitelist=[], - blacklist=[], update_requirements=False): - """Clone/install subset of OpenStack repos specified in projects dict.""" - global requirements_dir - installed = [] - - for proj, val in projects.items(): - # The project subset is chosen based on the following 3 rules: - # 1) If project is in blacklist, we don't clone/install it, period. - # 2) If whitelist is empty, we clone/install everything else. - # 3) If whitelist is not empty, we clone/install everything in the - # whitelist. - if proj in blacklist: - continue - if whitelist and proj not in whitelist: - continue - repo = val['repository'] - branch = val['branch'] - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, - update_requirements) - if proj == 'requirements': + for p in projects['repositories']: + repo = p['repository'] + branch = p['branch'] + if p['name'] == 'requirements': + repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + update_requirements=False) requirements_dir = repo_dir - installed.append(proj) - return installed - - -def _git_clone_and_install_single(repo, branch, parent_dir, - update_requirements=False): - """Clone and install a single git repository.""" - dest_dir = os.path.join(parent_dir, os.path.basename(repo)) - lock_dir = os.path.join(parent_dir, os.path.basename(repo) + '.lock') - - # Note(coreycb): The parent directory for storing git repositories can be - # shared by multiple charms via bind mount, etc, so we use exception - # handling to ensure the test for existence and mkdir are atomic. - try: - os.mkdir(parent_dir) - except OSError as e: - if e.errno == errno.EEXIST: - juju_log('Directory already exists at {}. ' - 'No need to create directory.'.format(parent_dir)) - pass - else: - juju_log('Host directory not mounted at {}. ' - 'Directory created.'.format(parent_dir)) - - # Note(coreycb): Similar to above, the cloned git repositories can be shared - # by multiple charms via bind mount, etc, so we use exception handling and - # special lock directories to ensure that a repository clone is only - # attempted once. - try: - os.mkdir(lock_dir) - except OSError as e: - if e.errno == errno.EEXIST: - juju_log('Lock directory exists at {}. Skip git clone and wait ' - 'for lock removal before installing.'.format(lock_dir)) - while os.path.exists(lock_dir): - juju_log('Waiting for git clone to complete before installing.') - time.sleep(1) - pass - else: - if not os.path.exists(dest_dir): - juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) - repo_dir = install_remote(repo, dest=parent_dir, branch=branch) else: - repo_dir = dest_dir + repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + update_requirements=True) - if update_requirements: - if not requirements_dir: - error_out('requirements repo must be cloned before ' - 'updating from global requirements.') - _git_update_requirements(repo_dir, requirements_dir) - os.rmdir(lock_dir) +def _git_validate_projects_yaml(projects, core_project): + """ + Validate the projects yaml. + """ + _git_ensure_key_exists('repositories', projects) + + for project in projects['repositories']: + _git_ensure_key_exists('name', project.keys()) + _git_ensure_key_exists('repository', project.keys()) + _git_ensure_key_exists('branch', project.keys()) + + if projects['repositories'][0]['name'] != 'requirements': + error_out('{} git repo must be specified first'.format('requirements')) + + if projects['repositories'][-1]['name'] != core_project: + error_out('{} git repo must be specified last'.format(core_project)) + + +def _git_ensure_key_exists(key, keys): + """ + Ensure that key exists in keys. + """ + if key not in keys: + error_out('openstack-origin-git key \'{}\' is missing'.format(key)) + + +def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements): + """ + Clone and install a single git repository. + """ + dest_dir = os.path.join(parent_dir, os.path.basename(repo)) + + if not os.path.exists(parent_dir): + juju_log('Directory already exists at {}. ' + 'No need to create directory.'.format(parent_dir)) + os.mkdir(parent_dir) + + if not os.path.exists(dest_dir): + juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) + repo_dir = install_remote(repo, dest=parent_dir, branch=branch) + else: + repo_dir = dest_dir + + if update_requirements: + if not requirements_dir: + error_out('requirements repo must be cloned before ' + 'updating from global requirements.') + _git_update_requirements(repo_dir, requirements_dir) juju_log('Installing git repo from dir: {}'.format(repo_dir)) pip_install(repo_dir) @@ -589,16 +594,39 @@ def _git_clone_and_install_single(repo, branch, parent_dir, def _git_update_requirements(package_dir, reqs_dir): - """Update from global requirements. + """ + Update from global requirements. - Update an OpenStack git directory's requirements.txt and - test-requirements.txt from global-requirements.txt.""" + Update an OpenStack git directory's requirements.txt and + test-requirements.txt from global-requirements.txt. + """ orig_dir = os.getcwd() os.chdir(reqs_dir) - cmd = "python update.py {}".format(package_dir) + cmd = ['python', 'update.py', package_dir] try: - subprocess.check_call(cmd.split(' ')) + subprocess.check_call(cmd) except subprocess.CalledProcessError: package = os.path.basename(package_dir) error_out("Error updating {} from global-requirements.txt".format(package)) os.chdir(orig_dir) + + +def git_src_dir(projects_yaml, project): + """ + Return the directory where the specified project's source is located. + """ + parent_dir = '/mnt/openstack-git' + + if not projects_yaml: + return + + projects = yaml.load(projects_yaml) + + if 'directory' in projects.keys(): + parent_dir = projects['directory'] + + for p in projects['repositories']: + if p['name'] == project: + return os.path.join(parent_dir, os.path.basename(p['repository'])) + + return None diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py index 15b21664..3eb5fb44 100644 --- a/hooks/charmhelpers/core/services/helpers.py +++ b/hooks/charmhelpers/core/services/helpers.py @@ -139,7 +139,7 @@ class MysqlRelation(RelationContext): def __init__(self, *args, **kwargs): self.required_keys = ['host', 'user', 'password', 'database'] - super(HttpRelation).__init__(self, *args, **kwargs) + RelationContext.__init__(self, *args, **kwargs) class HttpRelation(RelationContext): @@ -154,7 +154,7 @@ class HttpRelation(RelationContext): def __init__(self, *args, **kwargs): self.required_keys = ['host', 'port'] - super(HttpRelation).__init__(self, *args, **kwargs) + RelationContext.__init__(self, *args, **kwargs) def provide_data(self): return { diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py index 3000134a..406a35c5 100644 --- a/hooks/charmhelpers/core/unitdata.py +++ b/hooks/charmhelpers/core/unitdata.py @@ -443,7 +443,7 @@ class HookData(object): data = hookenv.execution_environment() self.conf = conf_delta = self.kv.delta(data['conf'], 'config') self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') - self.kv.set('env', data['env']) + self.kv.set('env', dict(data['env'])) self.kv.set('unit', data['unit']) self.kv.set('relid', data.get('relid')) return conf_delta, rels_delta diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 0cfeaa4c..0e0db566 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -15,6 +15,7 @@ # along with charm-helpers. If not, see . import six +from collections import OrderedDict from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment ) @@ -100,12 +101,34 @@ class OpenStackAmuletDeployment(AmuletDeployment): """ (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse) = range(6) + self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8) releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, ('precise', 'cloud:precise-grizzly'): self.precise_grizzly, ('precise', 'cloud:precise-havana'): self.precise_havana, ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, - ('trusty', None): self.trusty_icehouse} + ('trusty', None): self.trusty_icehouse, + ('trusty', 'cloud:trusty-juno'): self.trusty_juno, + ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo} return releases[(self.series, self.openstack)] + + def _get_openstack_release_string(self): + """Get openstack release string. + + Return a string representing the openstack release. + """ + releases = OrderedDict([ + ('precise', 'essex'), + ('quantal', 'folsom'), + ('raring', 'grizzly'), + ('saucy', 'havana'), + ('trusty', 'icehouse'), + ('utopic', 'juno'), + ('vivid', 'kilo'), + ]) + if self.openstack: + os_origin = self.openstack.split(':')[1] + return os_origin.split('%s-' % self.series)[1].split('/')[0] + else: + return releases[self.series] From 88012d92f9d9b51280ba2e8fdb6b832df30cc799 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 21 Mar 2015 01:36:16 +0000 Subject: [PATCH 05/33] Overall refresh to current install from source approach --- Makefile | 6 +- README.md | 154 +++++++++++------------ actions.yaml | 2 + actions/git-reinstall | 1 + actions/git_reinstall.py | 40 ++++++ charm-helpers-hooks.yaml | 3 +- charm-helpers-tests.yaml | 3 +- config.yaml | 15 +-- hooks/glance_relations.py | 8 +- hooks/glance_utils.py | 42 +++---- templates/upstart/glance.upstart | 11 -- tests/10-basic-precise-essex | 9 -- tests/11-basic-precise-folsom | 11 -- tests/12-basic-precise-grizzly | 11 -- tests/13-basic-precise-havana | 11 -- tests/16-basic-trusty-icehouse-git | 9 ++ tests/17-basic-trusty-juno | 11 ++ tests/18-basic-trusty-juno-git | 12 ++ tests/basic_deployment.py | 25 +++- unit_tests/__init__.py | 1 + unit_tests/test_actions_git_reinstall.py | 85 +++++++++++++ 21 files changed, 285 insertions(+), 185 deletions(-) create mode 100644 actions.yaml create mode 120000 actions/git-reinstall create mode 100755 actions/git_reinstall.py delete mode 100644 templates/upstart/glance.upstart delete mode 100755 tests/10-basic-precise-essex delete mode 100755 tests/11-basic-precise-folsom delete mode 100755 tests/12-basic-precise-grizzly delete mode 100755 tests/13-basic-precise-havana create mode 100755 tests/16-basic-trusty-icehouse-git create mode 100755 tests/17-basic-trusty-juno create mode 100755 tests/18-basic-trusty-juno-git create mode 100644 unit_tests/test_actions_git_reinstall.py diff --git a/Makefile b/Makefile index d0636a9e..3b45e0fc 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ PYTHON := /usr/bin/env python lint: @echo "Running flake8 tests: " - @flake8 --exclude hooks/charmhelpers hooks unit_tests tests + @flake8 --exclude hooks/charmhelpers actions hooks unit_tests tests @echo "OK" @echo "Running charm proof: " @charm proof @@ -27,7 +27,9 @@ test: # raise_status() messages to stderr: # https://bugs.launchpad.net/amulet/+bug/1320357 @juju test -v -p AMULET_HTTP_PROXY --timeout 900 \ - 00-setup 14-basic-precise-icehouse 15-basic-trusty-icehouse + 00-setup 14-basic-precise-icehouse 15-basic-trusty-icehouse \ + 16-basic-trusty-icehouse-git 17-basic-trusty-juno \ + 18-basic-trusty-juno-git publish: lint unit_test bzr push lp:charms/glance diff --git a/README.md b/README.md index 6645ecaa..ec1b348b 100644 --- a/README.md +++ b/README.md @@ -84,92 +84,84 @@ Swift providing backing image storage. Deploying from source --------------------- -The minimal openstack-origin-git config required to deploy from source is: +The minimum openstack-origin-git config required to deploy from source is: openstack-origin-git: - "{'glance': - {'repository': 'git://git.openstack.org/openstack/glance.git', - 'branch': 'stable/icehouse'}}" + "repositories: + - {name: requirements, + repository: 'git://git.openstack.org/openstack/requirements', + branch: stable/juno} + - {name: glance, + repository: 'git://git.openstack.org/openstack/glance', + branch: stable/juno}" -If you specify a 'requirements' repository, it will be used to update the -requirements.txt files of all other git repos that it applies to, before -they are installed: +Note that there are only two 'name' values the charm knows about: 'requirements' +and 'glance'. These repositories must correspond to these 'name' values. +Additionally, the requirements repository must be specified first and the +glance repository must be specified last. All other repostories are installed +in the order in which they are specified. + +The following is a full list of current tip repos (may not be up-to-date): openstack-origin-git: - "{'requirements': - {'repository': 'git://git.openstack.org/openstack/requirements.git', - 'branch': 'master'}, - 'glance': - {'repository': 'git://git.openstack.org/openstack/glance.git', - 'branch': 'master'}}" - -Note that there are only two key values the charm knows about for the outermost -dictionary: 'glance' and 'requirements'. These repositories must correspond to -these keys. If the requirements repository is specified, it will be installed -first. The glance repository is always installed last. All other repostories -will be installed in between. - -NOTE(coreycb): The following is temporary to keep track of the full list of -current tip repos (may not be up-to-date). - - openstack-origin-git: - "{'requirements': - {'repository': 'git://git.openstack.org/openstack/requirements.git', - 'branch': 'master'}, - 'glance-store': - {'repository': 'git://git.openstack.org/openstack/glance_store.git', - 'branch': 'master'}, - 'keystonemiddleware: - {'repository': 'git://git.openstack.org/openstack/keystonemiddleware.git', - 'branch: 'master'}, - 'oslo-concurrency': - {'repository': 'git://git.openstack.org/openstack/oslo.concurrency.git', - 'branch: 'master'}, - 'oslo-config': - {'repository': 'git://git.openstack.org/openstack/oslo.config.git', - 'branch: 'master'}, - 'oslo-db': - {'repository': 'git://git.openstack.org/openstack/oslo.db.git', - 'branch: 'master'}, - 'oslo-i18n': - {'repository': 'git://git.openstack.org/openstack/oslo.i18n.git', - 'branch: 'master'}, - 'oslo-messaging': - {'repository': 'git://git.openstack.org/openstack/oslo.messaging.git', - 'branch: 'master'}, - 'oslo-serialization': - {'repository': 'git://git.openstack.org/openstack/oslo.serialization.git', - 'branch: 'master'}, - 'oslo-utils': - {'repository': 'git://git.openstack.org/openstack/oslo.utils.git', - 'branch: 'master'}, - 'oslo-vmware': - {'repository': 'git://git.openstack.org/openstack/oslo.vmware.git', - 'branch: 'master'}, - 'osprofiler': - {'repository': 'git://git.openstack.org/stackforge/osprofiler.git', - 'branch: 'master'}, - 'pbr': - {'repository': 'git://git.openstack.org/openstack-dev/pbr.git', - 'branch: 'master'}, - 'python-keystoneclient': - {'repository': 'git://git.openstack.org/openstack/python-keystoneclient.git', - 'branch: 'master'}, - 'python-swiftclient': - {'repository': 'git://git.openstack.org/openstack/python-swiftclient.git', - 'branch: 'master'}, - 'stevedore': - {'repository': 'git://git.openstack.org/openstack/stevedore.git', - 'branch: 'master'}, - 'sqlalchemy-migrate': - {'repository': 'git://git.openstack.org/stackforge/sqlalchemy-migrate.git', - 'branch: 'master'}, - 'wsme': - {'repository': 'git://git.openstack.org/stackforge/wsme.git', - 'branch': 'master'}, - 'glance': - {'repository': 'git://git.openstack.org/openstack/glance.git', - 'branch': 'master'}}" + "repositories: + - {name: requirements, + repository: 'git://git.openstack.org/openstack/requirements', + branch: master} + - {name: oslo-concurrency, + repository: 'git://git.openstack.org/openstack/oslo.concurrency', + branch: master} + - {name: oslo-config, + repository: 'git://git.openstack.org/openstack/oslo.config', + branch: master} + - {name: oslo-db, + repository: 'git://git.openstack.org/openstack/oslo.db', + branch: master} + - {name: oslo-i18n, + repository: 'git://git.openstack.org/openstack/oslo.i18n', + branch: master} + - {name: oslo-messaging, + repository: 'git://git.openstack.org/openstack/oslo.messaging', + branch: master} + - {name: oslo-serialization, + repository: 'git://git.openstack.org/openstack/oslo.serialization', + branch: master} + - {name: oslo-utils, + repository: 'git://git.openstack.org/openstack/oslo.utils', + branch: master} + - {name: oslo-vmware, + repository: 'git://git.openstack.org/openstack/oslo.vmware', + branch: master} + - {name: osprofiler, + repository: 'git://git.openstack.org/stackforge/osprofiler', + branch: master} + - {name: pbr, + repository: 'git://git.openstack.org/openstack-dev/pbr', + branch: master} + - {name: python-keystoneclient, + repository: 'git://git.openstack.org/openstack/python-keystoneclient', + branch: master} + - {name: python-swiftclient, + repository: 'git://git.openstack.org/openstack/python-swiftclient', + branch: master} + - {name: sqlalchemy-migrate, + repository: 'git://git.openstack.org/stackforge/sqlalchemy-migrate', + branch: master} + - {name: stevedore, + repository: 'git://git.openstack.org/openstack/stevedore', + branch: master} + - {name: wsme, + repository: 'git://git.openstack.org/stackforge/wsme', + branch: master} + - {name: keystonemiddleware, + repository: 'git://git.openstack.org/openstack/keystonemiddleware', + branch: master} + - {name: glance-store, + repository: 'git://git.openstack.org/openstack/glance_store', + branch: master} + - {name: glance, + repository: 'git://git.openstack.org/openstack/glance', + branch: master}" Contact Information ------------------- diff --git a/actions.yaml b/actions.yaml new file mode 100644 index 00000000..7872146c --- /dev/null +++ b/actions.yaml @@ -0,0 +1,2 @@ +git-reinstall: + description: Reinstall glance from the openstack-origin-git repositories. diff --git a/actions/git-reinstall b/actions/git-reinstall new file mode 120000 index 00000000..ff684984 --- /dev/null +++ b/actions/git-reinstall @@ -0,0 +1 @@ +git_reinstall.py \ No newline at end of file diff --git a/actions/git_reinstall.py b/actions/git_reinstall.py new file mode 100755 index 00000000..a143d5dc --- /dev/null +++ b/actions/git_reinstall.py @@ -0,0 +1,40 @@ +#!/usr/bin/python +import sys +import traceback + +sys.path.append('hooks/') + +from charmhelpers.contrib.openstack.utils import ( + git_install_requested, +) + +from charmhelpers.core.hookenv import ( + action_set, + action_fail, + config, +) + +from glance_utils import ( + git_install, +) + + +def git_reinstall(): + """Reinstall from source and restart services. + + If the openstack-origin-git config option was used to install openstack + from source git repositories, then this action can be used to reinstall + from updated git repositories, followed by a restart of services.""" + if not git_install_requested(): + action_fail('openstack-origin-git is not configured') + return + + try: + git_install(config('openstack-origin-git')) + except: + action_set({'traceback': traceback.format_exc()}) + action_fail('git-reinstall resulted in an unexpected error') + + +if __name__ == '__main__': + git_reinstall() diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index 1053fa5f..c73186f1 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -1,5 +1,4 @@ -#branch: lp:charm-helpers -branch: /home/corey/src/charms/git/charm-helpers +branch: lp:charm-helpers destination: hooks/charmhelpers include: - core diff --git a/charm-helpers-tests.yaml b/charm-helpers-tests.yaml index aaa21c31..48b12f6f 100644 --- a/charm-helpers-tests.yaml +++ b/charm-helpers-tests.yaml @@ -1,5 +1,4 @@ -#branch: lp:charm-helpers -branch: /home/corey/src/charms/git/charm-helpers +branch: lp:charm-helpers destination: tests/charmhelpers include: - contrib.amulet diff --git a/config.yaml b/config.yaml index 20bd6929..9d086b40 100644 --- a/config.yaml +++ b/config.yaml @@ -15,23 +15,20 @@ options: provide a later version of OpenStack will trigger a software upgrade. - Note that when openstack-origin-git is specified, openstack-specific - packages will be installed from source rather than from the - openstack-origin repository. + Note that when openstack-origin-git is specified, openstack + specific packages will be installed from source rather than + from the openstack-origin repository. openstack-origin-git: default: None type: string description: | - Specifies a YAML-formatted two-dimensional array listing the git - repositories and branches from which to install OpenStack and its - dependencies. + Specifies a YAML-formatted dictionary listing the git + repositories and branches from which to install OpenStack and + its dependencies. Note that the installed config files will be determined based on the OpenStack release of the openstack-origin option. - Note also that this option is processed for the initial install - only. Setting this option after deployment is not supported. - For more details see README.md. database-user: default: glance diff --git a/hooks/glance_relations.py b/hooks/glance_relations.py index 1771408d..b95e38e3 100755 --- a/hooks/glance_relations.py +++ b/hooks/glance_relations.py @@ -105,8 +105,7 @@ def install_hook(): apt_update(fatal=True) apt_install(determine_packages(), fatal=True) - # NOTE(coreycb): This is temporary for sstack proxy, unless we decide - # we need to code proxy support into the charms. + # NOTE(coreycb): This is temporary until bug #1431286 is fixed. os.environ["http_proxy"] = "http://squid.internal:3128" os.environ["https_proxy"] = "https://squid.internal:3128" @@ -325,7 +324,10 @@ def config_changed(): sync_db_with_multi_ipv6_addresses(config('database'), config('database-user')) - if not git_install_requested(): + if git_install_requested(): + if config_value_changed('openstack-origin-git'): + git_install(config('openstack-origin-git')) + else: if openstack_upgrade_available('glance-common'): juju_log('Upgrading OpenStack release') do_openstack_upgrade(CONFIGS) diff --git a/hooks/glance_utils.py b/hooks/glance_utils.py index f0e9e1cd..3f00ebdd 100755 --- a/hooks/glance_utils.py +++ b/hooks/glance_utils.py @@ -47,6 +47,7 @@ from charmhelpers.contrib.openstack.utils import ( get_os_codename_package, git_install_requested, git_clone_and_install, + git_src_dir, configure_installation_source, os_release, ) @@ -302,16 +303,16 @@ def setup_ipv6(): apt_install('haproxy/trusty-backports', fatal=True) -def git_install(projects): +def git_install(projects_yaml): """Perform setup, and install git repos specified in yaml parameter.""" if git_install_requested(): git_pre_install() - git_clone_and_install(yaml.load(projects), core_project='glance') - git_post_install() + git_clone_and_install(projects_yaml, core_project='glance') + git_post_install(projects_yaml) def git_pre_install(): - """Perform pre glance installation setup.""" + """Perform glance pre-install setup.""" dirs = [ '/var/lib/glance', '/var/lib/glance/images', @@ -339,26 +340,10 @@ def git_pre_install(): write_file(l, '', owner='glance', group='glance', perms=0600) -def git_post_install(): - """Perform post glance installation setup.""" - src_etc = os.path.join(charm_dir(), '/mnt/openstack-git/glance.git/etc/') +def git_post_install(projects_yaml): + """Perform glance post-install setup.""" + src_etc = os.path.join(git_src_dir(projects_yaml, 'glance'), 'etc') configs = { - 'glance-api-paste': { - 'src': os.path.join(src_etc, 'glance-api-paste.ini'), - 'dest': '/etc/glance/glance-api-paste.ini', - }, - 'glance-api': { - 'src': os.path.join(src_etc, 'glance-api.conf'), - 'dest': '/etc/glance/glance-api.conf', - }, - 'glance-registry-paste': { - 'src': os.path.join(src_etc, 'glance-registry-paste.ini'), - 'dest': '/etc/glance/glance-registry-paste.ini', - }, - 'glance-registry': { - 'src': os.path.join(src_etc, 'glance-registry.conf'), - 'dest': '/etc/glance/glance-registry.conf', - }, 'glance-cache': { 'src': os.path.join(src_etc, 'glance-cache.conf'), 'dest': '/etc/glance/glance-cache.conf', @@ -398,10 +383,13 @@ def git_post_install(): 'executable_name': '/usr/local/bin/glance-registry', } - render('upstart/glance.upstart', '/etc/init/glance-api.conf', - glance_api_context, perms=0o644) - render('upstart/glance.upstart', '/etc/init/glance-registry.conf', - glance_registry_context, perms=0o644) + # NOTE(coreycb): Needs systemd support + templates_dir = 'hooks/charmhelpers/contrib/openstack/templates' + templates_dir = os.path.join(charm_dir(), templates_dir) + render('git.upstart', '/etc/init/glance-api.conf', + glance_api_context, perms=0o644, templates_dir=templates_dir) + render('git.upstart', '/etc/init/glance-registry.conf', + glance_registry_context, perms=0o644, templates_dir=templates_dir) service_start('glance-api') service_start('glance-registry') diff --git a/templates/upstart/glance.upstart b/templates/upstart/glance.upstart deleted file mode 100644 index e5f39741..00000000 --- a/templates/upstart/glance.upstart +++ /dev/null @@ -1,11 +0,0 @@ -description "{{ service_description }}" -author "Juju {{ service_name }} Charm " - -start on runlevel [2345] -stop on runlevel [!2345] - -respawn - -exec start-stop-daemon --start --chuid {{ user_name }} \ - --chdir {{ start_dir }} --name {{ process_name }} \ - --exec {{ executable_name }} diff --git a/tests/10-basic-precise-essex b/tests/10-basic-precise-essex deleted file mode 100755 index 5869c57a..00000000 --- a/tests/10-basic-precise-essex +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/python - -"""Amulet tests on a basic glance deployment on precise-essex.""" - -from basic_deployment import GlanceBasicDeployment - -if __name__ == '__main__': - deployment = GlanceBasicDeployment(series='precise') - deployment.run_tests() diff --git a/tests/11-basic-precise-folsom b/tests/11-basic-precise-folsom deleted file mode 100755 index 5549f601..00000000 --- a/tests/11-basic-precise-folsom +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/python - -"""Amulet tests on a basic glance deployment on precise-folsom.""" - -from basic_deployment import GlanceBasicDeployment - -if __name__ == '__main__': - deployment = GlanceBasicDeployment(series='precise', - openstack='cloud:precise-folsom', - source='cloud:precise-updates/folsom') - deployment.run_tests() diff --git a/tests/12-basic-precise-grizzly b/tests/12-basic-precise-grizzly deleted file mode 100755 index 03a93cb3..00000000 --- a/tests/12-basic-precise-grizzly +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/python - -"""Amulet tests on a basic glance deployment on precise-grizzly.""" - -from basic_deployment import GlanceBasicDeployment - -if __name__ == '__main__': - deployment = GlanceBasicDeployment(series='precise', - openstack='cloud:precise-grizzly', - source='cloud:precise-updates/grizzly') - deployment.run_tests() diff --git a/tests/13-basic-precise-havana b/tests/13-basic-precise-havana deleted file mode 100755 index 0ceaf36d..00000000 --- a/tests/13-basic-precise-havana +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/python - -"""Amulet tests on a basic glance deployment on precise-havana.""" - -from basic_deployment import GlanceBasicDeployment - -if __name__ == '__main__': - deployment = GlanceBasicDeployment(series='precise', - openstack='cloud:precise-havana', - source='cloud:precise-updates/havana') - deployment.run_tests() diff --git a/tests/16-basic-trusty-icehouse-git b/tests/16-basic-trusty-icehouse-git new file mode 100755 index 00000000..562a8486 --- /dev/null +++ b/tests/16-basic-trusty-icehouse-git @@ -0,0 +1,9 @@ +#!/usr/bin/python + +"""Amulet tests on a basic Glance git deployment on trusty-icehouse.""" + +from basic_deployment import GlanceBasicDeployment + +if __name__ == '__main__': + deployment = GlanceBasicDeployment(series='trusty', git=True) + deployment.run_tests() diff --git a/tests/17-basic-trusty-juno b/tests/17-basic-trusty-juno new file mode 100755 index 00000000..7a726c48 --- /dev/null +++ b/tests/17-basic-trusty-juno @@ -0,0 +1,11 @@ +#!/usr/bin/python + +"""Amulet tests on a basic Glance deployment on trusty-juno.""" + +from basic_deployment import GlanceBasicDeployment + +if __name__ == '__main__': + deployment = GlanceBasicDeployment(series='trusty', + openstack='cloud:trusty-juno', + source='cloud:trusty-updates/juno') + deployment.run_tests() diff --git a/tests/18-basic-trusty-juno-git b/tests/18-basic-trusty-juno-git new file mode 100755 index 00000000..54f0dc77 --- /dev/null +++ b/tests/18-basic-trusty-juno-git @@ -0,0 +1,12 @@ +#!/usr/bin/python + +"""Amulet tests on a basic Glance git deployment on trusty-juno.""" + +from basic_deployment import GlanceBasicDeployment + +if __name__ == '__main__': + deployment = GlanceBasicDeployment(series='trusty', + openstack='cloud:trusty-juno', + source='cloud:trusty-updates/juno', + git=True) + deployment.run_tests() diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 427bb665..458a766f 100755 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -23,9 +23,11 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment): # * Add tests with different storage back ends # * Resolve Essex->Havana juju set charm bug - def __init__(self, series=None, openstack=None, source=None, stable=False): + def __init__(self, series=None, openstack=None, source=None, git=False, + stable=False): '''Deploy the entire test environment.''' super(GlanceBasicDeployment, self).__init__(series, openstack, source, stable) + self.git = git self._add_services() self._add_relations() self._configure_services() @@ -56,13 +58,24 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment): def _configure_services(self): '''Configure all of the services.''' # NOTE(coreycb): Added the following temporarily to test deploy from source - glance_config = {'openstack-origin-git': - "{'glance':" - " {'repository': 'git://git.openstack.org/openstack/glance.git'," - " 'branch': 'stable/icehouse'}}"} + glance_config = {} + if self.git: + branch = 'stable/' + self._get_openstack_release_string() + openstack_origin_git = { + 'repositories': [ + {'name': 'requirements', + 'repository': 'git://git.openstack.org/openstack/requirements', + 'branch': branch}, + {'name': 'glance', + 'repository': 'git://git.openstack.org/openstack/glance', + 'branch': branch}, + ], + 'directory': '/mnt/openstack-git', + } + glance_config['openstack-origin-git'] = yaml.dump(openstack_origin_git) + keystone_config = {'admin-password': 'openstack', 'admin-token': 'ubuntutesting'} - mysql_config = {'dataset-size': '50%'} configs = {'glance': glance_config, 'keystone': keystone_config, diff --git a/unit_tests/__init__.py b/unit_tests/__init__.py index afaed60c..43aa3614 100644 --- a/unit_tests/__init__.py +++ b/unit_tests/__init__.py @@ -1,3 +1,4 @@ import sys +sys.path.append('actions/') sys.path.append('hooks/') diff --git a/unit_tests/test_actions_git_reinstall.py b/unit_tests/test_actions_git_reinstall.py new file mode 100644 index 00000000..c0d6b696 --- /dev/null +++ b/unit_tests/test_actions_git_reinstall.py @@ -0,0 +1,85 @@ +from mock import patch + +with patch('charmhelpers.core.hookenv.config') as config: + config.return_value = 'glance' + import glance_utils as utils # noqa + +import git_reinstall + +from test_utils import ( + CharmTestCase +) + +TO_PATCH = [ + 'config', +] + + +openstack_origin_git = \ + """repositories: + - {name: requirements, + repository: 'git://git.openstack.org/openstack/requirements', + branch: stable/juno} + - {name: glance, + repository: 'git://git.openstack.org/openstack/glance', + branch: stable/juno}""" + + +class TestKeystoneActions(CharmTestCase): + + def setUp(self): + super(TestKeystoneActions, self).setUp(git_reinstall, TO_PATCH) + self.config.side_effect = self.test_config.get + + @patch.object(git_reinstall, 'action_set') + @patch.object(git_reinstall, 'action_fail') + @patch.object(git_reinstall, 'git_install') + def test_git_reinstall(self, git_install, action_fail, action_set): + self.test_config.set('openstack-origin-git', openstack_origin_git) + + git_reinstall.git_reinstall() + + git_install.assert_called_with(openstack_origin_git) + self.assertTrue(git_install.called) + self.assertFalse(action_set.called) + self.assertFalse(action_fail.called) + + @patch.object(git_reinstall, 'action_set') + @patch.object(git_reinstall, 'action_fail') + @patch.object(git_reinstall, 'git_install') + @patch('charmhelpers.contrib.openstack.utils.config') + def test_git_reinstall_not_configured(self, _config, git_install, + action_fail, action_set): + _config.return_value = 'none' + + git_reinstall.git_reinstall() + + msg = 'openstack-origin-git is not configured' + action_fail.assert_called_with(msg) + self.assertFalse(git_install.called) + self.assertFalse(action_set.called) + + @patch.object(git_reinstall, 'action_set') + @patch.object(git_reinstall, 'action_fail') + @patch.object(git_reinstall, 'git_install') + @patch('charmhelpers.contrib.openstack.utils.config') + def test_git_reinstall_exception(self, _config, git_install, + action_fail, action_set): + _config.return_value = openstack_origin_git + e = OSError('something bad happened') + git_install.side_effect = e + traceback = ( + "Traceback (most recent call last):\n" + " File \"actions/git_reinstall.py\", line 33, in git_reinstall\n" + " git_install(config(\'openstack-origin-git\'))\n" + " File \"/usr/lib/python2.7/dist-packages/mock.py\", line 964, in __call__\n" # noqa + " return _mock_self._mock_call(*args, **kwargs)\n" + " File \"/usr/lib/python2.7/dist-packages/mock.py\", line 1019, in _mock_call\n" # noqa + " raise effect\n" + "OSError: something bad happened\n") + + git_reinstall.git_reinstall() + + msg = 'git-reinstall resulted in an unexpected error' + action_fail.assert_called_with(msg) + action_set.assert_called_with({'traceback': traceback}) From a30ee2ea299a5230ea9768f0e2e818088d621be4 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 21 Mar 2015 10:26:52 +0000 Subject: [PATCH 06/33] import yaml for amulet tests --- tests/basic_deployment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 458a766f..9080e0ae 100755 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -1,6 +1,7 @@ #!/usr/bin/python import amulet +import yaml from charmhelpers.contrib.openstack.amulet.deployment import ( OpenStackAmuletDeployment From 276ed78ba9dc76b862f7c44736cf8ab0bc056225 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 8 Apr 2015 17:47:30 +0000 Subject: [PATCH 07/33] Pass list of config_files to git.upstart template --- .../contrib/openstack/templates/git.upstart | 2 ++ hooks/charmhelpers/core/hookenv.py | 15 ++++++++++++++- hooks/glance_utils.py | 4 ++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/templates/git.upstart b/hooks/charmhelpers/contrib/openstack/templates/git.upstart index da94ad12..62af9204 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/git.upstart +++ b/hooks/charmhelpers/contrib/openstack/templates/git.upstart @@ -9,5 +9,7 @@ respawn exec start-stop-daemon --start --chuid {{ user_name }} \ --chdir {{ start_dir }} --name {{ process_name }} \ --exec {{ executable_name }} -- \ + {% for config_file in config_files -%} --config-file={{ config_file }} \ + {% endfor -%} --log-file={{ log_file }} diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index 715dd4c5..86f805f1 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -20,11 +20,13 @@ # Authors: # Charm Helpers Developers +from __future__ import print_function import os import json import yaml import subprocess import sys +import errno from subprocess import CalledProcessError import six @@ -87,7 +89,18 @@ def log(message, level=None): if not isinstance(message, six.string_types): message = repr(message) command += [message] - subprocess.call(command) + # Missing juju-log should not cause failures in unit tests + # Send log output to stderr + try: + subprocess.call(command) + except OSError as e: + if e.errno == errno.ENOENT: + if level: + message = "{}: {}".format(level, message) + message = "juju-log: {}".format(message) + print(message, file=sys.stderr) + else: + raise class Serializable(UserDict): diff --git a/hooks/glance_utils.py b/hooks/glance_utils.py index 6d3d9c3a..83bdcd1f 100755 --- a/hooks/glance_utils.py +++ b/hooks/glance_utils.py @@ -371,7 +371,7 @@ def git_post_install(projects_yaml): 'start_dir': '/var/lib/glance', 'process_name': 'glance-api', 'executable_name': '/usr/local/bin/glance-api', - 'config_file': '/etc/glance/glance-api.conf', + 'config_files': ['/etc/glance/glance-api.conf'], 'log_file': '/var/log/glance/api.log', } @@ -382,7 +382,7 @@ def git_post_install(projects_yaml): 'start_dir': '/var/lib/glance', 'process_name': 'glance-registry', 'executable_name': '/usr/local/bin/glance-registry', - 'config_file': '/etc/glance/glance-registry.conf', + 'config_files': ['/etc/glance/glance-registry.conf'], 'log_file': '/var/log/glance/registry.log', } From b9b47221a1503605890e8ee6bd6bf1a809a6f227 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 8 Apr 2015 18:16:54 +0000 Subject: [PATCH 08/33] Unit test updates for config file template change --- unit_tests/test_glance_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unit_tests/test_glance_utils.py b/unit_tests/test_glance_utils.py index 33b6444a..865ca5c0 100644 --- a/unit_tests/test_glance_utils.py +++ b/unit_tests/test_glance_utils.py @@ -259,7 +259,7 @@ class TestGlanceUtils(CharmTestCase): 'start_dir': '/var/lib/glance', 'process_name': 'glance-api', 'executable_name': '/usr/local/bin/glance-api', - 'config_file': '/etc/glance/glance-api.conf', + 'config_files': ['/etc/glance/glance-api.conf'], 'log_file': '/var/log/glance/api.log', } glance_registry_context = { @@ -269,7 +269,7 @@ class TestGlanceUtils(CharmTestCase): 'start_dir': '/var/lib/glance', 'process_name': 'glance-registry', 'executable_name': '/usr/local/bin/glance-registry', - 'config_file': '/etc/glance/glance-registry.conf', + 'config_files': ['/etc/glance/glance-registry.conf'], 'log_file': '/var/log/glance/registry.log', } expected = [ From 130b325a9bccfc99af3bddf3caee29dad2f7c1c5 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 13 Apr 2015 11:41:36 +0000 Subject: [PATCH 09/33] Get http_proxy from AMULET_HTTP_PROXY env var --- tests/basic_deployment.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index d811d7a7..bdbbf571 100755 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -1,6 +1,7 @@ #!/usr/bin/python import amulet +import os import yaml from charmhelpers.contrib.openstack.amulet.deployment import ( @@ -61,6 +62,7 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment): glance_config = {} if self.git: branch = 'stable/' + self._get_openstack_release_string() + amulet_http_proxy = os.environ.get('AMULET_HTTP_PROXY') openstack_origin_git = { 'repositories': [ {'name': 'requirements', @@ -71,8 +73,8 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment): 'branch': branch}, ], 'directory': '/mnt/openstack-git', - 'http_proxy': 'http://squid.internal:3128', - 'https_proxy': 'https://squid.internal:3128', + 'http_proxy': amulet_http_proxy, + 'https_proxy': amulet_http_proxy, } glance_config['openstack-origin-git'] = yaml.dump(openstack_origin_git) From cfc9b47e11b40adc03175a684b2476b5a2013d13 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 13 Apr 2015 17:43:17 +0000 Subject: [PATCH 10/33] Bulk copy files to /etc/glance --- hooks/glance_utils.py | 24 +++++------------------- unit_tests/test_glance_utils.py | 17 +++++++---------- 2 files changed, 12 insertions(+), 29 deletions(-) diff --git a/hooks/glance_utils.py b/hooks/glance_utils.py index 83bdcd1f..db37500b 100755 --- a/hooks/glance_utils.py +++ b/hooks/glance_utils.py @@ -320,7 +320,6 @@ def git_pre_install(): '/var/lib/glance/image-cache/invalid', '/var/lib/glance/image-cache/queue', '/var/log/glance', - '/etc/glance', ] logs = [ @@ -343,26 +342,13 @@ def git_post_install(projects_yaml): """Perform glance post-install setup.""" src_etc = os.path.join(git_src_dir(projects_yaml, 'glance'), 'etc') configs = { - 'glance-cache': { - 'src': os.path.join(src_etc, 'glance-cache.conf'), - 'dest': '/etc/glance/glance-cache.conf', - }, - 'glance-scrubber': { - 'src': os.path.join(src_etc, 'glance-scrubber.conf'), - 'dest': '/etc/glance/glance-scrubber.conf', - }, - 'policy': { - 'src': os.path.join(src_etc, 'policy.json'), - 'dest': '/etc/glance/policy.json', - }, - 'schema-image': { - 'src': os.path.join(src_etc, 'schema-image.json'), - 'dest': '/etc/glance/schema-image.json', - }, + 'src': src_etc, + 'dest': '/etc/glance', } - for conf, files in configs.iteritems(): - shutil.copyfile(files['src'], files['dest']) + if os.path.exists(configs['dest']): + shutil.rmtree(configs['dest']) + shutil.copytree(configs['src'], configs['dest']) glance_api_context = { 'service_description': 'Glance API server', diff --git a/unit_tests/test_glance_utils.py b/unit_tests/test_glance_utils.py index 865ca5c0..8b89ad9a 100644 --- a/unit_tests/test_glance_utils.py +++ b/unit_tests/test_glance_utils.py @@ -223,8 +223,6 @@ class TestGlanceUtils(CharmTestCase): group='glance', perms=0700, force=False), call('/var/log/glance', owner='glance', group='glance', perms=0700, force=False), - call('/etc/glance', owner='glance', - group='glance', perms=0700, force=False), ] self.assertEquals(mkdir.call_args_list, expected) expected = [ @@ -239,19 +237,18 @@ class TestGlanceUtils(CharmTestCase): @patch.object(utils, 'service_restart') @patch.object(utils, 'render') @patch('os.path.join') - @patch('shutil.copyfile') - def test_git_post_install(self, copyfile, join, render, service_restart, - git_src_dir): + @patch('os.path.exists') + @patch('shutil.copytree') + @patch('shutil.rmtree') + def test_git_post_install(self, rmtree, copytree, exists, join, render, + service_restart, git_src_dir): projects_yaml = openstack_origin_git join.return_value = 'joined-string' utils.git_post_install(projects_yaml) expected = [ - call('joined-string', '/etc/glance/glance-cache.conf'), - call('joined-string', '/etc/glance/glance-scrubber.conf'), - call('joined-string', '/etc/glance/policy.json'), - call('joined-string', '/etc/glance/schema-image.json'), + call('joined-string', '/etc/glance'), ] - copyfile.assert_has_calls(expected, any_order=True) + copytree.assert_has_calls(expected) glance_api_context = { 'service_description': 'Glance API server', 'service_name': 'Glance', From 782e59a93163cf9793922de71311f3244bfc659f Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 13 Apr 2015 19:16:31 +0000 Subject: [PATCH 11/33] Turn DEBUG on by default for amulet tests --- tests/basic_deployment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index bdbbf571..1b1fcb13 100755 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -15,7 +15,7 @@ from charmhelpers.contrib.openstack.amulet.utils import ( ) # Use DEBUG to turn on debug logging -u = OpenStackAmuletUtils(ERROR) +u = OpenStackAmuletUtils(DEBUG) class GlanceBasicDeployment(OpenStackAmuletDeployment): '''Amulet tests on a basic file-backed glance deployment. Verify relations, From ee064965308bae330fb2bd192991bedd41cd785d Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 13 Apr 2015 23:47:47 +0000 Subject: [PATCH 12/33] Run config-changed hook after git-reinstall action installs from source --- actions/git_reinstall.py | 5 +++++ unit_tests/test_actions_git_reinstall.py | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/actions/git_reinstall.py b/actions/git_reinstall.py index a143d5dc..7044ba43 100755 --- a/actions/git_reinstall.py +++ b/actions/git_reinstall.py @@ -18,6 +18,10 @@ from glance_utils import ( git_install, ) +from glance_relations import ( + config_changed, +) + def git_reinstall(): """Reinstall from source and restart services. @@ -38,3 +42,4 @@ def git_reinstall(): if __name__ == '__main__': git_reinstall() + config_changed() diff --git a/unit_tests/test_actions_git_reinstall.py b/unit_tests/test_actions_git_reinstall.py index 1ba56855..fb631bb2 100644 --- a/unit_tests/test_actions_git_reinstall.py +++ b/unit_tests/test_actions_git_reinstall.py @@ -3,7 +3,8 @@ import os os.environ['JUJU_UNIT_NAME'] = 'glance' -import git_reinstall +with patch('glance_utils.register_configs') as register_configs: + import git_reinstall from test_utils import ( CharmTestCase @@ -72,7 +73,7 @@ class TestGlanceActions(CharmTestCase): git_install.side_effect = e traceback = ( "Traceback (most recent call last):\n" - " File \"actions/git_reinstall.py\", line 33, in git_reinstall\n" + " File \"actions/git_reinstall.py\", line 37, in git_reinstall\n" " git_install(config(\'openstack-origin-git\'))\n" " File \"/usr/lib/python2.7/dist-packages/mock.py\", line 964, in __call__\n" # noqa " return _mock_self._mock_call(*args, **kwargs)\n" From b91121072b26aa5ac659038276ba5c4003947b67 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 15 Apr 2015 14:47:18 +0000 Subject: [PATCH 13/33] Fixup test_git_reinstall_exception() --- unit_tests/test_actions_git_reinstall.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/unit_tests/test_actions_git_reinstall.py b/unit_tests/test_actions_git_reinstall.py index fb631bb2..060d8ea8 100644 --- a/unit_tests/test_actions_git_reinstall.py +++ b/unit_tests/test_actions_git_reinstall.py @@ -65,9 +65,10 @@ class TestGlanceActions(CharmTestCase): @patch.object(git_reinstall, 'action_set') @patch.object(git_reinstall, 'action_fail') @patch.object(git_reinstall, 'git_install') + @patch('traceback.format_exc') @patch('charmhelpers.contrib.openstack.utils.config') - def test_git_reinstall_exception(self, _config, git_install, action_fail, - action_set): + def test_git_reinstall_exception(self, _config, format_exc, git_install, + action_fail, action_set): _config.return_value = openstack_origin_git e = OSError('something bad happened') git_install.side_effect = e @@ -80,6 +81,7 @@ class TestGlanceActions(CharmTestCase): " File \"/usr/lib/python2.7/dist-packages/mock.py\", line 1019, in _mock_call\n" # noqa " raise effect\n" "OSError: something bad happened\n") + format_exc.return_value = traceback git_reinstall.git_reinstall() From 5eaed90e5481c00f19301de6a537a78a5b3476d9 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 15 Apr 2015 15:23:50 +0000 Subject: [PATCH 14/33] Sync charm-helpers --- .../contrib/openstack/amulet/deployment.py | 2 +- hooks/charmhelpers/contrib/openstack/context.py | 15 +++++++++++++++ hooks/charmhelpers/contrib/openstack/neutron.py | 13 +++++++++++++ .../contrib/openstack/templates/git.upstart | 2 ++ hooks/charmhelpers/core/strutils.py | 4 ++-- tests/charmhelpers/contrib/amulet/utils.py | 4 +++- .../contrib/openstack/amulet/deployment.py | 2 +- 7 files changed, 37 insertions(+), 5 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 0e0db566..fef96384 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -44,7 +44,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): Determine if the local branch being tested is derived from its stable or next (dev) branch, and based on this, use the corresonding stable or next branches for the other_services.""" - base_charms = ['mysql', 'mongodb', 'rabbitmq-server'] + base_charms = ['mysql', 'mongodb'] if self.stable: for svc in other_services: diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index dd51bfbb..c9914d0d 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -808,6 +808,19 @@ class NeutronContext(OSContextGenerator): return ovs_ctxt + def nuage_ctxt(self): + driver = neutron_plugin_attribute(self.plugin, 'driver', + self.network_manager) + config = neutron_plugin_attribute(self.plugin, 'config', + self.network_manager) + nuage_ctxt = {'core_plugin': driver, + 'neutron_plugin': 'vsp', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': config} + + return nuage_ctxt + def nvp_ctxt(self): driver = neutron_plugin_attribute(self.plugin, 'driver', self.network_manager) @@ -891,6 +904,8 @@ class NeutronContext(OSContextGenerator): ctxt.update(self.n1kv_ctxt()) elif self.plugin == 'Calico': ctxt.update(self.calico_ctxt()) + elif self.plugin == 'vsp': + ctxt.update(self.nuage_ctxt()) alchemy_flags = config('neutron-alchemy-flags') if alchemy_flags: diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index f8851050..02c92e9c 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -180,6 +180,19 @@ def neutron_plugins(): 'nova-api-metadata']], 'server_packages': ['neutron-server', 'calico-control'], 'server_services': ['neutron-server'] + }, + 'vsp': { + 'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini', + 'driver': 'neutron.plugins.nuage.plugin.NuagePlugin', + 'contexts': [ + context.SharedDBContext(user=config('neutron-database-user'), + database=config('neutron-database'), + relation_prefix='neutron', + ssl_dir=NEUTRON_CONF_DIR)], + 'services': [], + 'packages': [], + 'server_packages': ['neutron-server', 'neutron-plugin-nuage'], + 'server_services': ['neutron-server'] } } if release >= 'icehouse': diff --git a/hooks/charmhelpers/contrib/openstack/templates/git.upstart b/hooks/charmhelpers/contrib/openstack/templates/git.upstart index 62af9204..4bed404b 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/git.upstart +++ b/hooks/charmhelpers/contrib/openstack/templates/git.upstart @@ -12,4 +12,6 @@ exec start-stop-daemon --start --chuid {{ user_name }} \ {% for config_file in config_files -%} --config-file={{ config_file }} \ {% endfor -%} + {% if log_file -%} --log-file={{ log_file }} + {% endif -%} diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py index efc4402e..a2a784aa 100644 --- a/hooks/charmhelpers/core/strutils.py +++ b/hooks/charmhelpers/core/strutils.py @@ -33,9 +33,9 @@ def bool_from_string(value): value = value.strip().lower() - if value in ['y', 'yes', 'true', 't']: + if value in ['y', 'yes', 'true', 't', 'on']: return True - elif value in ['n', 'no', 'false', 'f']: + elif value in ['n', 'no', 'false', 'f', 'off']: return False msg = "Unable to interpret string value '%s' as boolean" % (value) diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 65219d33..5088b1d1 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -118,6 +118,9 @@ class AmuletUtils(object): longs, or can be a function that evaluate a variable and returns a bool. """ + self.log.debug('actual: {}'.format(repr(actual))) + self.log.debug('expected: {}'.format(repr(expected))) + for k, v in six.iteritems(expected): if k in actual: if (isinstance(v, six.string_types) or @@ -134,7 +137,6 @@ class AmuletUtils(object): def validate_relation_data(self, sentry_unit, relation, expected): """Validate actual relation data based on expected relation data.""" actual = sentry_unit.relation(relation[0], relation[1]) - self.log.debug('actual: {}'.format(repr(actual))) return self._validate_dict_data(expected, actual) def _validate_list_data(self, expected, actual): diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 0e0db566..fef96384 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -44,7 +44,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): Determine if the local branch being tested is derived from its stable or next (dev) branch, and based on this, use the corresonding stable or next branches for the other_services.""" - base_charms = ['mysql', 'mongodb', 'rabbitmq-server'] + base_charms = ['mysql', 'mongodb'] if self.stable: for svc in other_services: From 640b76f65ecc82a1831e48584e012d0175e730db Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 15 Apr 2015 16:29:59 +0000 Subject: [PATCH 15/33] Move config_changed into try block --- actions/git_reinstall.py | 2 +- unit_tests/test_actions_git_reinstall.py | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/actions/git_reinstall.py b/actions/git_reinstall.py index 7044ba43..db21c882 100755 --- a/actions/git_reinstall.py +++ b/actions/git_reinstall.py @@ -35,6 +35,7 @@ def git_reinstall(): try: git_install(config('openstack-origin-git')) + config_changed() except: action_set({'traceback': traceback.format_exc()}) action_fail('git-reinstall resulted in an unexpected error') @@ -42,4 +43,3 @@ def git_reinstall(): if __name__ == '__main__': git_reinstall() - config_changed() diff --git a/unit_tests/test_actions_git_reinstall.py b/unit_tests/test_actions_git_reinstall.py index 060d8ea8..46972c1f 100644 --- a/unit_tests/test_actions_git_reinstall.py +++ b/unit_tests/test_actions_git_reinstall.py @@ -34,9 +34,10 @@ class TestGlanceActions(CharmTestCase): @patch.object(git_reinstall, 'action_set') @patch.object(git_reinstall, 'action_fail') @patch.object(git_reinstall, 'git_install') + @patch.object(git_reinstall, 'config_changed') @patch('charmhelpers.contrib.openstack.utils.config') - def test_git_reinstall(self, _config, git_install, action_fail, - action_set): + def test_git_reinstall(self, _config, config_changed, git_install, + action_fail, action_set): _config.return_value = openstack_origin_git self.test_config.set('openstack-origin-git', openstack_origin_git) @@ -44,15 +45,18 @@ class TestGlanceActions(CharmTestCase): git_install.assert_called_with(openstack_origin_git) self.assertTrue(git_install.called) + self.assertTrue(config_changed.called) self.assertFalse(action_set.called) self.assertFalse(action_fail.called) @patch.object(git_reinstall, 'action_set') @patch.object(git_reinstall, 'action_fail') @patch.object(git_reinstall, 'git_install') + @patch.object(git_reinstall, 'config_changed') @patch('charmhelpers.contrib.openstack.utils.config') - def test_git_reinstall_not_configured(self, _config, git_install, - action_fail, action_set): + def test_git_reinstall_not_configured(self, _config, config_changed, + git_install, action_fail, + action_set): _config.return_value = None git_reinstall.git_reinstall() @@ -65,10 +69,12 @@ class TestGlanceActions(CharmTestCase): @patch.object(git_reinstall, 'action_set') @patch.object(git_reinstall, 'action_fail') @patch.object(git_reinstall, 'git_install') + @patch.object(git_reinstall, 'config_changed') @patch('traceback.format_exc') @patch('charmhelpers.contrib.openstack.utils.config') - def test_git_reinstall_exception(self, _config, format_exc, git_install, - action_fail, action_set): + def test_git_reinstall_exception(self, _config, format_exc, + config_changed, git_install, action_fail, + action_set): _config.return_value = openstack_origin_git e = OSError('something bad happened') git_install.side_effect = e From 902cd8bee549469a3358f1fcfe99c4e5dda3ae96 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 16 Apr 2015 11:26:34 +0100 Subject: [PATCH 16/33] [gnuoy,trivial] Pre-release charmhelper sync --- .../contrib/openstack/amulet/deployment.py | 29 ++- .../charmhelpers/contrib/openstack/context.py | 169 +++++++++++++++- .../charmhelpers/contrib/openstack/neutron.py | 13 ++ .../contrib/openstack/templates/git.upstart | 17 ++ .../templates/{zeromq => section-zeromq} | 4 +- hooks/charmhelpers/contrib/openstack/utils.py | 185 ++++++++++++------ hooks/charmhelpers/core/hookenv.py | 15 +- hooks/charmhelpers/core/strutils.py | 4 +- hooks/charmhelpers/core/unitdata.py | 2 +- tests/charmhelpers/contrib/amulet/utils.py | 4 +- .../contrib/openstack/amulet/deployment.py | 29 ++- 11 files changed, 392 insertions(+), 79 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/templates/git.upstart rename hooks/charmhelpers/contrib/openstack/templates/{zeromq => section-zeromq} (66%) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 0cfeaa4c..fef96384 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -15,6 +15,7 @@ # along with charm-helpers. If not, see . import six +from collections import OrderedDict from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment ) @@ -43,7 +44,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): Determine if the local branch being tested is derived from its stable or next (dev) branch, and based on this, use the corresonding stable or next branches for the other_services.""" - base_charms = ['mysql', 'mongodb', 'rabbitmq-server'] + base_charms = ['mysql', 'mongodb'] if self.stable: for svc in other_services: @@ -100,12 +101,34 @@ class OpenStackAmuletDeployment(AmuletDeployment): """ (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse) = range(6) + self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8) releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, ('precise', 'cloud:precise-grizzly'): self.precise_grizzly, ('precise', 'cloud:precise-havana'): self.precise_havana, ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, - ('trusty', None): self.trusty_icehouse} + ('trusty', None): self.trusty_icehouse, + ('trusty', 'cloud:trusty-juno'): self.trusty_juno, + ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo} return releases[(self.series, self.openstack)] + + def _get_openstack_release_string(self): + """Get openstack release string. + + Return a string representing the openstack release. + """ + releases = OrderedDict([ + ('precise', 'essex'), + ('quantal', 'folsom'), + ('raring', 'grizzly'), + ('saucy', 'havana'), + ('trusty', 'icehouse'), + ('utopic', 'juno'), + ('vivid', 'kilo'), + ]) + if self.openstack: + os_origin = self.openstack.split(':')[1] + return os_origin.split('%s-' % self.series)[1].split('/')[0] + else: + return releases[self.series] diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 90ac6d69..c9914d0d 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -47,6 +47,7 @@ from charmhelpers.core.hookenv import ( ) from charmhelpers.core.sysctl import create as sysctl_create +from charmhelpers.core.strutils import bool_from_string from charmhelpers.core.host import ( list_nics, @@ -67,6 +68,7 @@ from charmhelpers.contrib.hahelpers.apache import ( ) from charmhelpers.contrib.openstack.neutron import ( neutron_plugin_attribute, + parse_data_port_mappings, ) from charmhelpers.contrib.openstack.ip import ( resolve_address, @@ -82,7 +84,6 @@ from charmhelpers.contrib.network.ip import ( is_bridge_member, ) from charmhelpers.contrib.openstack.utils import get_host_ip - CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' ADDRESS_TYPES = ['admin', 'internal', 'public'] @@ -319,14 +320,15 @@ def db_ssl(rdata, ctxt, ssl_dir): class IdentityServiceContext(OSContextGenerator): - interfaces = ['identity-service'] - def __init__(self, service=None, service_user=None): + def __init__(self, service=None, service_user=None, rel_name='identity-service'): self.service = service self.service_user = service_user + self.rel_name = rel_name + self.interfaces = [self.rel_name] def __call__(self): - log('Generating template context for identity-service', level=DEBUG) + log('Generating template context for ' + self.rel_name, level=DEBUG) ctxt = {} if self.service and self.service_user: @@ -340,7 +342,7 @@ class IdentityServiceContext(OSContextGenerator): ctxt['signing_dir'] = cachedir - for rid in relation_ids('identity-service'): + for rid in relation_ids(self.rel_name): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) serv_host = rdata.get('service_host') @@ -806,6 +808,19 @@ class NeutronContext(OSContextGenerator): return ovs_ctxt + def nuage_ctxt(self): + driver = neutron_plugin_attribute(self.plugin, 'driver', + self.network_manager) + config = neutron_plugin_attribute(self.plugin, 'config', + self.network_manager) + nuage_ctxt = {'core_plugin': driver, + 'neutron_plugin': 'vsp', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': config} + + return nuage_ctxt + def nvp_ctxt(self): driver = neutron_plugin_attribute(self.plugin, 'driver', self.network_manager) @@ -889,6 +904,8 @@ class NeutronContext(OSContextGenerator): ctxt.update(self.n1kv_ctxt()) elif self.plugin == 'Calico': ctxt.update(self.calico_ctxt()) + elif self.plugin == 'vsp': + ctxt.update(self.nuage_ctxt()) alchemy_flags = config('neutron-alchemy-flags') if alchemy_flags: @@ -1162,3 +1179,145 @@ class SysctlContext(OSContextGenerator): sysctl_create(sysctl_dict, '/etc/sysctl.d/50-{0}.conf'.format(charm_name())) return {'sysctl': sysctl_dict} + + +class NeutronAPIContext(OSContextGenerator): + ''' + Inspects current neutron-plugin-api relation for neutron settings. Return + defaults if it is not present. + ''' + interfaces = ['neutron-plugin-api'] + + def __call__(self): + self.neutron_defaults = { + 'l2_population': { + 'rel_key': 'l2-population', + 'default': False, + }, + 'overlay_network_type': { + 'rel_key': 'overlay-network-type', + 'default': 'gre', + }, + 'neutron_security_groups': { + 'rel_key': 'neutron-security-groups', + 'default': False, + }, + 'network_device_mtu': { + 'rel_key': 'network-device-mtu', + 'default': None, + }, + 'enable_dvr': { + 'rel_key': 'enable-dvr', + 'default': False, + }, + 'enable_l3ha': { + 'rel_key': 'enable-l3ha', + 'default': False, + }, + } + ctxt = self.get_neutron_options({}) + for rid in relation_ids('neutron-plugin-api'): + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + if 'l2-population' in rdata: + ctxt.update(self.get_neutron_options(rdata)) + + return ctxt + + def get_neutron_options(self, rdata): + settings = {} + for nkey in self.neutron_defaults.keys(): + defv = self.neutron_defaults[nkey]['default'] + rkey = self.neutron_defaults[nkey]['rel_key'] + if rkey in rdata.keys(): + if type(defv) is bool: + settings[nkey] = bool_from_string(rdata[rkey]) + else: + settings[nkey] = rdata[rkey] + else: + settings[nkey] = defv + return settings + + +class ExternalPortContext(NeutronPortContext): + + def __call__(self): + ctxt = {} + ports = config('ext-port') + if ports: + ports = [p.strip() for p in ports.split()] + ports = self.resolve_ports(ports) + if ports: + ctxt = {"ext_port": ports[0]} + napi_settings = NeutronAPIContext()() + mtu = napi_settings.get('network_device_mtu') + if mtu: + ctxt['ext_port_mtu'] = mtu + + return ctxt + + +class DataPortContext(NeutronPortContext): + + def __call__(self): + ports = config('data-port') + if ports: + portmap = parse_data_port_mappings(ports) + ports = portmap.values() + resolved = self.resolve_ports(ports) + normalized = {get_nic_hwaddr(port): port for port in resolved + if port not in ports} + normalized.update({port: port for port in resolved + if port in ports}) + if resolved: + return {bridge: normalized[port] for bridge, port in + six.iteritems(portmap) if port in normalized.keys()} + + return None + + +class PhyNICMTUContext(DataPortContext): + + def __call__(self): + ctxt = {} + mappings = super(PhyNICMTUContext, self).__call__() + if mappings and mappings.values(): + ports = mappings.values() + napi_settings = NeutronAPIContext()() + mtu = napi_settings.get('network_device_mtu') + if mtu: + ctxt["devs"] = '\\n'.join(ports) + ctxt['mtu'] = mtu + + return ctxt + + +class NetworkServiceContext(OSContextGenerator): + + def __init__(self, rel_name='quantum-network-service'): + self.rel_name = rel_name + self.interfaces = [rel_name] + + def __call__(self): + for rid in relation_ids(self.rel_name): + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + ctxt = { + 'keystone_host': rdata.get('keystone_host'), + 'service_port': rdata.get('service_port'), + 'auth_port': rdata.get('auth_port'), + 'service_tenant': rdata.get('service_tenant'), + 'service_username': rdata.get('service_username'), + 'service_password': rdata.get('service_password'), + 'quantum_host': rdata.get('quantum_host'), + 'quantum_port': rdata.get('quantum_port'), + 'quantum_url': rdata.get('quantum_url'), + 'region': rdata.get('region'), + 'service_protocol': + rdata.get('service_protocol') or 'http', + 'auth_protocol': + rdata.get('auth_protocol') or 'http', + } + if context_complete(ctxt): + return ctxt + return {} diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index f8851050..02c92e9c 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -180,6 +180,19 @@ def neutron_plugins(): 'nova-api-metadata']], 'server_packages': ['neutron-server', 'calico-control'], 'server_services': ['neutron-server'] + }, + 'vsp': { + 'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini', + 'driver': 'neutron.plugins.nuage.plugin.NuagePlugin', + 'contexts': [ + context.SharedDBContext(user=config('neutron-database-user'), + database=config('neutron-database'), + relation_prefix='neutron', + ssl_dir=NEUTRON_CONF_DIR)], + 'services': [], + 'packages': [], + 'server_packages': ['neutron-server', 'neutron-plugin-nuage'], + 'server_services': ['neutron-server'] } } if release >= 'icehouse': diff --git a/hooks/charmhelpers/contrib/openstack/templates/git.upstart b/hooks/charmhelpers/contrib/openstack/templates/git.upstart new file mode 100644 index 00000000..4bed404b --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/templates/git.upstart @@ -0,0 +1,17 @@ +description "{{ service_description }}" +author "Juju {{ service_name }} Charm " + +start on runlevel [2345] +stop on runlevel [!2345] + +respawn + +exec start-stop-daemon --start --chuid {{ user_name }} \ + --chdir {{ start_dir }} --name {{ process_name }} \ + --exec {{ executable_name }} -- \ + {% for config_file in config_files -%} + --config-file={{ config_file }} \ + {% endfor -%} + {% if log_file -%} + --log-file={{ log_file }} + {% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/zeromq b/hooks/charmhelpers/contrib/openstack/templates/section-zeromq similarity index 66% rename from hooks/charmhelpers/contrib/openstack/templates/zeromq rename to hooks/charmhelpers/contrib/openstack/templates/section-zeromq index 0695eef1..95f1a76c 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/zeromq +++ b/hooks/charmhelpers/contrib/openstack/templates/section-zeromq @@ -3,12 +3,12 @@ rpc_backend = zmq rpc_zmq_host = {{ zmq_host }} {% if zmq_redis_address -%} -rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_redis.MatchMakerRedis +rpc_zmq_matchmaker = redis matchmaker_heartbeat_freq = 15 matchmaker_heartbeat_ttl = 30 [matchmaker_redis] host = {{ zmq_redis_address }} {% else -%} -rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_ring.MatchMakerRing +rpc_zmq_matchmaker = ring {% endif -%} {% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 4f110c63..5a12c9d6 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -30,6 +30,10 @@ import yaml from charmhelpers.contrib.network import ip +from charmhelpers.core import ( + unitdata, +) + from charmhelpers.core.hookenv import ( config, log as juju_log, @@ -330,6 +334,21 @@ def configure_installation_source(rel): error_out("Invalid openstack-release specified: %s" % rel) +def config_value_changed(option): + """ + Determine if config value changed since last call to this function. + """ + hook_data = unitdata.HookData() + with hook_data(): + db = unitdata.kv() + current = config(option) + saved = db.get(option) + db.set(option, current) + if saved is None: + return False + return current != saved + + def save_script_rc(script_path="scripts/scriptrc", **env_vars): """ Write an rc file in the charm-delivered directory containing @@ -469,82 +488,103 @@ def os_requires_version(ostack_release, pkg): def git_install_requested(): - """Returns true if openstack-origin-git is specified.""" - return config('openstack-origin-git') != "None" + """ + Returns true if openstack-origin-git is specified. + """ + return config('openstack-origin-git') is not None requirements_dir = None -def git_clone_and_install(file_name, core_project): - """Clone/install all OpenStack repos specified in yaml config file.""" - global requirements_dir +def git_clone_and_install(projects_yaml, core_project): + """ + Clone/install all specified OpenStack repositories. - if file_name == "None": + The expected format of projects_yaml is: + repositories: + - {name: keystone, + repository: 'git://git.openstack.org/openstack/keystone.git', + branch: 'stable/icehouse'} + - {name: requirements, + repository: 'git://git.openstack.org/openstack/requirements.git', + branch: 'stable/icehouse'} + directory: /mnt/openstack-git + http_proxy: http://squid.internal:3128 + https_proxy: https://squid.internal:3128 + + The directory, http_proxy, and https_proxy keys are optional. + """ + global requirements_dir + parent_dir = '/mnt/openstack-git' + + if not projects_yaml: return - yaml_file = os.path.join(charm_dir(), file_name) + projects = yaml.load(projects_yaml) + _git_validate_projects_yaml(projects, core_project) - # clone/install the requirements project first - installed = _git_clone_and_install_subset(yaml_file, - whitelist=['requirements']) - if 'requirements' not in installed: - error_out('requirements git repository must be specified') + if 'http_proxy' in projects.keys(): + os.environ['http_proxy'] = projects['http_proxy'] - # clone/install all other projects except requirements and the core project - blacklist = ['requirements', core_project] - _git_clone_and_install_subset(yaml_file, blacklist=blacklist, - update_requirements=True) + if 'https_proxy' in projects.keys(): + os.environ['https_proxy'] = projects['https_proxy'] - # clone/install the core project - whitelist = [core_project] - installed = _git_clone_and_install_subset(yaml_file, whitelist=whitelist, - update_requirements=True) - if core_project not in installed: - error_out('{} git repository must be specified'.format(core_project)) + if 'directory' in projects.keys(): + parent_dir = projects['directory'] + + for p in projects['repositories']: + repo = p['repository'] + branch = p['branch'] + if p['name'] == 'requirements': + repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + update_requirements=False) + requirements_dir = repo_dir + else: + repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + update_requirements=True) -def _git_clone_and_install_subset(yaml_file, whitelist=[], blacklist=[], - update_requirements=False): - """Clone/install subset of OpenStack repos specified in yaml config file.""" - global requirements_dir - installed = [] +def _git_validate_projects_yaml(projects, core_project): + """ + Validate the projects yaml. + """ + _git_ensure_key_exists('repositories', projects) - with open(yaml_file, 'r') as fd: - projects = yaml.load(fd) - for proj, val in projects.items(): - # The project subset is chosen based on the following 3 rules: - # 1) If project is in blacklist, we don't clone/install it, period. - # 2) If whitelist is empty, we clone/install everything else. - # 3) If whitelist is not empty, we clone/install everything in the - # whitelist. - if proj in blacklist: - continue - if whitelist and proj not in whitelist: - continue - repo = val['repository'] - branch = val['branch'] - repo_dir = _git_clone_and_install_single(repo, branch, - update_requirements) - if proj == 'requirements': - requirements_dir = repo_dir - installed.append(proj) - return installed + for project in projects['repositories']: + _git_ensure_key_exists('name', project.keys()) + _git_ensure_key_exists('repository', project.keys()) + _git_ensure_key_exists('branch', project.keys()) + + if projects['repositories'][0]['name'] != 'requirements': + error_out('{} git repo must be specified first'.format('requirements')) + + if projects['repositories'][-1]['name'] != core_project: + error_out('{} git repo must be specified last'.format(core_project)) -def _git_clone_and_install_single(repo, branch, update_requirements=False): - """Clone and install a single git repository.""" - dest_parent_dir = "/mnt/openstack-git/" - dest_dir = os.path.join(dest_parent_dir, os.path.basename(repo)) +def _git_ensure_key_exists(key, keys): + """ + Ensure that key exists in keys. + """ + if key not in keys: + error_out('openstack-origin-git key \'{}\' is missing'.format(key)) - if not os.path.exists(dest_parent_dir): - juju_log('Host dir not mounted at {}. ' - 'Creating directory there instead.'.format(dest_parent_dir)) - os.mkdir(dest_parent_dir) + +def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements): + """ + Clone and install a single git repository. + """ + dest_dir = os.path.join(parent_dir, os.path.basename(repo)) + + if not os.path.exists(parent_dir): + juju_log('Directory already exists at {}. ' + 'No need to create directory.'.format(parent_dir)) + os.mkdir(parent_dir) if not os.path.exists(dest_dir): juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) - repo_dir = install_remote(repo, dest=dest_parent_dir, branch=branch) + repo_dir = install_remote(repo, dest=parent_dir, branch=branch) else: repo_dir = dest_dir @@ -561,16 +601,39 @@ def _git_clone_and_install_single(repo, branch, update_requirements=False): def _git_update_requirements(package_dir, reqs_dir): - """Update from global requirements. + """ + Update from global requirements. - Update an OpenStack git directory's requirements.txt and - test-requirements.txt from global-requirements.txt.""" + Update an OpenStack git directory's requirements.txt and + test-requirements.txt from global-requirements.txt. + """ orig_dir = os.getcwd() os.chdir(reqs_dir) - cmd = "python update.py {}".format(package_dir) + cmd = ['python', 'update.py', package_dir] try: - subprocess.check_call(cmd.split(' ')) + subprocess.check_call(cmd) except subprocess.CalledProcessError: package = os.path.basename(package_dir) error_out("Error updating {} from global-requirements.txt".format(package)) os.chdir(orig_dir) + + +def git_src_dir(projects_yaml, project): + """ + Return the directory where the specified project's source is located. + """ + parent_dir = '/mnt/openstack-git' + + if not projects_yaml: + return + + projects = yaml.load(projects_yaml) + + if 'directory' in projects.keys(): + parent_dir = projects['directory'] + + for p in projects['repositories']: + if p['name'] == project: + return os.path.join(parent_dir, os.path.basename(p['repository'])) + + return None diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index 715dd4c5..86f805f1 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -20,11 +20,13 @@ # Authors: # Charm Helpers Developers +from __future__ import print_function import os import json import yaml import subprocess import sys +import errno from subprocess import CalledProcessError import six @@ -87,7 +89,18 @@ def log(message, level=None): if not isinstance(message, six.string_types): message = repr(message) command += [message] - subprocess.call(command) + # Missing juju-log should not cause failures in unit tests + # Send log output to stderr + try: + subprocess.call(command) + except OSError as e: + if e.errno == errno.ENOENT: + if level: + message = "{}: {}".format(level, message) + message = "juju-log: {}".format(message) + print(message, file=sys.stderr) + else: + raise class Serializable(UserDict): diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py index efc4402e..a2a784aa 100644 --- a/hooks/charmhelpers/core/strutils.py +++ b/hooks/charmhelpers/core/strutils.py @@ -33,9 +33,9 @@ def bool_from_string(value): value = value.strip().lower() - if value in ['y', 'yes', 'true', 't']: + if value in ['y', 'yes', 'true', 't', 'on']: return True - elif value in ['n', 'no', 'false', 'f']: + elif value in ['n', 'no', 'false', 'f', 'off']: return False msg = "Unable to interpret string value '%s' as boolean" % (value) diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py index 3000134a..406a35c5 100644 --- a/hooks/charmhelpers/core/unitdata.py +++ b/hooks/charmhelpers/core/unitdata.py @@ -443,7 +443,7 @@ class HookData(object): data = hookenv.execution_environment() self.conf = conf_delta = self.kv.delta(data['conf'], 'config') self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') - self.kv.set('env', data['env']) + self.kv.set('env', dict(data['env'])) self.kv.set('unit', data['unit']) self.kv.set('relid', data.get('relid')) return conf_delta, rels_delta diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 65219d33..5088b1d1 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -118,6 +118,9 @@ class AmuletUtils(object): longs, or can be a function that evaluate a variable and returns a bool. """ + self.log.debug('actual: {}'.format(repr(actual))) + self.log.debug('expected: {}'.format(repr(expected))) + for k, v in six.iteritems(expected): if k in actual: if (isinstance(v, six.string_types) or @@ -134,7 +137,6 @@ class AmuletUtils(object): def validate_relation_data(self, sentry_unit, relation, expected): """Validate actual relation data based on expected relation data.""" actual = sentry_unit.relation(relation[0], relation[1]) - self.log.debug('actual: {}'.format(repr(actual))) return self._validate_dict_data(expected, actual) def _validate_list_data(self, expected, actual): diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 0cfeaa4c..fef96384 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -15,6 +15,7 @@ # along with charm-helpers. If not, see . import six +from collections import OrderedDict from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment ) @@ -43,7 +44,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): Determine if the local branch being tested is derived from its stable or next (dev) branch, and based on this, use the corresonding stable or next branches for the other_services.""" - base_charms = ['mysql', 'mongodb', 'rabbitmq-server'] + base_charms = ['mysql', 'mongodb'] if self.stable: for svc in other_services: @@ -100,12 +101,34 @@ class OpenStackAmuletDeployment(AmuletDeployment): """ (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse) = range(6) + self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8) releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, ('precise', 'cloud:precise-grizzly'): self.precise_grizzly, ('precise', 'cloud:precise-havana'): self.precise_havana, ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, - ('trusty', None): self.trusty_icehouse} + ('trusty', None): self.trusty_icehouse, + ('trusty', 'cloud:trusty-juno'): self.trusty_juno, + ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo} return releases[(self.series, self.openstack)] + + def _get_openstack_release_string(self): + """Get openstack release string. + + Return a string representing the openstack release. + """ + releases = OrderedDict([ + ('precise', 'essex'), + ('quantal', 'folsom'), + ('raring', 'grizzly'), + ('saucy', 'havana'), + ('trusty', 'icehouse'), + ('utopic', 'juno'), + ('vivid', 'kilo'), + ]) + if self.openstack: + os_origin = self.openstack.split(':')[1] + return os_origin.split('%s-' % self.series)[1].split('/')[0] + else: + return releases[self.series] From f61c2ff1b4ee49f9f9cdcbb7660db19ac32a2f53 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 16 Apr 2015 14:41:46 +0000 Subject: [PATCH 17/33] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 5a12c9d6..f90a0289 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -524,9 +524,10 @@ def git_clone_and_install(projects_yaml, core_project): projects = yaml.load(projects_yaml) _git_validate_projects_yaml(projects, core_project) + old_environ = dict(os.environ) + if 'http_proxy' in projects.keys(): os.environ['http_proxy'] = projects['http_proxy'] - if 'https_proxy' in projects.keys(): os.environ['https_proxy'] = projects['https_proxy'] @@ -544,6 +545,8 @@ def git_clone_and_install(projects_yaml, core_project): repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, update_requirements=True) + os.environ = old_environ + def _git_validate_projects_yaml(projects, core_project): """ From 078c6bcc234ccb7366ba904163fc1d62e2d22b1d Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Thu, 16 Apr 2015 21:31:48 +0000 Subject: [PATCH 18/33] auto set -x on amulet basic_deployment.py --- tests/basic_deployment.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 tests/basic_deployment.py diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py old mode 100755 new mode 100644 From 0311258a7dbf13642d8e6865285cf2e7176c2b28 Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Thu, 16 Apr 2015 21:31:53 +0000 Subject: [PATCH 19/33] auto add AMULET_OS_VIP to preserved env vars in makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3b45e0fc..024aeb55 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ test: # /!\ Note: The -v should only be temporary until Amulet sends # raise_status() messages to stderr: # https://bugs.launchpad.net/amulet/+bug/1320357 - @juju test -v -p AMULET_HTTP_PROXY --timeout 900 \ + @juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 900 \ 00-setup 14-basic-precise-icehouse 15-basic-trusty-icehouse \ 16-basic-trusty-icehouse-git 17-basic-trusty-juno \ 18-basic-trusty-juno-git From f4baf629a0634ffe80dfcb6ccbb57d6620f2f484 Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Thu, 16 Apr 2015 21:32:02 +0000 Subject: [PATCH 20/33] auto Makefile test target (amulet): bump juju test timeout to 2700s (same value as the juju-deployer default). Also remove explicit test names, which will cause all +x files in ./tests to be executed (as bundletester does by default). --- Makefile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 024aeb55..bb075a0f 100644 --- a/Makefile +++ b/Makefile @@ -26,10 +26,7 @@ test: # /!\ Note: The -v should only be temporary until Amulet sends # raise_status() messages to stderr: # https://bugs.launchpad.net/amulet/+bug/1320357 - @juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 900 \ - 00-setup 14-basic-precise-icehouse 15-basic-trusty-icehouse \ - 16-basic-trusty-icehouse-git 17-basic-trusty-juno \ - 18-basic-trusty-juno-git + @juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700 publish: lint unit_test bzr push lp:charms/glance From 8abb02dbb97b2c3a6a53cda49f31bff8b10b986e Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Thu, 16 Apr 2015 21:33:32 +0000 Subject: [PATCH 21/33] auto sync charmhelpers --- hooks/charmhelpers/contrib/openstack/context.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index c9914d0d..400eaf8e 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -459,6 +459,11 @@ class AMQPContext(OSContextGenerator): ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts)) + oslo_messaging_flags = conf.get('oslo-messaging-flags', None) + if oslo_messaging_flags: + ctxt['oslo_messaging_flags'] = config_flags_parser( + oslo_messaging_flags) + if not context_complete(ctxt): return {} From 48da324ff658dbbb92a1ca92a268a3a23ff80453 Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Thu, 16 Apr 2015 21:35:55 +0000 Subject: [PATCH 22/33] auto rename amulet tests --- tests/{14-basic-precise-icehouse => 014-basic-precise-icehouse} | 0 tests/{15-basic-trusty-icehouse => 015-basic-trusty-icehouse} | 0 tests/{17-basic-trusty-juno => 016-basic-trusty-juno} | 0 ...16-basic-trusty-icehouse-git => 050-basic-trusty-icehouse-git} | 0 tests/{18-basic-trusty-juno-git => 051-basic-trusty-juno-git} | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename tests/{14-basic-precise-icehouse => 014-basic-precise-icehouse} (100%) rename tests/{15-basic-trusty-icehouse => 015-basic-trusty-icehouse} (100%) rename tests/{17-basic-trusty-juno => 016-basic-trusty-juno} (100%) rename tests/{16-basic-trusty-icehouse-git => 050-basic-trusty-icehouse-git} (100%) rename tests/{18-basic-trusty-juno-git => 051-basic-trusty-juno-git} (100%) diff --git a/tests/14-basic-precise-icehouse b/tests/014-basic-precise-icehouse similarity index 100% rename from tests/14-basic-precise-icehouse rename to tests/014-basic-precise-icehouse diff --git a/tests/15-basic-trusty-icehouse b/tests/015-basic-trusty-icehouse similarity index 100% rename from tests/15-basic-trusty-icehouse rename to tests/015-basic-trusty-icehouse diff --git a/tests/17-basic-trusty-juno b/tests/016-basic-trusty-juno similarity index 100% rename from tests/17-basic-trusty-juno rename to tests/016-basic-trusty-juno diff --git a/tests/16-basic-trusty-icehouse-git b/tests/050-basic-trusty-icehouse-git similarity index 100% rename from tests/16-basic-trusty-icehouse-git rename to tests/050-basic-trusty-icehouse-git diff --git a/tests/18-basic-trusty-juno-git b/tests/051-basic-trusty-juno-git similarity index 100% rename from tests/18-basic-trusty-juno-git rename to tests/051-basic-trusty-juno-git From 6b9c241d5ba1f130cbb4926f02d11f4c143677bb Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Thu, 16 Apr 2015 21:36:12 +0000 Subject: [PATCH 23/33] auto add amulet tests for supported releases --- tests/017-basic-trusty-kilo | 11 +++++++++++ tests/018-basic-utopic-juno | 9 +++++++++ tests/019-basic-vivid-kilo | 9 +++++++++ 3 files changed, 29 insertions(+) create mode 100755 tests/017-basic-trusty-kilo create mode 100755 tests/018-basic-utopic-juno create mode 100755 tests/019-basic-vivid-kilo diff --git a/tests/017-basic-trusty-kilo b/tests/017-basic-trusty-kilo new file mode 100755 index 00000000..85b31cdb --- /dev/null +++ b/tests/017-basic-trusty-kilo @@ -0,0 +1,11 @@ +#!/usr/bin/python + +"""Amulet tests on a basic glance deployment on trusty-kilo.""" + +from basic_deployment import GlanceBasicDeployment + +if __name__ == '__main__': + deployment = GlanceBasicDeployment(series='trusty', + openstack='cloud:trusty-kilo', + source='cloud:trusty-updates/kilo') + deployment.run_tests() diff --git a/tests/018-basic-utopic-juno b/tests/018-basic-utopic-juno new file mode 100755 index 00000000..528ec22f --- /dev/null +++ b/tests/018-basic-utopic-juno @@ -0,0 +1,9 @@ +#!/usr/bin/python + +"""Amulet tests on a basic Glance deployment on utopic-juno.""" + +from basic_deployment import GlanceBasicDeployment + +if __name__ == '__main__': + deployment = GlanceBasicDeployment(series='utopic') + deployment.run_tests() diff --git a/tests/019-basic-vivid-kilo b/tests/019-basic-vivid-kilo new file mode 100755 index 00000000..6ad57f8a --- /dev/null +++ b/tests/019-basic-vivid-kilo @@ -0,0 +1,9 @@ +#!/usr/bin/python + +"""Amulet tests on a basic Glance deployment on vivid-kilo.""" + +from basic_deployment import GlanceBasicDeployment + +if __name__ == '__main__': + deployment = GlanceBasicDeployment(series='vivid') + deployment.run_tests() From ac4d331adaa63f299d815c5c046c76e51650e0d7 Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Thu, 16 Apr 2015 21:36:23 +0000 Subject: [PATCH 24/33] auto disable kilo amulet tests (until later confirmed as functional) --- tests/017-basic-trusty-kilo | 0 tests/019-basic-vivid-kilo | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 tests/017-basic-trusty-kilo mode change 100755 => 100644 tests/019-basic-vivid-kilo diff --git a/tests/017-basic-trusty-kilo b/tests/017-basic-trusty-kilo old mode 100755 new mode 100644 diff --git a/tests/019-basic-vivid-kilo b/tests/019-basic-vivid-kilo old mode 100755 new mode 100644 From 19d565207e0c8aeb93eadf05029e818c731c1799 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 17 Apr 2015 12:05:48 +0000 Subject: [PATCH 25/33] Change default mkdir permissions to 755 for deploy from source --- hooks/glance_utils.py | 2 +- unit_tests/test_glance_utils.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/hooks/glance_utils.py b/hooks/glance_utils.py index db37500b..37de5682 100755 --- a/hooks/glance_utils.py +++ b/hooks/glance_utils.py @@ -332,7 +332,7 @@ def git_pre_install(): add_user_to_group('glance', 'glance') for d in dirs: - mkdir(d, owner='glance', group='glance', perms=0700, force=False) + mkdir(d, owner='glance', group='glance', perms=0755, force=False) for l in logs: write_file(l, '', owner='glance', group='glance', perms=0600) diff --git a/unit_tests/test_glance_utils.py b/unit_tests/test_glance_utils.py index 8b89ad9a..2ac4518e 100644 --- a/unit_tests/test_glance_utils.py +++ b/unit_tests/test_glance_utils.py @@ -210,19 +210,19 @@ class TestGlanceUtils(CharmTestCase): add_user_to_group.assert_called_with('glance', 'glance') expected = [ call('/var/lib/glance', owner='glance', - group='glance', perms=0700, force=False), + group='glance', perms=0755, force=False), call('/var/lib/glance/images', owner='glance', - group='glance', perms=0700, force=False), + group='glance', perms=0755, force=False), call('/var/lib/glance/image-cache', owner='glance', - group='glance', perms=0700, force=False), + group='glance', perms=0755, force=False), call('/var/lib/glance/image-cache/incomplete', owner='glance', - group='glance', perms=0700, force=False), + group='glance', perms=0755, force=False), call('/var/lib/glance/image-cache/invalid', owner='glance', - group='glance', perms=0700, force=False), + group='glance', perms=0755, force=False), call('/var/lib/glance/image-cache/queue', owner='glance', - group='glance', perms=0700, force=False), + group='glance', perms=0755, force=False), call('/var/log/glance', owner='glance', - group='glance', perms=0700, force=False), + group='glance', perms=0755, force=False), ] self.assertEquals(mkdir.call_args_list, expected) expected = [ From 4d9ca436986fa9238d4eafc85b193a12a8f3ec30 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sun, 19 Apr 2015 10:00:04 +0100 Subject: [PATCH 26/33] [gnuoy,trivial] Pre-release charmhelper sync --- hooks/charmhelpers/contrib/charmsupport/nrpe.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py index 9d961cfb..95a79c2e 100644 --- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py +++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py @@ -247,7 +247,9 @@ class NRPE(object): service('restart', 'nagios-nrpe-server') - for rid in relation_ids("local-monitors"): + monitor_ids = relation_ids("local-monitors") + \ + relation_ids("nrpe-external-master") + for rid in monitor_ids: relation_set(relation_id=rid, monitors=yaml.dump(monitors)) From c46dbfd331e0fd26d72620e4c2fe8c2143c3ab57 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 21 Apr 2015 16:07:19 +0100 Subject: [PATCH 27/33] Ensure that migration only occurs when unit can assure that mysql has granted it permissions to access the database --- hooks/glance_relations.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/hooks/glance_relations.py b/hooks/glance_relations.py index 8f2f14ce..7e9e2755 100755 --- a/hooks/glance_relations.py +++ b/hooks/glance_relations.py @@ -164,18 +164,19 @@ def db_changed(): # acl entry has been added. So, if the db supports passing a list of # permitted units then check if we're in the list. allowed_units = relation_get('allowed_units') - if allowed_units and local_unit() not in allowed_units.split(): - juju_log('Allowed_units list provided and this unit not present') - return - if rel == "essex": - status = call(['glance-manage', 'db_version']) - if status != 0: - juju_log('Setting version_control to 0') - cmd = ["glance-manage", "version_control", "0"] - check_call(cmd) + if allowed_units and local_unit() in allowed_units.split(): + if rel == "essex": + status = call(['glance-manage', 'db_version']) + if status != 0: + juju_log('Setting version_control to 0') + cmd = ["glance-manage", "version_control", "0"] + check_call(cmd) - juju_log('Cluster leader, performing db sync') - migrate_database() + juju_log('Cluster leader, performing db sync') + migrate_database() + else: + juju_log('allowed_units either not presented, or local unit ' + 'not in acl list: %s' % allowed_units) @hooks.hook('pgsql-db-relation-changed') From 747b33f7451bf4613051c80747f28bb0f78cf194 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 21 Apr 2015 16:31:22 +0100 Subject: [PATCH 28/33] Add unit tests to ensure that acls are setup prior to db migration --- unit_tests/test_glance_relations.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/unit_tests/test_glance_relations.py b/unit_tests/test_glance_relations.py index 246f8630..eabd6fbe 100644 --- a/unit_tests/test_glance_relations.py +++ b/unit_tests/test_glance_relations.py @@ -207,8 +207,9 @@ class GlanceRelationTests(CharmTestCase): 'pgsql-db relation incomplete. Peer not ready?' ) - def _shared_db_test(self, configs, unit_name): - self.relation_get.return_value = 'glance/0 glance/3' + def _shared_db_test(self, configs, unit_name, + allowed_units = 'glance/0 glance/3'): + self.relation_get.return_value = allowed_units self.local_unit.return_value = unit_name configs.complete_contexts = MagicMock() configs.complete_contexts.return_value = ['shared-db'] @@ -240,6 +241,15 @@ class GlanceRelationTests(CharmTestCase): configs.write.call_args_list) self.assertFalse(self.migrate_database.called) + @patch.object(relations, 'CONFIGS') + def test_db_changed_no_acls(self, configs): + self._shared_db_test(configs, 'glance/2', None) + self.assertEquals([call('/etc/glance/glance-registry.conf'), + call('/etc/glance/glance-api.conf')], + configs.write.call_args_list) + self.assertFalse(self.migrate_database.called) + + @patch.object(relations, 'CONFIGS') def test_postgresql_db_changed_no_essex(self, configs): self._postgresql_db_test(configs) From d2c018e7e2aa0866cc93399e1f504596faa82e72 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 21 Apr 2015 16:33:15 +0100 Subject: [PATCH 29/33] Tidy lint --- unit_tests/test_glance_relations.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/unit_tests/test_glance_relations.py b/unit_tests/test_glance_relations.py index eabd6fbe..e41abdd1 100644 --- a/unit_tests/test_glance_relations.py +++ b/unit_tests/test_glance_relations.py @@ -208,7 +208,7 @@ class GlanceRelationTests(CharmTestCase): ) def _shared_db_test(self, configs, unit_name, - allowed_units = 'glance/0 glance/3'): + allowed_units='glance/0 glance/3'): self.relation_get.return_value = allowed_units self.local_unit.return_value = unit_name configs.complete_contexts = MagicMock() @@ -249,7 +249,6 @@ class GlanceRelationTests(CharmTestCase): configs.write.call_args_list) self.assertFalse(self.migrate_database.called) - @patch.object(relations, 'CONFIGS') def test_postgresql_db_changed_no_essex(self, configs): self._postgresql_db_test(configs) From 5ead4fbd9bf853144161ff411f9353c778ba155c Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 23 Apr 2015 15:52:07 +0100 Subject: [PATCH 30/33] [gnuoy,trivial] Pre-release charmhelper sync --- .../contrib/openstack/amulet/deployment.py | 21 +++++++++++++------ tests/charmhelpers/contrib/amulet/utils.py | 9 +++++++- .../contrib/openstack/amulet/deployment.py | 21 +++++++++++++------ 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 11d49a7c..461a702f 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -46,15 +46,22 @@ class OpenStackAmuletDeployment(AmuletDeployment): stable or next branches for the other_services.""" base_charms = ['mysql', 'mongodb'] + if self.series in ['precise', 'trusty']: + base_series = self.series + else: + base_series = self.current_next + if self.stable: for svc in other_services: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: for svc in other_services: if svc['name'] in base_charms: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: temp = 'lp:~openstack-charmers/charms/{}/{}/next' svc['location'] = temp.format(self.current_next, @@ -99,10 +106,12 @@ class OpenStackAmuletDeployment(AmuletDeployment): Return an integer representing the enum value of the openstack release. """ + # Must be ordered by OpenStack release (not by Ubuntu release): (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse, self.trusty_juno, self.trusty_kilo, - self.utopic_juno, self.vivid_kilo) = range(10) + self.trusty_icehouse, self.trusty_juno, self.utopic_juno, + self.trusty_kilo, self.vivid_kilo) = range(10) + releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 5088b1d1..f61c2e8b 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -79,6 +79,9 @@ class AmuletUtils(object): for k, v in six.iteritems(commands): for cmd in v: output, code = k.run(cmd) + self.log.debug('{} `{}` returned ' + '{}'.format(k.info['unit_name'], + cmd, code)) if code != 0: return "command `{}` returned {}".format(cmd, str(code)) return None @@ -86,7 +89,11 @@ class AmuletUtils(object): def _get_config(self, unit, filename): """Get a ConfigParser object for parsing a unit's config file.""" file_contents = unit.file_contents(filename) - config = ConfigParser.ConfigParser() + + # NOTE(beisner): by default, ConfigParser does not handle options + # with no value, such as the flags used in the mysql my.cnf file. + # https://bugs.python.org/issue7005 + config = ConfigParser.ConfigParser(allow_no_value=True) config.readfp(io.StringIO(file_contents)) return config diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 11d49a7c..461a702f 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -46,15 +46,22 @@ class OpenStackAmuletDeployment(AmuletDeployment): stable or next branches for the other_services.""" base_charms = ['mysql', 'mongodb'] + if self.series in ['precise', 'trusty']: + base_series = self.series + else: + base_series = self.current_next + if self.stable: for svc in other_services: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: for svc in other_services: if svc['name'] in base_charms: - temp = 'lp:charms/{}' - svc['location'] = temp.format(svc['name']) + temp = 'lp:charms/{}/{}' + svc['location'] = temp.format(base_series, + svc['name']) else: temp = 'lp:~openstack-charmers/charms/{}/{}/next' svc['location'] = temp.format(self.current_next, @@ -99,10 +106,12 @@ class OpenStackAmuletDeployment(AmuletDeployment): Return an integer representing the enum value of the openstack release. """ + # Must be ordered by OpenStack release (not by Ubuntu release): (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse, self.trusty_juno, self.trusty_kilo, - self.utopic_juno, self.vivid_kilo) = range(10) + self.trusty_icehouse, self.trusty_juno, self.utopic_juno, + self.trusty_kilo, self.vivid_kilo) = range(10) + releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, From 30268e9e974a7c5ac53ca27c063cd898f839ddb4 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 24 Apr 2015 14:06:32 +0000 Subject: [PATCH 31/33] [corey.bryant,trivial] Fix deploy from source README --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ec1b348b..ad91cdde 100644 --- a/README.md +++ b/README.md @@ -86,8 +86,8 @@ Deploying from source The minimum openstack-origin-git config required to deploy from source is: - openstack-origin-git: - "repositories: + openstack-origin-git: + "repositories: - {name: requirements, repository: 'git://git.openstack.org/openstack/requirements', branch: stable/juno} @@ -103,8 +103,8 @@ in the order in which they are specified. The following is a full list of current tip repos (may not be up-to-date): - openstack-origin-git: - "repositories: + openstack-origin-git: + "repositories: - {name: requirements, repository: 'git://git.openstack.org/openstack/requirements', branch: master} From 1de569b0957cd8b0c5f2696fed8a00b349f5da9f Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 30 Apr 2015 10:49:30 +0000 Subject: [PATCH 32/33] [corey.bryant,trivial] Update deploy from source README samples. --- README.md | 138 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 71 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index ad91cdde..8fcca6d3 100644 --- a/README.md +++ b/README.md @@ -86,14 +86,16 @@ Deploying from source The minimum openstack-origin-git config required to deploy from source is: - openstack-origin-git: - "repositories: - - {name: requirements, - repository: 'git://git.openstack.org/openstack/requirements', - branch: stable/juno} - - {name: glance, - repository: 'git://git.openstack.org/openstack/glance', - branch: stable/juno}" + openstack-origin-git: include-file://glance-juno.yaml + + glance-juno.yaml + repositories: + - {name: requirements, + repository: 'git://github.com/openstack/requirements', + branch: stable/juno} + - {name: glance, + repository: 'git://github.com/openstack/glance', + branch: stable/juno} Note that there are only two 'name' values the charm knows about: 'requirements' and 'glance'. These repositories must correspond to these 'name' values. @@ -103,65 +105,67 @@ in the order in which they are specified. The following is a full list of current tip repos (may not be up-to-date): - openstack-origin-git: - "repositories: - - {name: requirements, - repository: 'git://git.openstack.org/openstack/requirements', - branch: master} - - {name: oslo-concurrency, - repository: 'git://git.openstack.org/openstack/oslo.concurrency', - branch: master} - - {name: oslo-config, - repository: 'git://git.openstack.org/openstack/oslo.config', - branch: master} - - {name: oslo-db, - repository: 'git://git.openstack.org/openstack/oslo.db', - branch: master} - - {name: oslo-i18n, - repository: 'git://git.openstack.org/openstack/oslo.i18n', - branch: master} - - {name: oslo-messaging, - repository: 'git://git.openstack.org/openstack/oslo.messaging', - branch: master} - - {name: oslo-serialization, - repository: 'git://git.openstack.org/openstack/oslo.serialization', - branch: master} - - {name: oslo-utils, - repository: 'git://git.openstack.org/openstack/oslo.utils', - branch: master} - - {name: oslo-vmware, - repository: 'git://git.openstack.org/openstack/oslo.vmware', - branch: master} - - {name: osprofiler, - repository: 'git://git.openstack.org/stackforge/osprofiler', - branch: master} - - {name: pbr, - repository: 'git://git.openstack.org/openstack-dev/pbr', - branch: master} - - {name: python-keystoneclient, - repository: 'git://git.openstack.org/openstack/python-keystoneclient', - branch: master} - - {name: python-swiftclient, - repository: 'git://git.openstack.org/openstack/python-swiftclient', - branch: master} - - {name: sqlalchemy-migrate, - repository: 'git://git.openstack.org/stackforge/sqlalchemy-migrate', - branch: master} - - {name: stevedore, - repository: 'git://git.openstack.org/openstack/stevedore', - branch: master} - - {name: wsme, - repository: 'git://git.openstack.org/stackforge/wsme', - branch: master} - - {name: keystonemiddleware, - repository: 'git://git.openstack.org/openstack/keystonemiddleware', - branch: master} - - {name: glance-store, - repository: 'git://git.openstack.org/openstack/glance_store', - branch: master} - - {name: glance, - repository: 'git://git.openstack.org/openstack/glance', - branch: master}" + openstack-origin-git: include-file://glance-master.yaml + + glance-master.yaml + repositories: + - {name: requirements, + repository: 'git://github.com/openstack/requirements', + branch: master} + - {name: oslo-concurrency, + repository: 'git://github.com/openstack/oslo.concurrency', + branch: master} + - {name: oslo-config, + repository: 'git://github.com/openstack/oslo.config', + branch: master} + - {name: oslo-db, + repository: 'git://github.com/openstack/oslo.db', + branch: master} + - {name: oslo-i18n, + repository: 'git://github.com/openstack/oslo.i18n', + branch: master} + - {name: oslo-messaging, + repository: 'git://github.com/openstack/oslo.messaging', + branch: master} + - {name: oslo-serialization, + repository: 'git://github.com/openstack/oslo.serialization', + branch: master} + - {name: oslo-utils, + repository: 'git://github.com/openstack/oslo.utils', + branch: master} + - {name: oslo-vmware, + repository: 'git://github.com/openstack/oslo.vmware', + branch: master} + - {name: osprofiler, + repository: 'git://github.com/stackforge/osprofiler', + branch: master} + - {name: pbr, + repository: 'git://github.com/openstack-dev/pbr', + branch: master} + - {name: python-keystoneclient, + repository: 'git://github.com/openstack/python-keystoneclient', + branch: master} + - {name: python-swiftclient, + repository: 'git://github.com/openstack/python-swiftclient', + branch: master} + - {name: sqlalchemy-migrate, + repository: 'git://github.com/stackforge/sqlalchemy-migrate', + branch: master} + - {name: stevedore, + repository: 'git://github.com/openstack/stevedore', + branch: master} + - {name: wsme, + repository: 'git://github.com/stackforge/wsme', + branch: master} + - {name: keystonemiddleware, + repository: 'git://github.com/openstack/keystonemiddleware', + branch: master} + - {name: glance-store, + repository: 'git://github.com/openstack/glance_store', + branch: master} + - {name: glance, + repository: 'git://github.com/openstack/glance', + branch: master} Contact Information ------------------- From 46eb0846798c3300f4c34972522d60b6868dfe42 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Thu, 30 Apr 2015 19:06:12 +0200 Subject: [PATCH 33/33] [hopem,r=] Ensure glance-api service is restarted when we recieve ack from ceph broker request so that we are sure it is able to connect to newly created ceph resources. Closes-Bug: 1450543 --- hooks/glance_relations.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hooks/glance_relations.py b/hooks/glance_relations.py index 7e9e2755..7f1644c2 100755 --- a/hooks/glance_relations.py +++ b/hooks/glance_relations.py @@ -44,6 +44,7 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.host import ( restart_on_change, service_reload, + service_restart, service_stop, ) from charmhelpers.fetch import ( @@ -265,6 +266,9 @@ def ceph_changed(): (rsp.exit_code, rsp.exit_msg), level=INFO) CONFIGS.write(GLANCE_API_CONF) CONFIGS.write(ceph_config_file()) + # Ensure that glance-api is restarted since only now can we + # guarantee that ceph resources are ready. + service_restart('glance-api') else: rq = CephBrokerRq() replicas = config('ceph-osd-replication-count')