From 599a64229e8cd6561b37bc2fa02025e4ef3d76ed Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 23 Jan 2014 16:40:59 +0000 Subject: [PATCH 1/8] First cut of multi-backend support --- .../contrib/openstack/alternatives.py | 17 +++++ .../charmhelpers/contrib/openstack/context.py | 76 ++++++++++++++----- hooks/charmhelpers/contrib/openstack/utils.py | 20 +++-- .../contrib/storage/linux/utils.py | 2 +- hooks/charmhelpers/core/host.py | 44 +++++++++++ hooks/charmhelpers/fetch/__init__.py | 42 +++++++--- hooks/cinder_hooks.py | 7 ++ hooks/cinder_utils.py | 6 +- hooks/storage-backend-relation-broken | 1 + hooks/storage-backend-relation-changed | 1 + hooks/storage-backend-relation-departed | 1 + hooks/storage-backend-relation-joined | 1 + metadata.yaml | 3 + templates/cinder.conf | 14 ++++ 14 files changed, 197 insertions(+), 38 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/alternatives.py create mode 120000 hooks/storage-backend-relation-broken create mode 120000 hooks/storage-backend-relation-changed create mode 120000 hooks/storage-backend-relation-departed create mode 120000 hooks/storage-backend-relation-joined diff --git a/hooks/charmhelpers/contrib/openstack/alternatives.py b/hooks/charmhelpers/contrib/openstack/alternatives.py new file mode 100644 index 00000000..b413259c --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/alternatives.py @@ -0,0 +1,17 @@ +''' Helper for managing alternatives for file conflict resolution ''' + +import subprocess +import shutil +import os + + +def install_alternative(name, target, source, priority=50): + ''' Install alternative configuration ''' + if (os.path.exists(target) and not os.path.islink(target)): + # Move existing file/directory away before installing + shutil.move(target, '{}.bak'.format(target)) + cmd = [ + 'update-alternatives', '--force', '--install', + target, name, source, str(priority) + ] + subprocess.check_call(cmd) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 8a982ffa..6331a92e 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -23,7 +23,6 @@ from charmhelpers.core.hookenv import ( unit_get, unit_private_ip, ERROR, - WARNING, ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -182,10 +181,12 @@ class AMQPContext(OSContextGenerator): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - ctxt['rabbitmq_hosts'] = [] - for unit in related_units(rid): - ctxt['rabbitmq_hosts'].append(relation_get('private-address', - rid=rid, unit=unit)) + if 'clustered' not in ctxt and len(related_units(rid)) > 1: + rabbitmq_hosts = [] + for unit in related_units(rid): + rabbitmq_hosts.append(relation_get('private-address', + rid=rid, unit=unit)) + ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts) if not context_complete(ctxt): return {} else: @@ -286,6 +287,7 @@ class ImageServiceContext(OSContextGenerator): class ApacheSSLContext(OSContextGenerator): + """ Generates a context for an apache vhost configuration that configures HTTPS reverse proxying for one or many endpoints. Generated context @@ -433,28 +435,61 @@ class NeutronContext(object): class OSConfigFlagContext(OSContextGenerator): - ''' - Responsible adding user-defined config-flags in charm config to a - to a template context. - ''' + + """ + Responsible for adding user-defined config-flags in charm config to a + template context. + + NOTE: the value of config-flags may be a comma-separated list of + key=value pairs and some Openstack config files support + comma-separated lists as values. + """ + def __call__(self): config_flags = config('config-flags') - if not config_flags or config_flags in ['None', '']: + if not config_flags: return {} - config_flags = config_flags.split(',') + + if config_flags.find('==') >= 0: + log("config_flags is not in expected format (key=value)", + level=ERROR) + raise OSContextError + + # strip the following from each value. + post_strippers = ' ,' + # we strip any leading/trailing '=' or ' ' from the string then + # split on '='. + split = config_flags.strip(' =').split('=') + limit = len(split) flags = {} - for flag in config_flags: - if '=' not in flag: - log('Improperly formatted config-flag, expected k=v ' - 'got %s' % flag, level=WARNING) - continue - k, v = flag.split('=') - flags[k.strip()] = v - ctxt = {'user_config_flags': flags} - return ctxt + for i in xrange(0, limit - 1): + current = split[i] + next = split[i + 1] + vindex = next.rfind(',') + if (i == limit - 2) or (vindex < 0): + value = next + else: + value = next[:vindex] + + if i == 0: + key = current + else: + # if this not the first entry, expect an embedded key. + index = current.rfind(',') + if index < 0: + log("invalid config value(s) at index %s" % (i), + level=ERROR) + raise OSContextError + key = current[index + 1:] + + # Add to collection. + flags[key.strip(post_strippers)] = value.rstrip(post_strippers) + + return {'user_config_flags': flags} class SubordinateConfigContext(OSContextGenerator): + """ Responsible for inspecting relations to subordinates that may be exporting required config via a json blob. @@ -495,6 +530,7 @@ class SubordinateConfigContext(OSContextGenerator): } """ + def __init__(self, service, config_file, interface): """ :param service : Service name key to query in any subordinate diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index d66afd74..56d04245 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -41,6 +41,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('quantal', 'folsom'), ('raring', 'grizzly'), ('saucy', 'havana'), + ('trusty', 'icehouse') ]) @@ -201,7 +202,7 @@ def os_release(package, base='essex'): def import_key(keyid): - cmd = "apt-key adv --keyserver keyserver.ubuntu.com " \ + cmd = "apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 " \ "--recv-keys %s" % keyid try: subprocess.check_call(cmd.split(' ')) @@ -260,6 +261,9 @@ def configure_installation_source(rel): 'havana': 'precise-updates/havana', 'havana/updates': 'precise-updates/havana', 'havana/proposed': 'precise-proposed/havana', + 'icehouse': 'precise-updates/icehouse', + 'icehouse/updates': 'precise-updates/icehouse', + 'icehouse/proposed': 'precise-proposed/icehouse', } try: @@ -411,7 +415,7 @@ def get_host_ip(hostname): return ns_query(hostname) -def get_hostname(address): +def get_hostname(address, fqdn=True): """ Resolves hostname for given IP, or returns the input if it is already a hostname. @@ -430,7 +434,11 @@ def get_hostname(address): if not result: return None - # strip trailing . - if result.endswith('.'): - return result[:-1] - return result + if fqdn: + # strip trailing . + if result.endswith('.'): + return result[:-1] + else: + return result + else: + return result.split('.')[0] diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index 5b9b6d47..c40218f0 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -22,4 +22,4 @@ def zap_disk(block_device): :param block_device: str: Full path of block device to clean. ''' - check_call(['sgdisk', '--zap-all', block_device]) + check_call(['sgdisk', '--zap-all', '--mbrtogpt', block_device]) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 4a6a4a8c..c8c81b28 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -245,3 +245,47 @@ def pwgen(length=None): random_chars = [ random.choice(alphanumeric_chars) for _ in range(length)] return(''.join(random_chars)) + + +def list_nics(nic_type): + '''Return a list of nics of given type(s)''' + if isinstance(nic_type, basestring): + int_types = [nic_type] + else: + int_types = nic_type + interfaces = [] + for int_type in int_types: + cmd = ['ip', 'addr', 'show', 'label', int_type + '*'] + ip_output = subprocess.check_output(cmd).split('\n') + ip_output = (line for line in ip_output if line) + for line in ip_output: + if line.split()[1].startswith(int_type): + interfaces.append(line.split()[1].replace(":", "")) + return interfaces + + +def set_nic_mtu(nic, mtu): + '''Set MTU on a network interface''' + cmd = ['ip', 'link', 'set', nic, 'mtu', mtu] + subprocess.check_call(cmd) + + +def get_nic_mtu(nic): + cmd = ['ip', 'addr', 'show', nic] + ip_output = subprocess.check_output(cmd).split('\n') + mtu = "" + for line in ip_output: + words = line.split() + if 'mtu' in words: + mtu = words[words.index("mtu") + 1] + return mtu + + +def get_nic_hwaddr(nic): + cmd = ['ip', '-o', '-0', 'addr', 'show', nic] + ip_output = subprocess.check_output(cmd) + hwaddr = "" + words = ip_output.split() + if 'link/ether' in words: + hwaddr = words[words.index('link/ether') + 1] + return hwaddr diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index fa0172a9..1f4f6315 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -13,6 +13,7 @@ from charmhelpers.core.hookenv import ( log, ) import apt_pkg +import os CLOUD_ARCHIVE = """# Ubuntu Cloud Archive deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main @@ -43,8 +44,16 @@ CLOUD_ARCHIVE_POCKETS = { 'precise-havana/updates': 'precise-updates/havana', 'precise-updates/havana': 'precise-updates/havana', 'havana/proposed': 'precise-proposed/havana', - 'precies-havana/proposed': 'precise-proposed/havana', + 'precise-havana/proposed': 'precise-proposed/havana', 'precise-proposed/havana': 'precise-proposed/havana', + # Icehouse + 'icehouse': 'precise-updates/icehouse', + 'precise-icehouse': 'precise-updates/icehouse', + 'precise-icehouse/updates': 'precise-updates/icehouse', + 'precise-updates/icehouse': 'precise-updates/icehouse', + 'icehouse/proposed': 'precise-proposed/icehouse', + 'precise-icehouse/proposed': 'precise-proposed/icehouse', + 'precise-proposed/icehouse': 'precise-proposed/icehouse', } @@ -66,8 +75,10 @@ def filter_installed_packages(packages): def apt_install(packages, options=None, fatal=False): """Install one or more packages""" - options = options or [] - cmd = ['apt-get', '-y'] + if options is None: + options = ['--option=Dpkg::Options::=--force-confold'] + + cmd = ['apt-get', '--assume-yes'] cmd.extend(options) cmd.append('install') if isinstance(packages, basestring): @@ -76,10 +87,14 @@ def apt_install(packages, options=None, fatal=False): cmd.extend(packages) log("Installing {} with options: {}".format(packages, options)) + env = os.environ.copy() + if 'DEBIAN_FRONTEND' not in env: + env['DEBIAN_FRONTEND'] = 'noninteractive' + if fatal: - subprocess.check_call(cmd) + subprocess.check_call(cmd, env=env) else: - subprocess.call(cmd) + subprocess.call(cmd, env=env) def apt_update(fatal=False): @@ -93,7 +108,7 @@ def apt_update(fatal=False): def apt_purge(packages, fatal=False): """Purge one or more packages""" - cmd = ['apt-get', '-y', 'purge'] + cmd = ['apt-get', '--assume-yes', 'purge'] if isinstance(packages, basestring): cmd.append(packages) else: @@ -123,14 +138,16 @@ def add_source(source, key=None): if (source.startswith('ppa:') or source.startswith('http:') or source.startswith('deb ') or - source.startswith('cloud-archive:')): + source.startswith('cloud-archive:')): subprocess.check_call(['add-apt-repository', '--yes', source]) elif source.startswith('cloud:'): apt_install(filter_installed_packages(['ubuntu-cloud-keyring']), fatal=True) pocket = source.split(':')[-1] if pocket not in CLOUD_ARCHIVE_POCKETS: - raise SourceConfigError('Unsupported cloud: source option %s' % pocket) + raise SourceConfigError( + 'Unsupported cloud: source option %s' % + pocket) actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket] with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt: apt.write(CLOUD_ARCHIVE.format(actual_pocket)) @@ -220,7 +237,9 @@ def install_from_config(config_var_name): class BaseFetchHandler(object): + """Base class for FetchHandler implementations in fetch plugins""" + def can_handle(self, source): """Returns True if the source can be handled. Otherwise returns a string explaining why it cannot""" @@ -248,10 +267,13 @@ def plugins(fetch_handlers=None): for handler_name in fetch_handlers: package, classname = handler_name.rsplit('.', 1) try: - handler_class = getattr(importlib.import_module(package), classname) + handler_class = getattr( + importlib.import_module(package), + classname) plugin_list.append(handler_class()) except (ImportError, AttributeError): # Skip missing plugins so that they can be ommitted from # installation if desired - log("FetchHandler {} not found, skipping plugin".format(handler_name)) + log("FetchHandler {} not found, skipping plugin".format( + handler_name)) return plugin_list diff --git a/hooks/cinder_hooks.py b/hooks/cinder_hooks.py index 1ceb504a..178f7b4b 100755 --- a/hooks/cinder_hooks.py +++ b/hooks/cinder_hooks.py @@ -272,6 +272,13 @@ def upgrade_charm(): amqp_joined(relation_id=rel_id) +@hooks.hook('storage-backend-relation-changed') +@hooks.hook('storage-backend-relation-broken') +@restart_on_change(restart_map()) +def storage_backend(): + CONFIGS.write(CINDER_CONF) + + if __name__ == '__main__': try: hooks.execute(sys.argv) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index e9bdf15a..631641f7 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -103,7 +103,11 @@ CONFIG_FILES = OrderedDict([ context.OSConfigFlagContext(), cinder_contexts.CephContext(), cinder_contexts.HAProxyContext(), - cinder_contexts.ImageServiceContext()], + cinder_contexts.ImageServiceContext(), + context.SubordinateConfigContext( + interface='storage-backend', + service='cinder', + config_file=CINDER_CONF)], 'services': ['cinder-api', 'cinder-volume', 'cinder-scheduler', 'haproxy'] }), diff --git a/hooks/storage-backend-relation-broken b/hooks/storage-backend-relation-broken new file mode 120000 index 00000000..6dcd0084 --- /dev/null +++ b/hooks/storage-backend-relation-broken @@ -0,0 +1 @@ +cinder_hooks.py \ No newline at end of file diff --git a/hooks/storage-backend-relation-changed b/hooks/storage-backend-relation-changed new file mode 120000 index 00000000..6dcd0084 --- /dev/null +++ b/hooks/storage-backend-relation-changed @@ -0,0 +1 @@ +cinder_hooks.py \ No newline at end of file diff --git a/hooks/storage-backend-relation-departed b/hooks/storage-backend-relation-departed new file mode 120000 index 00000000..6dcd0084 --- /dev/null +++ b/hooks/storage-backend-relation-departed @@ -0,0 +1 @@ +cinder_hooks.py \ No newline at end of file diff --git a/hooks/storage-backend-relation-joined b/hooks/storage-backend-relation-joined new file mode 120000 index 00000000..6dcd0084 --- /dev/null +++ b/hooks/storage-backend-relation-joined @@ -0,0 +1 @@ +cinder_hooks.py \ No newline at end of file diff --git a/metadata.yaml b/metadata.yaml index 132f0a15..74ee664f 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -22,6 +22,9 @@ requires: ha: interface: hacluster scope: container + storage-backend: + interface: cinder-backend + scope: container peers: cluster: interface: cinder-ha diff --git a/templates/cinder.conf b/templates/cinder.conf index c62f1b34..974e4435 100644 --- a/templates/cinder.conf +++ b/templates/cinder.conf @@ -49,3 +49,17 @@ glance_api_version = {{ glance_api_version }} {% endfor -%} {% endif -%} +{% if sections and 'DEFAULT' in sections -%} +{% for key, value in sections['DEFAULT'] -%} +{{ key }} = {{ value }} +{% endfor -%} +{% endif -%} + +{% for section in sections -%} +{% if section != 'DEFAULT' -%} +[{{ section }}] +{% for key, value in sections[section] -%} +{{ key }} = {{ value }} +{% endfor -%} +{% endif -%} +{% endfor -%} From 259dd6f50d345acb08c7e52803dec5639f6fad8c Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 24 Jan 2014 14:22:28 +0000 Subject: [PATCH 2/8] Temp fix for charm-helpers to support multiple related storage-backends --- hooks/charmhelpers/contrib/openstack/context.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 6331a92e..99743989 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -569,7 +569,10 @@ class SubordinateConfigContext(OSContextGenerator): sub_config = sub_config[self.config_file] for k, v in sub_config.iteritems(): - ctxt[k] = v + if k in ctxt: + ctxt[k].update(v) + else: + ctxt[k] = v if not ctxt: ctxt['sections'] = {} From aaf5f48a044abe009070c258a30b3527cddd9cae Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 24 Jan 2014 14:30:23 +0000 Subject: [PATCH 3/8] Context + handling for backend names --- hooks/cinder_contexts.py | 19 +++++++++++++++++++ hooks/cinder_utils.py | 3 ++- templates/cinder.conf | 4 ++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/hooks/cinder_contexts.py b/hooks/cinder_contexts.py index b5c89b04..be9e990e 100644 --- a/hooks/cinder_contexts.py +++ b/hooks/cinder_contexts.py @@ -2,6 +2,8 @@ from charmhelpers.core.hookenv import ( config, relation_ids, service_name, + related_units, + relation_get, ) from charmhelpers.contrib.openstack.context import ( @@ -75,3 +77,20 @@ class ApacheSSLContext(SSLContext): if not service_enabled('cinder-api'): return {} return super(ApacheSSLContext, self).__call__() + + +class StorageBackendContext(OSContextGenerator): + interfaces = ['storage-backend'] + + def __call__(self): + backends = [] + for rid in relation_ids('storage-backend'): + for unit in related_units(rid): + backend_name = relation_get('backend_name', + unit, rid) + if backend_name: + backends.append(backend_name) + if len(backends) > 0: + return {'backends': ",".join(backends)} + else: + return {} diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index 631641f7..68188c25 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -107,7 +107,8 @@ CONFIG_FILES = OrderedDict([ context.SubordinateConfigContext( interface='storage-backend', service='cinder', - config_file=CINDER_CONF)], + config_file=CINDER_CONF), + cinder_contexts.StorageBackendContext()], 'services': ['cinder-api', 'cinder-volume', 'cinder-scheduler', 'haproxy'] }), diff --git a/templates/cinder.conf b/templates/cinder.conf index 974e4435..90223318 100644 --- a/templates/cinder.conf +++ b/templates/cinder.conf @@ -55,6 +55,10 @@ glance_api_version = {{ glance_api_version }} {% endfor -%} {% endif -%} +{% if backends -%} +enabled_backends= {{ backends }} +{% endif -%} + {% for section in sections -%} {% if section != 'DEFAULT' -%} [{{ section }}] From 340c492654886265b8d406db3430ddfafb11cf30 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 11 Feb 2014 15:23:22 +0000 Subject: [PATCH 4/8] Tidy template --- templates/cinder.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/cinder.conf b/templates/cinder.conf index 90223318..6c6a6baa 100644 --- a/templates/cinder.conf +++ b/templates/cinder.conf @@ -56,7 +56,7 @@ glance_api_version = {{ glance_api_version }} {% endif -%} {% if backends -%} -enabled_backends= {{ backends }} +enabled_backends = {{ backends }} {% endif -%} {% for section in sections -%} From b91246818a3376d9d7e6ba054e558d827713a3b6 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 11 Feb 2014 15:26:15 +0000 Subject: [PATCH 5/8] Autopep8 --- hooks/cinder_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index 68188c25..dda82f5b 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -105,9 +105,9 @@ CONFIG_FILES = OrderedDict([ cinder_contexts.HAProxyContext(), cinder_contexts.ImageServiceContext(), context.SubordinateConfigContext( - interface='storage-backend', - service='cinder', - config_file=CINDER_CONF), + interface='storage-backend', + service='cinder', + config_file=CINDER_CONF), cinder_contexts.StorageBackendContext()], 'services': ['cinder-api', 'cinder-volume', 'cinder-scheduler', 'haproxy'] From 8b720acf753637db678b1f96db56eef1ed7a6345 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 12 Feb 2014 09:32:56 +0000 Subject: [PATCH 6/8] Add unit tests for multibackend naming --- unit_tests/test_cinder_contexts.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/unit_tests/test_cinder_contexts.py b/unit_tests/test_cinder_contexts.py index b5a6ca7d..c1f678aa 100644 --- a/unit_tests/test_cinder_contexts.py +++ b/unit_tests/test_cinder_contexts.py @@ -12,6 +12,8 @@ TO_PATCH = [ 'service_name', 'determine_haproxy_port', 'determine_api_port', + 'related_units', + 'relation_get' ] @@ -70,3 +72,22 @@ class TestCinderContext(CharmTestCase): service_enabled.return_value = True https.return_value = False self.assertEquals(contexts.ApacheSSLContext()(), {}) + + def test_storage_backend_no_backends(self): + self.relation_ids.return_value = [] + self.assertEquals(contexts.StorageBackendContext()(), {}) + + def test_storage_backend_single_backend(self): + self.relation_ids.return_value = ['cinder-ceph:0'] + self.related_units.return_value = ['cinder-ceph/0'] + self.relation_get.return_value = 'cinder-ceph' + self.assertEquals(contexts.StorageBackendContext()(), + {'backends': 'cinder-ceph'}) + + def test_storage_backend_multi_backend(self): + self.relation_ids.return_value = ['cinder-ceph:0', 'cinder-vmware:0'] + self.related_units.side_effect = [['cinder-ceph/0'], + ['cinder-vmware/0']] + self.relation_get.side_effect = ['cinder-ceph', 'cinder-vmware'] + self.assertEquals(contexts.StorageBackendContext()(), + {'backends': 'cinder-ceph,cinder-vmware'}) From 302ced7aa49be5c9fb2afa94b4e10d9940431a0f Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 12 Feb 2014 09:39:02 +0000 Subject: [PATCH 7/8] Add unit tests for storage backend hooks --- unit_tests/test_cinder_hooks.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/unit_tests/test_cinder_hooks.py b/unit_tests/test_cinder_hooks.py index 4f5602a2..9e3977fc 100644 --- a/unit_tests/test_cinder_hooks.py +++ b/unit_tests/test_cinder_hooks.py @@ -229,6 +229,14 @@ class TestChangedHooks(CharmTestCase): hooks.hooks.execute(['hooks/image-service-relation-broken']) self.assertTrue(self.CONFIGS.write_all.called) + def test_storage_backend_changed(self): + hooks.hooks.execute(['hooks/storage-backend-relation-changed']) + self.CONFIGS.write.assert_called_with(utils.CINDER_CONF) + + def test_storage_backend_broken(self): + hooks.hooks.execute(['hooks/storage-backend-relation-broken']) + self.CONFIGS.write.assert_called_with(utils.CINDER_CONF) + class TestJoinedHooks(CharmTestCase): def setUp(self): From 5ab469c14cb763cdba3451db343d67438d091098 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 19 Feb 2014 16:03:59 +0000 Subject: [PATCH 8/8] Resync helpers --- .../charmhelpers/contrib/hahelpers/cluster.py | 8 +- .../charmhelpers/contrib/openstack/context.py | 111 ++++++++++-------- .../contrib/openstack/templates/haproxy.cfg | 5 +- hooks/charmhelpers/core/hookenv.py | 6 + hooks/charmhelpers/core/host.py | 12 +- hooks/charmhelpers/fetch/__init__.py | 6 +- 6 files changed, 88 insertions(+), 60 deletions(-) diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 074855f4..bf832f7d 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -126,17 +126,17 @@ def determine_api_port(public_port): return public_port - (i * 10) -def determine_haproxy_port(public_port): +def determine_apache_port(public_port): ''' - Description: Determine correct proxy listening port based on public IP + - existence of HTTPS reverse proxy. + Description: Determine correct apache listening port based on public IP + + state of the cluster. public_port: int: standard public port for given service returns: int: the correct listening port for the HAProxy service ''' i = 0 - if https(): + if len(peer_units()) > 0 or is_clustered(): i += 1 return public_port - (i * 10) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 99743989..7cc94ed0 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -26,11 +26,9 @@ from charmhelpers.core.hookenv import ( ) from charmhelpers.contrib.hahelpers.cluster import ( + determine_apache_port, determine_api_port, - determine_haproxy_port, https, - is_clustered, - peer_units, ) from charmhelpers.contrib.hahelpers.apache import ( @@ -67,6 +65,43 @@ def context_complete(ctxt): return True +def config_flags_parser(config_flags): + if config_flags.find('==') >= 0: + log("config_flags is not in expected format (key=value)", + level=ERROR) + raise OSContextError + # strip the following from each value. + post_strippers = ' ,' + # we strip any leading/trailing '=' or ' ' from the string then + # split on '='. + split = config_flags.strip(' =').split('=') + limit = len(split) + flags = {} + for i in xrange(0, limit - 1): + current = split[i] + next = split[i + 1] + vindex = next.rfind(',') + if (i == limit - 2) or (vindex < 0): + value = next + else: + value = next[:vindex] + + if i == 0: + key = current + else: + # if this not the first entry, expect an embedded key. + index = current.rfind(',') + if index < 0: + log("invalid config value(s) at index %s" % (i), + level=ERROR) + raise OSContextError + key = current[index + 1:] + + # Add to collection. + flags[key.strip(post_strippers)] = value.rstrip(post_strippers) + return flags + + class OSContextGenerator(object): interfaces = [] @@ -181,7 +216,12 @@ class AMQPContext(OSContextGenerator): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - if 'clustered' not in ctxt and len(related_units(rid)) > 1: + if ('clustered' not in ctxt or relation_get('ha-vip-only') == 'True') and \ + len(related_units(rid)) > 1: + if relation_get('ha_queues'): + ctxt['rabbitmq_ha_queues'] = relation_get('ha_queues') + else: + ctxt['rabbitmq_ha_queues'] = False rabbitmq_hosts = [] for unit in related_units(rid): rabbitmq_hosts.append(relation_get('private-address', @@ -343,11 +383,9 @@ class ApacheSSLContext(OSContextGenerator): 'private_address': unit_get('private-address'), 'endpoints': [] } - for ext_port in self.external_ports: - if peer_units() or is_clustered(): - int_port = determine_haproxy_port(ext_port) - else: - int_port = determine_api_port(ext_port) + for api_port in self.external_ports: + ext_port = determine_apache_port(api_port) + int_port = determine_api_port(api_port) portmap = (int(ext_port), int(int_port)) ctxt['endpoints'].append(portmap) return ctxt @@ -430,6 +468,11 @@ class NeutronContext(object): elif self.plugin == 'nvp': ctxt.update(self.nvp_ctxt()) + alchemy_flags = config('neutron-alchemy-flags') + if alchemy_flags: + flags = config_flags_parser(alchemy_flags) + ctxt['neutron_alchemy_flags'] = flags + self._save_flag_file() return ctxt @@ -450,41 +493,7 @@ class OSConfigFlagContext(OSContextGenerator): if not config_flags: return {} - if config_flags.find('==') >= 0: - log("config_flags is not in expected format (key=value)", - level=ERROR) - raise OSContextError - - # strip the following from each value. - post_strippers = ' ,' - # we strip any leading/trailing '=' or ' ' from the string then - # split on '='. - split = config_flags.strip(' =').split('=') - limit = len(split) - flags = {} - for i in xrange(0, limit - 1): - current = split[i] - next = split[i + 1] - vindex = next.rfind(',') - if (i == limit - 2) or (vindex < 0): - value = next - else: - value = next[:vindex] - - if i == 0: - key = current - else: - # if this not the first entry, expect an embedded key. - index = current.rfind(',') - if index < 0: - log("invalid config value(s) at index %s" % (i), - level=ERROR) - raise OSContextError - key = current[index + 1:] - - # Add to collection. - flags[key.strip(post_strippers)] = value.rstrip(post_strippers) - + flags = config_flags_parser(config_flags) return {'user_config_flags': flags} @@ -569,12 +578,18 @@ class SubordinateConfigContext(OSContextGenerator): sub_config = sub_config[self.config_file] for k, v in sub_config.iteritems(): - if k in ctxt: - ctxt[k].update(v) - else: - ctxt[k] = v + ctxt[k] = v if not ctxt: ctxt['sections'] = {} return ctxt + + +class SyslogContext(OSContextGenerator): + + def __call__(self): + ctxt = { + 'use_syslog': config('use-syslog') + } + return ctxt diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index a1694e44..56ed913e 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -8,8 +8,8 @@ global defaults log global - mode http - option httplog + mode tcp + option tcplog option dontlognull retries 3 timeout queue 1000 @@ -29,7 +29,6 @@ listen stats :8888 {% for service, ports in service_ports.iteritems() -%} listen {{ service }} 0.0.0.0:{{ ports[0] }} balance roundrobin - option tcplog {% for unit, address in units.iteritems() -%} server {{ unit }} {{ address }}:{{ ports[1] }} check {% endfor %} diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index bb196dfa..505c202d 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -8,6 +8,7 @@ import os import json import yaml import subprocess +import sys import UserDict from subprocess import CalledProcessError @@ -149,6 +150,11 @@ def service_name(): return local_unit().split('/')[0] +def hook_name(): + """The name of the currently executing hook""" + return os.path.basename(sys.argv[0]) + + @cached def config(scope=None): """Juju charm configuration""" diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index c8c81b28..cfd26847 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -194,7 +194,7 @@ def file_hash(path): return None -def restart_on_change(restart_map): +def restart_on_change(restart_map, stopstart=False): """Restart services based on configuration files changing This function is used a decorator, for example @@ -219,8 +219,14 @@ def restart_on_change(restart_map): for path in restart_map: if checksums[path] != file_hash(path): restarts += restart_map[path] - for service_name in list(OrderedDict.fromkeys(restarts)): - service('restart', service_name) + services_list = list(OrderedDict.fromkeys(restarts)) + if not stopstart: + for service_name in services_list: + service('restart', service_name) + else: + for action in ['stop', 'start']: + for service_name in services_list: + service(action, service_name) return wrapped_f return wrap diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 1f4f6315..07bb707d 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -136,7 +136,7 @@ def apt_hold(packages, fatal=False): def add_source(source, key=None): if (source.startswith('ppa:') or - source.startswith('http:') or + source.startswith('http') or source.startswith('deb ') or source.startswith('cloud-archive:')): subprocess.check_call(['add-apt-repository', '--yes', source]) @@ -156,7 +156,9 @@ def add_source(source, key=None): with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt: apt.write(PROPOSED_POCKET.format(release)) if key: - subprocess.check_call(['apt-key', 'import', key]) + subprocess.check_call(['apt-key', 'adv', '--keyserver', + 'keyserver.ubuntu.com', '--recv', + key]) class SourceConfigError(Exception):