From 1506413a71d4ac0d8f9685c1fa52d1f9555aa47c Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 24 Feb 2014 17:47:55 +0000 Subject: [PATCH] Resync helpers --- .../charmhelpers/contrib/hahelpers/cluster.py | 8 +- .../charmhelpers/contrib/openstack/context.py | 114 +++++++++++++----- hooks/charmhelpers/contrib/openstack/utils.py | 14 ++- .../contrib/storage/linux/ceph.py | 8 +- hooks/charmhelpers/core/hookenv.py | 6 + hooks/charmhelpers/core/host.py | 12 +- hooks/charmhelpers/fetch/__init__.py | 6 +- hooks/quantum_contexts.py | 6 +- unit_tests/test_quantum_contexts.py | 9 ++ unit_tests/test_quantum_utils.py | 6 +- unit_tests/test_utils.py | 3 + 11 files changed, 145 insertions(+), 47 deletions(-) diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 074855f4..bf832f7d 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -126,17 +126,17 @@ def determine_api_port(public_port): return public_port - (i * 10) -def determine_haproxy_port(public_port): +def determine_apache_port(public_port): ''' - Description: Determine correct proxy listening port based on public IP + - existence of HTTPS reverse proxy. + Description: Determine correct apache listening port based on public IP + + state of the cluster. public_port: int: standard public port for given service returns: int: the correct listening port for the HAProxy service ''' i = 0 - if https(): + if len(peer_units()) > 0 or is_clustered(): i += 1 return public_port - (i * 10) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 8a982ffa..81ac12b6 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -23,15 +23,12 @@ from charmhelpers.core.hookenv import ( unit_get, unit_private_ip, ERROR, - WARNING, ) from charmhelpers.contrib.hahelpers.cluster import ( + determine_apache_port, determine_api_port, - determine_haproxy_port, https, - is_clustered, - peer_units, ) from charmhelpers.contrib.hahelpers.apache import ( @@ -68,6 +65,43 @@ def context_complete(ctxt): return True +def config_flags_parser(config_flags): + if config_flags.find('==') >= 0: + log("config_flags is not in expected format (key=value)", + level=ERROR) + raise OSContextError + # strip the following from each value. + post_strippers = ' ,' + # we strip any leading/trailing '=' or ' ' from the string then + # split on '='. + split = config_flags.strip(' =').split('=') + limit = len(split) + flags = {} + for i in xrange(0, limit - 1): + current = split[i] + next = split[i + 1] + vindex = next.rfind(',') + if (i == limit - 2) or (vindex < 0): + value = next + else: + value = next[:vindex] + + if i == 0: + key = current + else: + # if this not the first entry, expect an embedded key. + index = current.rfind(',') + if index < 0: + log("invalid config value(s) at index %s" % (i), + level=ERROR) + raise OSContextError + key = current[index + 1:] + + # Add to collection. + flags[key.strip(post_strippers)] = value.rstrip(post_strippers) + return flags + + class OSContextGenerator(object): interfaces = [] @@ -182,10 +216,17 @@ class AMQPContext(OSContextGenerator): # Sufficient information found = break out! break # Used for active/active rabbitmq >= grizzly - ctxt['rabbitmq_hosts'] = [] - for unit in related_units(rid): - ctxt['rabbitmq_hosts'].append(relation_get('private-address', - rid=rid, unit=unit)) + if ('clustered' not in ctxt or relation_get('ha-vip-only') == 'True') and \ + len(related_units(rid)) > 1: + if relation_get('ha_queues'): + ctxt['rabbitmq_ha_queues'] = relation_get('ha_queues') + else: + ctxt['rabbitmq_ha_queues'] = False + rabbitmq_hosts = [] + for unit in related_units(rid): + rabbitmq_hosts.append(relation_get('private-address', + rid=rid, unit=unit)) + ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts) if not context_complete(ctxt): return {} else: @@ -209,11 +250,13 @@ class CephContext(OSContextGenerator): unit=unit)) auth = relation_get('auth', rid=rid, unit=unit) key = relation_get('key', rid=rid, unit=unit) + use_syslog = str(config('use-syslog')).lower() ctxt = { 'mon_hosts': ' '.join(mon_hosts), 'auth': auth, 'key': key, + 'use_syslog': use_syslog } if not os.path.isdir('/etc/ceph'): @@ -286,6 +329,7 @@ class ImageServiceContext(OSContextGenerator): class ApacheSSLContext(OSContextGenerator): + """ Generates a context for an apache vhost configuration that configures HTTPS reverse proxying for one or many endpoints. Generated context @@ -341,11 +385,9 @@ class ApacheSSLContext(OSContextGenerator): 'private_address': unit_get('private-address'), 'endpoints': [] } - for ext_port in self.external_ports: - if peer_units() or is_clustered(): - int_port = determine_haproxy_port(ext_port) - else: - int_port = determine_api_port(ext_port) + for api_port in self.external_ports: + ext_port = determine_apache_port(api_port) + int_port = determine_api_port(api_port) portmap = (int(ext_port), int(int_port)) ctxt['endpoints'].append(portmap) return ctxt @@ -428,33 +470,37 @@ class NeutronContext(object): elif self.plugin == 'nvp': ctxt.update(self.nvp_ctxt()) + alchemy_flags = config('neutron-alchemy-flags') + if alchemy_flags: + flags = config_flags_parser(alchemy_flags) + ctxt['neutron_alchemy_flags'] = flags + self._save_flag_file() return ctxt class OSConfigFlagContext(OSContextGenerator): - ''' - Responsible adding user-defined config-flags in charm config to a - to a template context. - ''' + + """ + Responsible for adding user-defined config-flags in charm config to a + template context. + + NOTE: the value of config-flags may be a comma-separated list of + key=value pairs and some Openstack config files support + comma-separated lists as values. + """ + def __call__(self): config_flags = config('config-flags') - if not config_flags or config_flags in ['None', '']: + if not config_flags: return {} - config_flags = config_flags.split(',') - flags = {} - for flag in config_flags: - if '=' not in flag: - log('Improperly formatted config-flag, expected k=v ' - 'got %s' % flag, level=WARNING) - continue - k, v = flag.split('=') - flags[k.strip()] = v - ctxt = {'user_config_flags': flags} - return ctxt + + flags = config_flags_parser(config_flags) + return {'user_config_flags': flags} class SubordinateConfigContext(OSContextGenerator): + """ Responsible for inspecting relations to subordinates that may be exporting required config via a json blob. @@ -495,6 +541,7 @@ class SubordinateConfigContext(OSContextGenerator): } """ + def __init__(self, service, config_file, interface): """ :param service : Service name key to query in any subordinate @@ -539,3 +586,12 @@ class SubordinateConfigContext(OSContextGenerator): ctxt['sections'] = {} return ctxt + + +class SyslogContext(OSContextGenerator): + + def __call__(self): + ctxt = { + 'use_syslog': config('use-syslog') + } + return ctxt diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index b6e491cf..d0d0998a 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -416,7 +416,7 @@ def get_host_ip(hostname): return ns_query(hostname) -def get_hostname(address): +def get_hostname(address, fqdn=True): """ Resolves hostname for given IP, or returns the input if it is already a hostname. @@ -435,7 +435,11 @@ def get_hostname(address): if not result: return None - # strip trailing . - if result.endswith('.'): - return result[:-1] - return result + if fqdn: + # strip trailing . + if result.endswith('.'): + return result[:-1] + else: + return result + else: + return result.split('.')[0] diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 69b879ca..12417410 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -49,6 +49,9 @@ CEPH_CONF = """[global] auth supported = {auth} keyring = {keyring} mon host = {mon_hosts} + log to syslog = {use_syslog} + err to syslog = {use_syslog} + clog to syslog = {use_syslog} """ @@ -194,7 +197,7 @@ def get_ceph_nodes(): return hosts -def configure(service, key, auth): +def configure(service, key, auth, use_syslog): ''' Perform basic configuration of Ceph ''' create_keyring(service, key) create_key_file(service, key) @@ -202,7 +205,8 @@ def configure(service, key, auth): with open('/etc/ceph/ceph.conf', 'w') as ceph_conf: ceph_conf.write(CEPH_CONF.format(auth=auth, keyring=_keyring_path(service), - mon_hosts=",".join(map(str, hosts)))) + mon_hosts=",".join(map(str, hosts)), + use_syslog=use_syslog)) modprobe('rbd') diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index bb196dfa..505c202d 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -8,6 +8,7 @@ import os import json import yaml import subprocess +import sys import UserDict from subprocess import CalledProcessError @@ -149,6 +150,11 @@ def service_name(): return local_unit().split('/')[0] +def hook_name(): + """The name of the currently executing hook""" + return os.path.basename(sys.argv[0]) + + @cached def config(scope=None): """Juju charm configuration""" diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index c8c81b28..cfd26847 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -194,7 +194,7 @@ def file_hash(path): return None -def restart_on_change(restart_map): +def restart_on_change(restart_map, stopstart=False): """Restart services based on configuration files changing This function is used a decorator, for example @@ -219,8 +219,14 @@ def restart_on_change(restart_map): for path in restart_map: if checksums[path] != file_hash(path): restarts += restart_map[path] - for service_name in list(OrderedDict.fromkeys(restarts)): - service('restart', service_name) + services_list = list(OrderedDict.fromkeys(restarts)) + if not stopstart: + for service_name in services_list: + service('restart', service_name) + else: + for action in ['stop', 'start']: + for service_name in services_list: + service(action, service_name) return wrapped_f return wrap diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 1f4f6315..07bb707d 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -136,7 +136,7 @@ def apt_hold(packages, fatal=False): def add_source(source, key=None): if (source.startswith('ppa:') or - source.startswith('http:') or + source.startswith('http') or source.startswith('deb ') or source.startswith('cloud-archive:')): subprocess.check_call(['add-apt-repository', '--yes', source]) @@ -156,7 +156,9 @@ def add_source(source, key=None): with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt: apt.write(PROPOSED_POCKET.format(release)) if key: - subprocess.check_call(['apt-key', 'import', key]) + subprocess.check_call(['apt-key', 'adv', '--keyserver', + 'keyserver.ubuntu.com', '--recv', + key]) class SourceConfigError(Exception): diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index 408bd848..554ead9a 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -66,7 +66,8 @@ CORE_PLUGIN = { def core_plugin(): - if (get_os_codename_install_source(config('openstack-origin')) >= 'icehouse' + if (get_os_codename_install_source(config('openstack-origin')) + >= 'icehouse' and config('plugin') == OVS): return NEUTRON_ML2_PLUGIN else: @@ -109,6 +110,7 @@ class NetworkServiceContext(OSContextGenerator): class L3AgentContext(OSContextGenerator): + def __call__(self): ctxt = {} if config('run-internal-router') == 'leader': @@ -126,6 +128,7 @@ class L3AgentContext(OSContextGenerator): class ExternalPortContext(OSContextGenerator): + def __call__(self): if config('ext-port'): return {"ext_port": config('ext-port')} @@ -134,6 +137,7 @@ class ExternalPortContext(OSContextGenerator): class QuantumGatewayContext(OSContextGenerator): + def __call__(self): ctxt = { 'shared_secret': get_shared_secret(), diff --git a/unit_tests/test_quantum_contexts.py b/unit_tests/test_quantum_contexts.py index c7733c3c..beb5fbd5 100644 --- a/unit_tests/test_quantum_contexts.py +++ b/unit_tests/test_quantum_contexts.py @@ -38,6 +38,7 @@ def patch_open(): class _TestQuantumContext(CharmTestCase): + def setUp(self): super(_TestQuantumContext, self).setUp(quantum_contexts, TO_PATCH) self.config.side_effect = self.test_config.get @@ -75,6 +76,7 @@ class _TestQuantumContext(CharmTestCase): class TestQuantumSharedDBContext(_TestQuantumContext): + def setUp(self): super(TestQuantumSharedDBContext, self).setUp() self.context = quantum_contexts.QuantumSharedDBContext() @@ -95,6 +97,7 @@ class TestQuantumSharedDBContext(_TestQuantumContext): class TestNetworkServiceContext(_TestQuantumContext): + def setUp(self): super(TestNetworkServiceContext, self).setUp() self.context = quantum_contexts.NetworkServiceContext() @@ -127,6 +130,7 @@ class TestNetworkServiceContext(_TestQuantumContext): class TestExternalPortContext(CharmTestCase): + def setUp(self): super(TestExternalPortContext, self).setUp(quantum_contexts, TO_PATCH) @@ -143,6 +147,7 @@ class TestExternalPortContext(CharmTestCase): class TestL3AgentContext(CharmTestCase): + def setUp(self): super(TestL3AgentContext, self).setUp(quantum_contexts, TO_PATCH) @@ -173,6 +178,7 @@ class TestL3AgentContext(CharmTestCase): class TestQuantumGatewayContext(CharmTestCase): + def setUp(self): super(TestQuantumGatewayContext, self).setUp(quantum_contexts, TO_PATCH) @@ -199,6 +205,7 @@ class TestQuantumGatewayContext(CharmTestCase): class TestSharedSecret(CharmTestCase): + def setUp(self): super(TestSharedSecret, self).setUp(quantum_contexts, TO_PATCH) @@ -228,6 +235,7 @@ class TestSharedSecret(CharmTestCase): class TestHostIP(CharmTestCase): + def setUp(self): super(TestHostIP, self).setUp(quantum_contexts, TO_PATCH) @@ -261,6 +269,7 @@ class TestHostIP(CharmTestCase): class TestNetworkingName(CharmTestCase): + def setUp(self): super(TestNetworkingName, self).setUp(quantum_contexts, diff --git a/unit_tests/test_quantum_utils.py b/unit_tests/test_quantum_utils.py index bce6933b..2cde8c35 100644 --- a/unit_tests/test_quantum_utils.py +++ b/unit_tests/test_quantum_utils.py @@ -42,6 +42,7 @@ TO_PATCH = [ class TestQuantumUtils(CharmTestCase): + def setUp(self): super(TestQuantumUtils, self).setUp(quantum_utils, TO_PATCH) self.networking_name.return_value = 'neutron' @@ -63,7 +64,8 @@ class TestQuantumUtils(CharmTestCase): def test_get_early_packages_ovs(self): self.config.return_value = 'ovs' - self.determine_dkms_package.return_value = ['openvswitch-datapath-dkms'] + self.determine_dkms_package.return_value = [ + 'openvswitch-datapath-dkms'] self.assertEquals( quantum_utils.get_early_packages(), ['openvswitch-datapath-dkms', 'linux-headers-2.6.18']) @@ -254,6 +256,7 @@ network_context = { class DummyNetworkServiceContext(): + def __init__(self, return_value): self.return_value = return_value @@ -385,6 +388,7 @@ cluster2 = ['cluster2-machine1.internal', 'cluster2-machine2.internal' class TestQuantumAgentReallocation(CharmTestCase): + def setUp(self): if not neutronclient: raise self.skipTest('Skipping, no neutronclient installed') diff --git a/unit_tests/test_utils.py b/unit_tests/test_utils.py index fd7fe233..e04a9dc8 100644 --- a/unit_tests/test_utils.py +++ b/unit_tests/test_utils.py @@ -44,6 +44,7 @@ def get_default_config(): class CharmTestCase(unittest.TestCase): + def setUp(self, obj, patches): super(CharmTestCase, self).setUp() self.patches = patches @@ -64,6 +65,7 @@ class CharmTestCase(unittest.TestCase): class TestConfig(object): + def __init__(self): self.config = get_default_config() @@ -83,6 +85,7 @@ class TestConfig(object): class TestRelation(object): + def __init__(self, relation_data={}): self.relation_data = relation_data