From 24c63641bf1bc1ca84a7f12d2cb5766fdcf3ecee Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 23 Jun 2014 16:00:19 +0100 Subject: [PATCH 01/26] Add configuration options for various openstack networks --- charm-helpers.yaml | 3 +- config.yaml | 17 +++ .../charmhelpers/contrib/hahelpers/cluster.py | 4 +- .../charmhelpers/contrib/network/__init__.py | 0 hooks/charmhelpers/contrib/network/ip.py | 69 +++++++++++ .../charmhelpers/contrib/openstack/context.py | 6 +- hooks/charmhelpers/core/fstab.py | 114 ++++++++++++++++++ hooks/charmhelpers/core/host.py | 26 +++- hooks/charmhelpers/fetch/bzrurl.py | 3 +- hooks/cinder_contexts.py | 1 + hooks/cinder_hooks.py | 28 +++-- hooks/cinder_utils.py | 2 +- 12 files changed, 253 insertions(+), 20 deletions(-) create mode 100644 hooks/charmhelpers/contrib/network/__init__.py create mode 100644 hooks/charmhelpers/contrib/network/ip.py create mode 100644 hooks/charmhelpers/core/fstab.py diff --git a/charm-helpers.yaml b/charm-helpers.yaml index b4a77d14..de3d3853 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-helpers +branch: lp:~james-page/charm-helpers/network-splits destination: hooks/charmhelpers include: - core @@ -10,3 +10,4 @@ include: - cluster - fetch - payload.execd + - contrib.network.ip diff --git a/config.yaml b/config.yaml index 762d3ed7..37cb8e9e 100644 --- a/config.yaml +++ b/config.yaml @@ -142,4 +142,21 @@ options: config-flags: type: string description: Comma separated list of key=value config flags to be set in cinder.conf. + # Network configuration options + # by default all access is over 'private-address' + os-admin-network: + type: string + description: | + The IP address and netmask of the OpenStack Administration network (e.g., + 192.168.0.0/24) + os-public-network: + type: string + description: | + The IP address and netmask of the OpenStack Public network (e.g., + 192.168.0.0/24) + database-network: + type: string + description: | + The IP address and netmask of the Database access network (e.g., + 192.168.0.0/24) diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index bf832f7d..cc0d1095 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -163,7 +163,7 @@ def get_hacluster_config(): return conf -def canonical_url(configs, vip_setting='vip'): +def canonical_url(configs, vip_setting='vip', address=None): ''' Returns the correct HTTP URL to this host given the state of HTTPS configuration and hacluster. @@ -179,5 +179,5 @@ def canonical_url(configs, vip_setting='vip'): if is_clustered(): addr = config_get(vip_setting) else: - addr = unit_get('private-address') + addr = address or unit_get('private-address') return '%s://%s' % (scheme, addr) diff --git a/hooks/charmhelpers/contrib/network/__init__.py b/hooks/charmhelpers/contrib/network/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py new file mode 100644 index 00000000..44c7c975 --- /dev/null +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -0,0 +1,69 @@ +import sys + +from charmhelpers.fetch import apt_install +from charmhelpers.core.hookenv import ( + ERROR, log, +) + +try: + import netifaces +except ImportError: + apt_install('python-netifaces') + import netifaces + +try: + import netaddr +except ImportError: + apt_install('python-netaddr') + import netaddr + + +def _validate_cidr(network): + try: + netaddr.IPNetwork(network) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Network (%s) is not in CIDR presentation format" % + network) + + +def get_address_in_network(network, fallback=None, fatal=False): + """ + Get an IPv4 address within the network from the host. + + Args: + network (str): CIDR presentation format. For example, + '192.168.1.0/24'. + fallback (str): If no address is found, return fallback. + fatal (boolean): If no address is found, fallback is not + set and fatal is True then exit(1). + """ + + def not_found_error_out(): + log("No IP address found in network: %s" % network, + level=ERROR) + sys.exit(1) + + if network is None: + if fallback is not None: + return fallback + else: + if fatal: + not_found_error_out() + + _validate_cidr(network) + for iface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(iface) + if netifaces.AF_INET in addresses: + addr = addresses[netifaces.AF_INET][0]['addr'] + netmask = addresses[netifaces.AF_INET][0]['netmask'] + cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + if cidr in netaddr.IPNetwork(network): + return str(cidr.ip) + + if fallback is not None: + return fallback + + if fatal: + not_found_error_out() + + return None diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 474d51ea..eaeb7eb2 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -332,10 +332,12 @@ class CephContext(OSContextGenerator): use_syslog = str(config('use-syslog')).lower() for rid in relation_ids('ceph'): for unit in related_units(rid): - mon_hosts.append(relation_get('private-address', rid=rid, - unit=unit)) auth = relation_get('auth', rid=rid, unit=unit) key = relation_get('key', rid=rid, unit=unit) + ceph_addr = \ + relation_get('ceph-public-address', rid=rid, unit=unit) or \ + relation_get('private-address', rid=rid, unit=unit) + mon_hosts.append(ceph_addr) ctxt = { 'mon_hosts': ' '.join(mon_hosts), diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py new file mode 100644 index 00000000..cdd72616 --- /dev/null +++ b/hooks/charmhelpers/core/fstab.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +__author__ = 'Jorge Niedbalski R. ' + +import os + + +class Fstab(file): + """This class extends file in order to implement a file reader/writer + for file `/etc/fstab` + """ + + class Entry(object): + """Entry class represents a non-comment line on the `/etc/fstab` file + """ + def __init__(self, device, mountpoint, filesystem, + options, d=0, p=0): + self.device = device + self.mountpoint = mountpoint + self.filesystem = filesystem + + if not options: + options = "defaults" + + self.options = options + self.d = d + self.p = p + + def __eq__(self, o): + return str(self) == str(o) + + def __str__(self): + return "{} {} {} {} {} {}".format(self.device, + self.mountpoint, + self.filesystem, + self.options, + self.d, + self.p) + + DEFAULT_PATH = os.path.join(os.path.sep, 'etc', 'fstab') + + def __init__(self, path=None): + if path: + self._path = path + else: + self._path = self.DEFAULT_PATH + file.__init__(self, self._path, 'r+') + + def _hydrate_entry(self, line): + return Fstab.Entry(*filter( + lambda x: x not in ('', None), + line.strip("\n").split(" "))) + + @property + def entries(self): + self.seek(0) + for line in self.readlines(): + try: + if not line.startswith("#"): + yield self._hydrate_entry(line) + except ValueError: + pass + + def get_entry_by_attr(self, attr, value): + for entry in self.entries: + e_attr = getattr(entry, attr) + if e_attr == value: + return entry + return None + + def add_entry(self, entry): + if self.get_entry_by_attr('device', entry.device): + return False + + self.write(str(entry) + '\n') + self.truncate() + return entry + + def remove_entry(self, entry): + self.seek(0) + + lines = self.readlines() + + found = False + for index, line in enumerate(lines): + if not line.startswith("#"): + if self._hydrate_entry(line) == entry: + found = True + break + + if not found: + return False + + lines.remove(line) + + self.seek(0) + self.write(''.join(lines)) + self.truncate() + return True + + @classmethod + def remove_by_mountpoint(cls, mountpoint, path=None): + fstab = cls(path=path) + entry = fstab.get_entry_by_attr('mountpoint', mountpoint) + if entry: + return fstab.remove_entry(entry) + return False + + @classmethod + def add(cls, device, mountpoint, filesystem, options=None, path=None): + return cls(path=path).add_entry(Fstab.Entry(device, + mountpoint, filesystem, + options=options)) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 186147f6..46bfd36a 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -17,6 +17,7 @@ import apt_pkg from collections import OrderedDict from hookenv import log +from fstab import Fstab def service_start(service_name): @@ -35,7 +36,8 @@ def service_restart(service_name): def service_reload(service_name, restart_on_failure=False): - """Reload a system service, optionally falling back to restart if reload fails""" + """Reload a system service, optionally falling back to restart if + reload fails""" service_result = service('reload', service_name) if not service_result and restart_on_failure: service_result = service('restart', service_name) @@ -144,7 +146,19 @@ def write_file(path, content, owner='root', group='root', perms=0444): target.write(content) -def mount(device, mountpoint, options=None, persist=False): +def fstab_remove(mp): + """Remove the given mountpoint entry from /etc/fstab + """ + return Fstab.remove_by_mountpoint(mp) + + +def fstab_add(dev, mp, fs, options=None): + """Adds the given device entry to the /etc/fstab file + """ + return Fstab.add(dev, mp, fs, options=options) + + +def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"): """Mount a filesystem at a particular mountpoint""" cmd_args = ['mount'] if options is not None: @@ -155,9 +169,9 @@ def mount(device, mountpoint, options=None, persist=False): except subprocess.CalledProcessError, e: log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output)) return False + if persist: - # TODO: update fstab - pass + return fstab_add(device, mountpoint, filesystem, options=options) return True @@ -169,9 +183,9 @@ def umount(mountpoint, persist=False): except subprocess.CalledProcessError, e: log('Error unmounting {}\n{}'.format(mountpoint, e.output)) return False + if persist: - # TODO: update fstab - pass + return fstab_remove(mountpoint) return True diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py index db5dd9a3..0e580e47 100644 --- a/hooks/charmhelpers/fetch/bzrurl.py +++ b/hooks/charmhelpers/fetch/bzrurl.py @@ -39,7 +39,8 @@ class BzrUrlFetchHandler(BaseFetchHandler): def install(self, source): url_parts = self.parse_url(source) branch_name = url_parts.path.strip("/").split("/")[-1] - dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", branch_name) + dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", + branch_name) if not os.path.exists(dest_dir): mkdir(dest_dir, perms=0755) try: diff --git a/hooks/cinder_contexts.py b/hooks/cinder_contexts.py index 2c5a31ed..98f9949b 100644 --- a/hooks/cinder_contexts.py +++ b/hooks/cinder_contexts.py @@ -105,5 +105,6 @@ class StorageBackendContext(OSContextGenerator): class LoggingConfigContext(OSContextGenerator): + def __call__(self): return {'debug': config('debug'), 'verbose': config('verbose')} diff --git a/hooks/cinder_hooks.py b/hooks/cinder_hooks.py index 78ab9c1b..94225083 100755 --- a/hooks/cinder_hooks.py +++ b/hooks/cinder_hooks.py @@ -53,6 +53,7 @@ from charmhelpers.contrib.hahelpers.cluster import ( ) from charmhelpers.payload.execd import execd_preinstall +from charmhelpers.contrib.network.ip import get_address_in_network hooks = Hooks() @@ -65,7 +66,7 @@ def install(): conf = config() src = conf['openstack-origin'] if (lsb_release()['DISTRIB_CODENAME'] == 'precise' and - src == 'distro'): + src == 'distro'): src = 'cloud:precise-folsom' configure_installation_source(src) apt_update() @@ -105,7 +106,8 @@ def db_joined(): conf = config() relation_set(database=conf['database'], username=conf['database-user'], - hostname=unit_get('private-address')) + hostname=get_address_in_network(config('database-network'), + unit_get('private-address'))) @hooks.hook('pgsql-db-relation-joined') @@ -176,16 +178,28 @@ def identity_joined(rid=None): return conf = config() - port = conf['api-listening-port'] - url = canonical_url(CONFIGS) + ':%s/v1/$(tenant_id)s' % port + public_url = '{}:{}/v1/$(tenant_id)s'.format( + canonical_url( + CONFIGS, + address=get_address_in_network(conf.get('os-public-network'), + unit_get('public-address'))), + port + ) + admin_internal_url = '{}:{}/v1/$(tenant_id)s'.format( + canonical_url( + CONFIGS, + address=get_address_in_network(conf.get('os-admin-network'), + unit_get('private-address'))), + port + ) settings = { 'region': conf['region'], 'service': 'cinder', - 'public_url': url, - 'internal_url': url, - 'admin_url': url, + 'public_url': public_url, + 'internal_url': admin_internal_url, + 'admin_url': admin_internal_url, } relation_set(relation_id=rid, **settings) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index 29cfb0d4..378287fc 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -391,7 +391,7 @@ def set_ceph_env_variables(service): with open('/etc/environment', 'a') as out: out.write('CEPH_ARGS="--id %s"\n' % service) with open('/etc/init/cinder-volume.override', 'w') as out: - out.write('env CEPH_ARGS="--id %s"\n' % service) + out.write('env CEPH_ARGS="--id %s"\n' % service) def do_openstack_upgrade(configs): From 23b47f3773e63135493bc6b44af46dc54df16274 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 25 Jun 2014 11:02:54 +0100 Subject: [PATCH 02/26] Switch to using internal-network --- config.yaml | 8 ++++++-- hooks/cinder_hooks.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/config.yaml b/config.yaml index 37cb8e9e..616e183f 100644 --- a/config.yaml +++ b/config.yaml @@ -144,16 +144,20 @@ options: description: Comma separated list of key=value config flags to be set in cinder.conf. # Network configuration options # by default all access is over 'private-address' - os-admin-network: + os-internal-network: type: string description: | - The IP address and netmask of the OpenStack Administration network (e.g., + The IP address and netmask of the OpenStack Internal network (e.g., 192.168.0.0/24) + . + This network will be used for admin and internal endpoints. os-public-network: type: string description: | The IP address and netmask of the OpenStack Public network (e.g., 192.168.0.0/24) + . + This network will be used for public endpoints. database-network: type: string description: | diff --git a/hooks/cinder_hooks.py b/hooks/cinder_hooks.py index 94225083..e01e1e25 100755 --- a/hooks/cinder_hooks.py +++ b/hooks/cinder_hooks.py @@ -189,7 +189,7 @@ def identity_joined(rid=None): admin_internal_url = '{}:{}/v1/$(tenant_id)s'.format( canonical_url( CONFIGS, - address=get_address_in_network(conf.get('os-admin-network'), + address=get_address_in_network(conf.get('os-internal-network'), unit_get('private-address'))), port ) From ddb6818387827094eb870640f823b1c32be32ffd Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 27 Jun 2014 11:35:52 +0100 Subject: [PATCH 03/26] Add extra admin-network configuration --- config.yaml | 9 ++++++++- hooks/cinder_hooks.py | 16 +++++++++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/config.yaml b/config.yaml index 616e183f..26460d99 100644 --- a/config.yaml +++ b/config.yaml @@ -144,13 +144,20 @@ options: description: Comma separated list of key=value config flags to be set in cinder.conf. # Network configuration options # by default all access is over 'private-address' + os-admin-network: + type: string + description: | + The IP address and netmask of the OpenStack Admin network (e.g., + 192.168.0.0/24) + . + This network will be used for admin endpoints. os-internal-network: type: string description: | The IP address and netmask of the OpenStack Internal network (e.g., 192.168.0.0/24) . - This network will be used for admin and internal endpoints. + This network will be used for internal endpoints. os-public-network: type: string description: | diff --git a/hooks/cinder_hooks.py b/hooks/cinder_hooks.py index e01e1e25..e92ce7d4 100755 --- a/hooks/cinder_hooks.py +++ b/hooks/cinder_hooks.py @@ -106,7 +106,7 @@ def db_joined(): conf = config() relation_set(database=conf['database'], username=conf['database-user'], - hostname=get_address_in_network(config('database-network'), + hostname=get_address_in_network(conf.get('database-network'), unit_get('private-address'))) @@ -186,20 +186,26 @@ def identity_joined(rid=None): unit_get('public-address'))), port ) - admin_internal_url = '{}:{}/v1/$(tenant_id)s'.format( + internal_url = '{}:{}/v1/$(tenant_id)s'.format( canonical_url( CONFIGS, address=get_address_in_network(conf.get('os-internal-network'), unit_get('private-address'))), port ) - + admin_url = '{}:{}/v1/$(tenant_id)s'.format( + canonical_url( + CONFIGS, + address=get_address_in_network(conf.get('os-admin-network'), + unit_get('private-address'))), + port + ) settings = { 'region': conf['region'], 'service': 'cinder', 'public_url': public_url, - 'internal_url': admin_internal_url, - 'admin_url': admin_internal_url, + 'internal_url': internal_url, + 'admin_url': admin_url, } relation_set(relation_id=rid, **settings) From d0b42f999cf4c42568441a54e546a6ab3d926738 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 27 Jun 2014 14:49:24 +0100 Subject: [PATCH 04/26] Resync helpers --- .coverage | 8 + .../charmhelpers/contrib/hahelpers/cluster.py | 1 + hooks/charmhelpers/contrib/network/ip.py | 12 +- .../contrib/openstack/amulet/__init__.py | 0 .../contrib/openstack/amulet/deployment.py | 38 ++++ .../contrib/openstack/amulet/utils.py | 209 ++++++++++++++++++ .../charmhelpers/contrib/openstack/context.py | 58 +++-- .../charmhelpers/contrib/openstack/neutron.py | 14 ++ .../contrib/openstack/templating.py | 37 ++-- hooks/charmhelpers/contrib/openstack/utils.py | 7 +- .../contrib/storage/linux/ceph.py | 2 +- .../contrib/storage/linux/utils.py | 1 + hooks/charmhelpers/core/fstab.py | 4 +- hooks/charmhelpers/core/hookenv.py | 9 +- hooks/charmhelpers/core/host.py | 14 +- hooks/charmhelpers/fetch/__init__.py | 40 ++-- 16 files changed, 386 insertions(+), 68 deletions(-) create mode 100644 .coverage create mode 100644 hooks/charmhelpers/contrib/openstack/amulet/__init__.py create mode 100644 hooks/charmhelpers/contrib/openstack/amulet/deployment.py create mode 100644 hooks/charmhelpers/contrib/openstack/amulet/utils.py diff --git a/.coverage b/.coverage new file mode 100644 index 00000000..de1f6d92 --- /dev/null +++ b/.coverage @@ -0,0 +1,8 @@ +}q(U collectorqUcoverage v3.7.1qUlinesq}q(Ud/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/hahelpers/__init__.pyq]qKaUc/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/openstack/neutron.pyq]q (KKK KKKKK*KOKRKKeUd/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/openstack/__init__.pyq +]q KaUZ/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/fetch/archiveurl.pyq ]Uc/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/storage/linux/lvm.pyq ]q(KK KK(K3KEKOeUV/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/fetch/bzrurl.pyq]US/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/core/host.pyq]q(KKK K +K K K KKKKKKKK!K&K/K5KBKXKcKnKzKKKKKKKKKKKKKKKKKKKKKKKKKKKKKMMM M&M1M;eUI/home/jamespage/src/charms/reference-network/cinder/hooks/cinder_hooks.pyq]q(KKKKK KK(K)K+K.K0K7K8K:KMDMEMFMGMIMJMLMMMPMRMSMVMWMXMZM]eUb/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/hahelpers/apache.pyq]q(K K KK)K6eUa/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/openstack/utils.pyq]q(KKKKK K +K KKKKKKK K"K&K'K(K)K*K+K,K-K1K2K3K4K5K6K7K8KK?K@KAKBKCKDKEKFKGKHKIKLKOKTKmKrK{KKKKKKM*M<MOMpMMMMeUE/home/jamespage/src/charms/reference-network/cinder/hooks/__init__.pyq]Un/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/openstack/templates/__init__.pyq]Ub/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/storage/__init__.pyq]qKaUh/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/storage/linux/__init__.pyq]qKaUd/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/storage/linux/ceph.pyq]q(K K K KKKK K)K-K.K7K:KBKMK]KgKtKKKKKKKKKKKMMM#M/MbMxeUV/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/core/hookenv.pyq ]q!(KKKK K +K K K KKKKKKKKK&K/K2K=KFKGKIKNK\K`KdKhKmK{KKKKKKKKKKKKKKKKKKM M M M/M0M:M;MDMEMPMQM\M]MhMwMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMeUc/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/hahelpers/cluster.pyq"]q#(K K +K KKKKK)K9KAKJKWKpKKKeUb/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/network/__init__.pyq$]q%KaUe/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/storage/linux/utils.pyq&]q'(KKKKK KK(K0K1eUR/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/__init__.pyq(]q)KaUT/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/core/fstab.pyq*]q+(KKK K KKKKK!K)K+K2K7KAKHKPKfKgKnKoeUh/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/openstack/alternatives.pyq,]q-(KKKKKeUW/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/payload/execd.pyq.]q/(KKKKK K KK$K0eUn/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/network.~1~/ovs.~1~/__init__.pyq0]UL/home/jamespage/src/charms/reference-network/cinder/hooks/cinder_contexts.pyq1]q2(KK KKKKKKKKK!K"K$K)K*K+K,K-K.K0K1K2K4K5K6K:K;K=KMKNKOKPKRKTKUKVKWKZK[K]K^K_K`KaKbKcKdKeKfKhKkKmeUc/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/openstack/context.pyq3]q4(KKKKKK KKK$K)K-K0K1K4K;KFKkKlKnKrKsKvK|K}K~KKKKKKKKKKKKKKKKM?M@MBMhMiMkMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM"M2MKMTMVM_MMMMMMMMeU\/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/network/ip.pyq5]q6(KKKKK KKKKK)K.K/K0eUX/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/fetch/__init__.pyq7]q8(KKKKKK K KKKKKKKKKK K!K"K$K%K&K'K(K)K*K,K-K.K/K0K1K2K4K5K6K7K8K9K:KK?K@KAKBKCKJKMKNKOKRKSKVKWKZK[K^K`KbKgKlKoKvKKKKKKKKKM M"M(M;eUh/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/storage/linux/loopback.pyq9]q:(KKKKK K-eUW/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/core/__init__.pyq;]q(KKKK K KKKKKFKJKKKUKaKkKKKKKKKMMMeUZ/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/__init__.pyq?]q@KaUI/home/jamespage/src/charms/reference-network/cinder/hooks/cinder_utils.pyqA]qB(KKKKKKKKK!K"K&K,K5K9K>KDKGKHKIKJKKKLKMKNKOKRKSKTKVKYK\K]K_K`KaKbKcKeKfKgKjKmKnKrKsKtKuKvKwKxKyKzK{K|K}K~KKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKMMMMMM +MMMMM M"M#M$M%M&M*M+M,M-M.M/M1M2M3M4M5M6M8M:M<M=M?MAMBMEMNMOMPMQMRMTMUMVMXM[MdMeMfMgMhMiMjMkMlMnMoMpMrMuMwMxM{M~MMMMMMMMMMMMMMMMMMeUf/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/contrib/network.~1~/__init__.pyqC]UZ/home/jamespage/src/charms/reference-network/cinder/hooks/charmhelpers/payload/__init__.pyqD]qEKauu. \ No newline at end of file diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index cc0d1095..dd89f347 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -170,6 +170,7 @@ def canonical_url(configs, vip_setting='vip', address=None): :configs : OSTemplateRenderer: A config tempating object to inspect for a complete https context. + :vip_setting: str: Setting in charm config that specifies VIP address. ''' diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 44c7c975..15a6731c 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -30,12 +30,12 @@ def get_address_in_network(network, fallback=None, fatal=False): """ Get an IPv4 address within the network from the host. - Args: - network (str): CIDR presentation format. For example, - '192.168.1.0/24'. - fallback (str): If no address is found, return fallback. - fatal (boolean): If no address is found, fallback is not - set and fatal is True then exit(1). + :param network (str): CIDR presentation format. For example, + '192.168.1.0/24'. + :param fallback (str): If no address is found, return fallback. + :param fatal (boolean): If no address is found, fallback is not + set and fatal is True then exit(1). + """ def not_found_error_out(): diff --git a/hooks/charmhelpers/contrib/openstack/amulet/__init__.py b/hooks/charmhelpers/contrib/openstack/amulet/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py new file mode 100644 index 00000000..9e164821 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -0,0 +1,38 @@ +from charmhelpers.contrib.amulet.deployment import ( + AmuletDeployment +) + + +class OpenStackAmuletDeployment(AmuletDeployment): + """This class inherits from AmuletDeployment and has additional support + that is specifically for use by OpenStack charms.""" + + def __init__(self, series=None, openstack=None): + """Initialize the deployment environment.""" + self.openstack = None + super(OpenStackAmuletDeployment, self).__init__(series) + + if openstack: + self.openstack = openstack + + def _configure_services(self, configs): + """Configure all of the services.""" + for service, config in configs.iteritems(): + if service == self.this_service: + config['openstack-origin'] = self.openstack + self.d.configure(service, config) + + def _get_openstack_release(self): + """Return an integer representing the enum value of the openstack + release.""" + self.precise_essex, self.precise_folsom, self.precise_grizzly, \ + self.precise_havana, self.precise_icehouse, \ + self.trusty_icehouse = range(6) + releases = { + ('precise', None): self.precise_essex, + ('precise', 'cloud:precise-folsom'): self.precise_folsom, + ('precise', 'cloud:precise-grizzly'): self.precise_grizzly, + ('precise', 'cloud:precise-havana'): self.precise_havana, + ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, + ('trusty', None): self.trusty_icehouse} + return releases[(self.series, self.openstack)] diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py new file mode 100644 index 00000000..6515f907 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -0,0 +1,209 @@ +import logging +import os +import time +import urllib + +import glanceclient.v1.client as glance_client +import keystoneclient.v2_0 as keystone_client +import novaclient.v1_1.client as nova_client + +from charmhelpers.contrib.amulet.utils import ( + AmuletUtils +) + +DEBUG = logging.DEBUG +ERROR = logging.ERROR + + +class OpenStackAmuletUtils(AmuletUtils): + """This class inherits from AmuletUtils and has additional support + that is specifically for use by OpenStack charms.""" + + def __init__(self, log_level=ERROR): + """Initialize the deployment environment.""" + super(OpenStackAmuletUtils, self).__init__(log_level) + + def validate_endpoint_data(self, endpoints, admin_port, internal_port, + public_port, expected): + """Validate actual endpoint data vs expected endpoint data. The ports + are used to find the matching endpoint.""" + found = False + for ep in endpoints: + self.log.debug('endpoint: {}'.format(repr(ep))) + if admin_port in ep.adminurl and internal_port in ep.internalurl \ + and public_port in ep.publicurl: + found = True + actual = {'id': ep.id, + 'region': ep.region, + 'adminurl': ep.adminurl, + 'internalurl': ep.internalurl, + 'publicurl': ep.publicurl, + 'service_id': ep.service_id} + ret = self._validate_dict_data(expected, actual) + if ret: + return 'unexpected endpoint data - {}'.format(ret) + + if not found: + return 'endpoint not found' + + def validate_svc_catalog_endpoint_data(self, expected, actual): + """Validate a list of actual service catalog endpoints vs a list of + expected service catalog endpoints.""" + self.log.debug('actual: {}'.format(repr(actual))) + for k, v in expected.iteritems(): + if k in actual: + ret = self._validate_dict_data(expected[k][0], actual[k][0]) + if ret: + return self.endpoint_error(k, ret) + else: + return "endpoint {} does not exist".format(k) + return ret + + def validate_tenant_data(self, expected, actual): + """Validate a list of actual tenant data vs list of expected tenant + data.""" + self.log.debug('actual: {}'.format(repr(actual))) + for e in expected: + found = False + for act in actual: + a = {'enabled': act.enabled, 'description': act.description, + 'name': act.name, 'id': act.id} + if e['name'] == a['name']: + found = True + ret = self._validate_dict_data(e, a) + if ret: + return "unexpected tenant data - {}".format(ret) + if not found: + return "tenant {} does not exist".format(e.name) + return ret + + def validate_role_data(self, expected, actual): + """Validate a list of actual role data vs a list of expected role + data.""" + self.log.debug('actual: {}'.format(repr(actual))) + for e in expected: + found = False + for act in actual: + a = {'name': act.name, 'id': act.id} + if e['name'] == a['name']: + found = True + ret = self._validate_dict_data(e, a) + if ret: + return "unexpected role data - {}".format(ret) + if not found: + return "role {} does not exist".format(e.name) + return ret + + def validate_user_data(self, expected, actual): + """Validate a list of actual user data vs a list of expected user + data.""" + self.log.debug('actual: {}'.format(repr(actual))) + for e in expected: + found = False + for act in actual: + a = {'enabled': act.enabled, 'name': act.name, + 'email': act.email, 'tenantId': act.tenantId, + 'id': act.id} + if e['name'] == a['name']: + found = True + ret = self._validate_dict_data(e, a) + if ret: + return "unexpected user data - {}".format(ret) + if not found: + return "user {} does not exist".format(e.name) + return ret + + def validate_flavor_data(self, expected, actual): + """Validate a list of actual flavors vs a list of expected flavors.""" + self.log.debug('actual: {}'.format(repr(actual))) + act = [a.name for a in actual] + return self._validate_list_data(expected, act) + + def tenant_exists(self, keystone, tenant): + """Return True if tenant exists""" + return tenant in [t.name for t in keystone.tenants.list()] + + def authenticate_keystone_admin(self, keystone_sentry, user, password, + tenant): + """Authenticates admin user with the keystone admin endpoint.""" + service_ip = \ + keystone_sentry.relation('shared-db', + 'mysql:shared-db')['private-address'] + ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8')) + return keystone_client.Client(username=user, password=password, + tenant_name=tenant, auth_url=ep) + + def authenticate_keystone_user(self, keystone, user, password, tenant): + """Authenticates a regular user with the keystone public endpoint.""" + ep = keystone.service_catalog.url_for(service_type='identity', + endpoint_type='publicURL') + return keystone_client.Client(username=user, password=password, + tenant_name=tenant, auth_url=ep) + + def authenticate_glance_admin(self, keystone): + """Authenticates admin user with glance.""" + ep = keystone.service_catalog.url_for(service_type='image', + endpoint_type='adminURL') + return glance_client.Client(ep, token=keystone.auth_token) + + def authenticate_nova_user(self, keystone, user, password, tenant): + """Authenticates a regular user with nova-api.""" + ep = keystone.service_catalog.url_for(service_type='identity', + endpoint_type='publicURL') + return nova_client.Client(username=user, api_key=password, + project_id=tenant, auth_url=ep) + + def create_cirros_image(self, glance, image_name): + """Download the latest cirros image and upload it to glance.""" + http_proxy = os.getenv('AMULET_HTTP_PROXY') + self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) + if http_proxy: + proxies = {'http': http_proxy} + opener = urllib.FancyURLopener(proxies) + else: + opener = urllib.FancyURLopener() + + f = opener.open("http://download.cirros-cloud.net/version/released") + version = f.read().strip() + cirros_img = "tests/cirros-{}-x86_64-disk.img".format(version) + + if not os.path.exists(cirros_img): + cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net", + version, cirros_img) + opener.retrieve(cirros_url, cirros_img) + f.close() + + with open(cirros_img) as f: + image = glance.images.create(name=image_name, is_public=True, + disk_format='qcow2', + container_format='bare', data=f) + return image + + def delete_image(self, glance, image): + """Delete the specified image.""" + glance.images.delete(image) + + def create_instance(self, nova, image_name, instance_name, flavor): + """Create the specified instance.""" + image = nova.images.find(name=image_name) + flavor = nova.flavors.find(name=flavor) + instance = nova.servers.create(name=instance_name, image=image, + flavor=flavor) + + count = 1 + status = instance.status + while status == 'BUILD' and count < 10: + time.sleep(5) + instance = nova.servers.get(instance.id) + status = instance.status + self.log.debug('instance status: {}'.format(status)) + count += 1 + + if status == 'BUILD': + return None + + return instance + + def delete_instance(self, nova, instance): + """Delete the specified instance.""" + nova.servers.delete(instance) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index eaeb7eb2..aea11b34 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -243,23 +243,31 @@ class IdentityServiceContext(OSContextGenerator): class AMQPContext(OSContextGenerator): - interfaces = ['amqp'] - def __init__(self, ssl_dir=None): + def __init__(self, ssl_dir=None, rel_name='amqp', relation_prefix=None): self.ssl_dir = ssl_dir + self.rel_name = rel_name + self.relation_prefix = relation_prefix + self.interfaces = [rel_name] def __call__(self): log('Generating template context for amqp') conf = config() + user_setting = 'rabbit-user' + vhost_setting = 'rabbit-vhost' + if self.relation_prefix: + user_setting = self.relation_prefix + '-rabbit-user' + vhost_setting = self.relation_prefix + '-rabbit-vhost' + try: - username = conf['rabbit-user'] - vhost = conf['rabbit-vhost'] + username = conf[user_setting] + vhost = conf[vhost_setting] except KeyError as e: log('Could not generate shared_db context. ' 'Missing required charm config options: %s.' % e) raise OSContextError ctxt = {} - for rid in relation_ids('amqp'): + for rid in relation_ids(self.rel_name): ha_vip_only = False for unit in related_units(rid): if relation_get('clustered', rid=rid, unit=unit): @@ -420,12 +428,13 @@ class ApacheSSLContext(OSContextGenerator): """ Generates a context for an apache vhost configuration that configures HTTPS reverse proxying for one or many endpoints. Generated context - looks something like: - { - 'namespace': 'cinder', - 'private_address': 'iscsi.mycinderhost.com', - 'endpoints': [(8776, 8766), (8777, 8767)] - } + looks something like:: + + { + 'namespace': 'cinder', + 'private_address': 'iscsi.mycinderhost.com', + 'endpoints': [(8776, 8766), (8777, 8767)] + } The endpoints list consists of a tuples mapping external ports to internal ports. @@ -543,6 +552,26 @@ class NeutronContext(OSContextGenerator): return nvp_ctxt + def n1kv_ctxt(self): + driver = neutron_plugin_attribute(self.plugin, 'driver', + self.network_manager) + n1kv_config = neutron_plugin_attribute(self.plugin, 'config', + self.network_manager) + n1kv_ctxt = { + 'core_plugin': driver, + 'neutron_plugin': 'n1kv', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': n1kv_config, + 'vsm_ip': config('n1kv-vsm-ip'), + 'vsm_username': config('n1kv-vsm-username'), + 'vsm_password': config('n1kv-vsm-password'), + 'restrict_policy_profiles': config( + 'n1kv_restrict_policy_profiles'), + } + + return n1kv_ctxt + def neutron_ctxt(self): if https(): proto = 'https' @@ -574,6 +603,8 @@ class NeutronContext(OSContextGenerator): ctxt.update(self.ovs_ctxt()) elif self.plugin in ['nvp', 'nsx']: ctxt.update(self.nvp_ctxt()) + elif self.plugin == 'n1kv': + ctxt.update(self.n1kv_ctxt()) alchemy_flags = config('neutron-alchemy-flags') if alchemy_flags: @@ -613,7 +644,7 @@ class SubordinateConfigContext(OSContextGenerator): The subordinate interface allows subordinates to export their configuration requirements to the principle for multiple config files and multiple serivces. Ie, a subordinate that has interfaces - to both glance and nova may export to following yaml blob as json: + to both glance and nova may export to following yaml blob as json:: glance: /etc/glance/glance-api.conf: @@ -632,7 +663,8 @@ class SubordinateConfigContext(OSContextGenerator): It is then up to the principle charms to subscribe this context to the service+config file it is interestd in. Configuration data will - be available in the template context, in glance's case, as: + be available in the template context, in glance's case, as:: + ctxt = { ... other context ... 'subordinate_config': { diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index ba97622c..84d97bca 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -128,6 +128,20 @@ def neutron_plugins(): 'server_packages': ['neutron-server', 'neutron-plugin-vmware'], 'server_services': ['neutron-server'] + }, + 'n1kv': { + 'config': '/etc/neutron/plugins/cisco/cisco_plugins.ini', + 'driver': 'neutron.plugins.cisco.network_plugin.PluginV2', + 'contexts': [ + context.SharedDBContext(user=config('neutron-database-user'), + database=config('neutron-database'), + relation_prefix='neutron', + ssl_dir=NEUTRON_CONF_DIR)], + 'services': [], + 'packages': [['neutron-plugin-cisco']], + 'server_packages': ['neutron-server', + 'neutron-plugin-cisco'], + 'server_services': ['neutron-server'] } } if release >= 'icehouse': diff --git a/hooks/charmhelpers/contrib/openstack/templating.py b/hooks/charmhelpers/contrib/openstack/templating.py index 4595778c..f5442712 100644 --- a/hooks/charmhelpers/contrib/openstack/templating.py +++ b/hooks/charmhelpers/contrib/openstack/templating.py @@ -30,17 +30,17 @@ def get_loader(templates_dir, os_release): loading dir. A charm may also ship a templates dir with this module - and it will be appended to the bottom of the search list, eg: - hooks/charmhelpers/contrib/openstack/templates. + and it will be appended to the bottom of the search list, eg:: - :param templates_dir: str: Base template directory containing release - sub-directories. - :param os_release : str: OpenStack release codename to construct template - loader. + hooks/charmhelpers/contrib/openstack/templates - :returns : jinja2.ChoiceLoader constructed with a list of - jinja2.FilesystemLoaders, ordered in descending - order by OpenStack release. + :param templates_dir (str): Base template directory containing release + sub-directories. + :param os_release (str): OpenStack release codename to construct template + loader. + :returns: jinja2.ChoiceLoader constructed with a list of + jinja2.FilesystemLoaders, ordered in descending + order by OpenStack release. """ tmpl_dirs = [(rel, os.path.join(templates_dir, rel)) for rel in OPENSTACK_CODENAMES.itervalues()] @@ -111,7 +111,8 @@ class OSConfigRenderer(object): and ease the burden of managing config templates across multiple OpenStack releases. - Basic usage: + Basic usage:: + # import some common context generates from charmhelpers from charmhelpers.contrib.openstack import context @@ -131,21 +132,19 @@ class OSConfigRenderer(object): # write out all registered configs configs.write_all() - Details: + **OpenStack Releases and template loading** - OpenStack Releases and template loading - --------------------------------------- When the object is instantiated, it is associated with a specific OS release. This dictates how the template loader will be constructed. The constructed loader attempts to load the template from several places in the following order: - - from the most recent OS release-specific template dir (if one exists) - - the base templates_dir - - a template directory shipped in the charm with this helper file. + - from the most recent OS release-specific template dir (if one exists) + - the base templates_dir + - a template directory shipped in the charm with this helper file. + For the example above, '/tmp/templates' contains the following structure:: - For the example above, '/tmp/templates' contains the following structure: /tmp/templates/nova.conf /tmp/templates/api-paste.ini /tmp/templates/grizzly/api-paste.ini @@ -169,8 +168,8 @@ class OSConfigRenderer(object): $CHARM/hooks/charmhelpers/contrib/openstack/templates. This allows us to ship common templates (haproxy, apache) with the helpers. - Context generators - --------------------------------------- + **Context generators** + Context generators are used to generate template contexts during hook execution. Doing so may require inspecting service relations, charm config, etc. When registered, a config file is associated with a list diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 1a44ab1f..127b03fe 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -3,7 +3,6 @@ # Common python helper functions used for OpenStack charms. from collections import OrderedDict -import apt_pkg as apt import subprocess import os import socket @@ -85,6 +84,8 @@ def get_os_codename_install_source(src): '''Derive OpenStack release codename from a given installation source.''' ubuntu_rel = lsb_release()['DISTRIB_CODENAME'] rel = '' + if src is None: + return rel if src in ['distro', 'distro-proposed']: try: rel = UBUNTU_OPENSTACK_RELEASE[ubuntu_rel] @@ -132,6 +133,7 @@ def get_os_version_codename(codename): def get_os_codename_package(package, fatal=True): '''Derive OpenStack release codename from an installed package.''' + import apt_pkg as apt apt.init() # Tell apt to build an in-memory cache to prevent race conditions (if @@ -189,7 +191,7 @@ def get_os_version_package(pkg, fatal=True): for version, cname in vers_map.iteritems(): if cname == codename: return version - #e = "Could not determine OpenStack version for package: %s" % pkg + # e = "Could not determine OpenStack version for package: %s" % pkg # error_out(e) @@ -325,6 +327,7 @@ def openstack_upgrade_available(package): """ + import apt_pkg as apt src = config('openstack-origin') cur_vers = get_os_version_package(package) available_vers = get_os_version_install_source(src) diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 12417410..768438a4 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -303,7 +303,7 @@ def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point, blk_device, fstype, system_services=[]): """ NOTE: This function must only be called from a single service unit for - the same rbd_img otherwise data loss will occur. + the same rbd_img otherwise data loss will occur. Ensures given pool and RBD image exists, is mapped to a block device, and the device is formatted and mounted at the given mount_point. diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index b87ef26d..8d0f6116 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -37,6 +37,7 @@ def zap_disk(block_device): check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device), 'bs=512', 'count=100', 'seek=%s' % (gpt_end)]) + def is_device_mounted(device): '''Given a device path, return True if that device is mounted, and False if it isn't. diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py index cdd72616..cfaf0a65 100644 --- a/hooks/charmhelpers/core/fstab.py +++ b/hooks/charmhelpers/core/fstab.py @@ -48,9 +48,11 @@ class Fstab(file): file.__init__(self, self._path, 'r+') def _hydrate_entry(self, line): + # NOTE: use split with no arguments to split on any + # whitespace including tabs return Fstab.Entry(*filter( lambda x: x not in ('', None), - line.strip("\n").split(" "))) + line.strip("\n").split())) @property def entries(self): diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index c2e66f66..c9530433 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -25,7 +25,7 @@ cache = {} def cached(func): """Cache return values for multiple executions of func + args - For example: + For example:: @cached def unit_get(attribute): @@ -445,18 +445,19 @@ class UnregisteredHookError(Exception): class Hooks(object): """A convenient handler for hook functions. - Example: + Example:: + hooks = Hooks() # register a hook, taking its name from the function name @hooks.hook() def install(): - ... + pass # your code here # register a hook, providing a custom hook name @hooks.hook("config-changed") def config_changed(): - ... + pass # your code here if __name__ == "__main__": # execute a hook based on the name the program is called by diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 46bfd36a..8b617a42 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -12,7 +12,6 @@ import random import string import subprocess import hashlib -import apt_pkg from collections import OrderedDict @@ -212,13 +211,13 @@ def file_hash(path): def restart_on_change(restart_map, stopstart=False): """Restart services based on configuration files changing - This function is used a decorator, for example + This function is used a decorator, for example:: @restart_on_change({ '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ] }) def ceph_client_changed(): - ... + pass # your code here In this example, the cinder-api and cinder-volume services would be restarted if /etc/ceph/ceph.conf is changed by the @@ -314,10 +313,13 @@ def get_nic_hwaddr(nic): def cmp_pkgrevno(package, revno, pkgcache=None): '''Compare supplied revno with the revno of the installed package - 1 => Installed revno is greater than supplied arg - 0 => Installed revno is the same as supplied arg - -1 => Installed revno is less than supplied arg + + * 1 => Installed revno is greater than supplied arg + * 0 => Installed revno is the same as supplied arg + * -1 => Installed revno is less than supplied arg + ''' + import apt_pkg if not pkgcache: apt_pkg.init() pkgcache = apt_pkg.Cache() diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index e8e837a5..5be512ce 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -13,7 +13,6 @@ from charmhelpers.core.hookenv import ( config, log, ) -import apt_pkg import os @@ -117,6 +116,7 @@ class BaseFetchHandler(object): def filter_installed_packages(packages): """Returns a list of packages that require installation""" + import apt_pkg apt_pkg.init() # Tell apt to build an in-memory cache to prevent race conditions (if @@ -235,31 +235,39 @@ def configure_sources(update=False, sources_var='install_sources', keys_var='install_keys'): """ - Configure multiple sources from charm configuration + Configure multiple sources from charm configuration. + + The lists are encoded as yaml fragments in the configuration. + The frament needs to be included as a string. Example config: - install_sources: + install_sources: | - "ppa:foo" - "http://example.com/repo precise main" - install_keys: + install_keys: | - null - "a1b2c3d4" Note that 'null' (a.k.a. None) should not be quoted. """ - sources = safe_load(config(sources_var)) - keys = config(keys_var) - if keys is not None: - keys = safe_load(keys) - if isinstance(sources, basestring) and ( - keys is None or isinstance(keys, basestring)): - add_source(sources, keys) + sources = safe_load((config(sources_var) or '').strip()) or [] + keys = safe_load((config(keys_var) or '').strip()) or None + + if isinstance(sources, basestring): + sources = [sources] + + if keys is None: + for source in sources: + add_source(source, None) else: - if not len(sources) == len(keys): - msg = 'Install sources and keys lists are different lengths' - raise SourceConfigError(msg) - for src_num in range(len(sources)): - add_source(sources[src_num], keys[src_num]) + if isinstance(keys, basestring): + keys = [keys] + + if len(sources) != len(keys): + raise SourceConfigError( + 'Install sources and keys lists are different lengths') + for source, key in zip(sources, keys): + add_source(source, key) if update: apt_update(fatal=True) From 730b876845ddf1c4d04cddb0327710f7996904ca Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 2 Jul 2014 09:07:17 +0100 Subject: [PATCH 05/26] Resync helpers for mysql changes --- Makefile | 12 ++++++--- config.yaml | 5 ---- hooks/charmhelpers/contrib/network/ip.py | 26 +++++++++++++++++++ .../charmhelpers/contrib/openstack/context.py | 17 ++++++++++++ hooks/cinder_hooks.py | 6 ++--- 5 files changed, 54 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 8d872e13..e101bf32 100644 --- a/Makefile +++ b/Makefile @@ -6,11 +6,15 @@ lint: @charm proof test: - @echo Starting tests... - @$(PYTHON) /usr/bin/nosetests --nologcapture unit_tests + @$(PYTHON) /usr/bin/nosetests --nologcapture --with-coverage unit_tests -sync: - @charm-helper-sync -c charm-helpers.yaml +bin/charm_helpers_sync.py: + @mkdir -p bin + @bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py \ + > bin/charm_helpers_sync.py + +sync: bin/charm_helpers_sync.py + @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers.yaml publish: lint test bzr push lp:charms/cinder diff --git a/config.yaml b/config.yaml index 26460d99..d21d24f9 100644 --- a/config.yaml +++ b/config.yaml @@ -165,9 +165,4 @@ options: 192.168.0.0/24) . This network will be used for public endpoints. - database-network: - type: string - description: | - The IP address and netmask of the Database access network (e.g., - 192.168.0.0/24) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 15a6731c..f2fa263f 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -67,3 +67,29 @@ def get_address_in_network(network, fallback=None, fatal=False): not_found_error_out() return None + + +def is_address_in_network(network, address): + """ + Determine whether the provided address is within a network range. + + :param network (str): CIDR presentation format. For example, + '192.168.1.0/24'. + :param address: An individual IPv4 or IPv6 address without a net + mask or subnet prefix. For example, '192.168.1.1'. + :returns boolean: Flag indicating whether address is in network. + """ + try: + network = netaddr.IPNetwork(network) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Network (%s) is not in CIDR presentation format" % + network) + try: + address = netaddr.IPAddress(address) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Address (%s) is not in correct presentation format" % + address) + if address in network: + return True + else: + return False diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index aea11b34..b21fca60 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -21,6 +21,7 @@ from charmhelpers.core.hookenv import ( relation_get, relation_ids, related_units, + relation_set, unit_get, unit_private_ip, ERROR, @@ -42,6 +43,8 @@ from charmhelpers.contrib.openstack.neutron import ( neutron_plugin_attribute, ) +from charmhelpers.contrib.network.ip import get_address_in_network + CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' @@ -134,8 +137,22 @@ class SharedDBContext(OSContextGenerator): 'Missing required charm config options. ' '(database name and user)') raise OSContextError + ctxt = {} + # NOTE(jamespage) if mysql charm provides a network upon which + # access to the database should be made, reconfigure relation + # with the service units local address and defer execution + access_network = relation_get('access-network') + if access_network is not None: + access_hostname = get_address_in_network(access_network, + unit_get('private-address')) + set_hostname = relation_get(attribute='hostname', + unit=local_unit()) + if set_hostname != access_hostname: + relation_set(hostname=access_hostname) + return ctxt # Defer any further hook execution for now.... + password_setting = 'password' if self.relation_prefix: password_setting = self.relation_prefix + '_password' diff --git a/hooks/cinder_hooks.py b/hooks/cinder_hooks.py index e92ce7d4..ef7d6eb3 100755 --- a/hooks/cinder_hooks.py +++ b/hooks/cinder_hooks.py @@ -34,7 +34,8 @@ from charmhelpers.core.hookenv import ( service_name, unit_get, log, - ERROR + ERROR, + local_unit, ) from charmhelpers.fetch import apt_install, apt_update @@ -106,8 +107,7 @@ def db_joined(): conf = config() relation_set(database=conf['database'], username=conf['database-user'], - hostname=get_address_in_network(conf.get('database-network'), - unit_get('private-address'))) + hostname=unit_get('private-address')) @hooks.hook('pgsql-db-relation-joined') From 92ed63861a21c6aa09db2aab17833c1ef1c5e664 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 2 Jul 2014 09:13:36 +0100 Subject: [PATCH 06/26] Resync helpers --- .bzrignore | 2 ++ .../contrib/openstack/amulet/deployment.py | 29 +++++++++++++++---- .../contrib/openstack/amulet/utils.py | 10 +++---- 3 files changed, 30 insertions(+), 11 deletions(-) create mode 100644 .bzrignore diff --git a/.bzrignore b/.bzrignore new file mode 100644 index 00000000..a2c7a097 --- /dev/null +++ b/.bzrignore @@ -0,0 +1,2 @@ +bin +.coverage diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 9e164821..e476b6f2 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -7,19 +7,36 @@ class OpenStackAmuletDeployment(AmuletDeployment): """This class inherits from AmuletDeployment and has additional support that is specifically for use by OpenStack charms.""" - def __init__(self, series=None, openstack=None): + def __init__(self, series=None, openstack=None, source=None): """Initialize the deployment environment.""" - self.openstack = None super(OpenStackAmuletDeployment, self).__init__(series) + self.openstack = openstack + self.source = source - if openstack: - self.openstack = openstack + def _add_services(self, this_service, other_services): + """Add services to the deployment and set openstack-origin.""" + super(OpenStackAmuletDeployment, self)._add_services(this_service, + other_services) + name = 0 + services = other_services + services.append(this_service) + use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph'] + + if self.openstack: + for svc in services: + if svc[name] not in use_source: + config = {'openstack-origin': self.openstack} + self.d.configure(svc[name], config) + + if self.source: + for svc in services: + if svc[name] in use_source: + config = {'source': self.source} + self.d.configure(svc[name], config) def _configure_services(self, configs): """Configure all of the services.""" for service, config in configs.iteritems(): - if service == self.this_service: - config['openstack-origin'] = self.openstack self.d.configure(service, config) def _get_openstack_release(self): diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index 6515f907..222281e3 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -74,7 +74,7 @@ class OpenStackAmuletUtils(AmuletUtils): if ret: return "unexpected tenant data - {}".format(ret) if not found: - return "tenant {} does not exist".format(e.name) + return "tenant {} does not exist".format(e['name']) return ret def validate_role_data(self, expected, actual): @@ -91,7 +91,7 @@ class OpenStackAmuletUtils(AmuletUtils): if ret: return "unexpected role data - {}".format(ret) if not found: - return "role {} does not exist".format(e.name) + return "role {} does not exist".format(e['name']) return ret def validate_user_data(self, expected, actual): @@ -110,7 +110,7 @@ class OpenStackAmuletUtils(AmuletUtils): if ret: return "unexpected user data - {}".format(ret) if not found: - return "user {} does not exist".format(e.name) + return "user {} does not exist".format(e['name']) return ret def validate_flavor_data(self, expected, actual): @@ -192,8 +192,8 @@ class OpenStackAmuletUtils(AmuletUtils): count = 1 status = instance.status - while status == 'BUILD' and count < 10: - time.sleep(5) + while status != 'ACTIVE' and count < 60: + time.sleep(3) instance = nova.servers.get(instance.id) status = instance.status self.log.debug('instance status: {}'.format(status)) From e686c896aa2dda2e50f520bcfbcfb5e672d4529c Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 2 Jul 2014 10:39:14 +0100 Subject: [PATCH 07/26] Resync helpers --- .../charmhelpers/contrib/openstack/context.py | 8 ++++-- hooks/charmhelpers/contrib/openstack/vip.py | 25 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/vip.py diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index b21fca60..9da9c1ff 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -145,12 +145,16 @@ class SharedDBContext(OSContextGenerator): # with the service units local address and defer execution access_network = relation_get('access-network') if access_network is not None: + if self.relation_prefix is not None: + hostname_key = "{}_hostname".format(self.relation_prefix) + else: + hostname_key = "hostname" access_hostname = get_address_in_network(access_network, unit_get('private-address')) - set_hostname = relation_get(attribute='hostname', + set_hostname = relation_get(attribute=hostname_key, unit=local_unit()) if set_hostname != access_hostname: - relation_set(hostname=access_hostname) + relation_set(relation_settings={hostname_key: access_hostname}) return ctxt # Defer any further hook execution for now.... password_setting = 'password' diff --git a/hooks/charmhelpers/contrib/openstack/vip.py b/hooks/charmhelpers/contrib/openstack/vip.py new file mode 100644 index 00000000..d8c42f90 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/vip.py @@ -0,0 +1,25 @@ + +from netaddr import IPAddress, IPNetwork + +class VIPConfiguration(): + + def __init__(self, configuration): + self.vip = [] + for vip in configuration.split(): + self.vips.append(IPAddress(vip)) + + def getVIP(self, network): + ''' Determine the VIP for the provided network + :network str: CIDR presented network, e.g. 192.168.1.1/24 + :returns str: IP address of VIP in provided network or None + ''' + network = IPNetwork(network) + for vip in self.vips: + if vip in network: + return str(vip) + return None + + def getNIC(self, network): + ''' Determine the physical network interface in use + for the specified network''' + From 571186bfee71eb7a8dd4f5458a4059a05fa16a38 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 2 Jul 2014 14:09:56 +0100 Subject: [PATCH 08/26] Deal with multiple VIP requirements for network splits --- hooks/charmhelpers/contrib/network/ip.py | 47 +++++++++++++++++++++ hooks/charmhelpers/contrib/openstack/vip.py | 25 ----------- hooks/cinder_hooks.py | 29 ++++++++++--- 3 files changed, 71 insertions(+), 30 deletions(-) delete mode 100644 hooks/charmhelpers/contrib/openstack/vip.py diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index f2fa263f..43851e9c 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -93,3 +93,50 @@ def is_address_in_network(network, address): return True else: return False + + +def _get_for_address(address, key): + """Retrieve an attribute of or the physical interface that + the IP address provided could be bound to. + + :param address (str): An individual IPv4 or IPv6 address without a net + mask or subnet prefix. For example, '192.168.1.1'. + :param key: 'iface' for the physical interface name or an attribute + of the configured interface, for example 'netmask'. + :returns str: Requested attribute or None if address is not bindable. + """ + address = netaddr.IPAddress(address) + for iface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(iface) + if netifaces.AF_INET in addresses: + addr = addresses[netifaces.AF_INET][0]['addr'] + netmask = addresses[netifaces.AF_INET][0]['netmask'] + cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + if address in cidr: + if key == 'iface': + return iface + else: + return addresses[netifaces.AF_INET][0][key] + return None + + +def get_iface_for_address(address): + """Determine the physical interface to which an IP address could be bound + + :param address (str): An individual IPv4 or IPv6 address without a net + mask or subnet prefix. For example, '192.168.1.1'. + :returns str: Interface name or None if address is not bindable. + """ + return _get_for_address(address, 'iface') + + +def get_netmask_for_address(address): + """Determine the netmask of the physical interface to which and IP address + could be bound + + :param address (str): An individual IPv4 or IPv6 address without a net + mask or subnet prefix. For example, '192.168.1.1'. + :returns str: Netmask of configured interface or None if address is + not bindable. + """ + return _get_for_address(address, 'netmask') diff --git a/hooks/charmhelpers/contrib/openstack/vip.py b/hooks/charmhelpers/contrib/openstack/vip.py deleted file mode 100644 index d8c42f90..00000000 --- a/hooks/charmhelpers/contrib/openstack/vip.py +++ /dev/null @@ -1,25 +0,0 @@ - -from netaddr import IPAddress, IPNetwork - -class VIPConfiguration(): - - def __init__(self, configuration): - self.vip = [] - for vip in configuration.split(): - self.vips.append(IPAddress(vip)) - - def getVIP(self, network): - ''' Determine the VIP for the provided network - :network str: CIDR presented network, e.g. 192.168.1.1/24 - :returns str: IP address of VIP in provided network or None - ''' - network = IPNetwork(network) - for vip in self.vips: - if vip in network: - return str(vip) - return None - - def getNIC(self, network): - ''' Determine the physical network interface in use - for the specified network''' - diff --git a/hooks/cinder_hooks.py b/hooks/cinder_hooks.py index ef7d6eb3..295e4ade 100755 --- a/hooks/cinder_hooks.py +++ b/hooks/cinder_hooks.py @@ -54,7 +54,11 @@ from charmhelpers.contrib.hahelpers.cluster import ( ) from charmhelpers.payload.execd import execd_preinstall -from charmhelpers.contrib.network.ip import get_address_in_network +from charmhelpers.contrib.network.ip import ( + get_address_in_network, + get_iface_for_address, + get_netmask_for_address +) hooks = Hooks() @@ -258,17 +262,32 @@ def cluster_changed(): @hooks.hook('ha-relation-joined') def ha_joined(): config = get_hacluster_config() + resources = { - 'res_cinder_vip': 'ocf:heartbeat:IPaddr2', 'res_cinder_haproxy': 'lsb:haproxy' } - vip_params = 'params ip="%s" cidr_netmask="%s" nic="%s"' % \ - (config['vip'], config['vip_cidr'], config['vip_iface']) resource_params = { - 'res_cinder_vip': vip_params, 'res_cinder_haproxy': 'op monitor interval="5s"' } + + vip_group = [] + for vip in config['vip'].split(): + iface = get_iface_for_address(vip) + if iface is not None: + vip_key = 'res_cinder_{}_vip'.format(iface) + resources[vip_key] = 'ocf:heartbeat:IPaddr2', + resource_params[vip_key] = ( + 'params ip="{vip}" cidr_netmask="{netmask}"' + ' nic="{iface}"'.format(vip=vip, + iface=iface, + netmask=get_netmask_for_address(vip)) + ) + vip_group.append(vip_key) + + if len(vip_group) > 1: + relation_set(groups={'grp_cinder_vips': ' '.join(vip_group)}) + init_services = { 'res_cinder_haproxy': 'haproxy' } From b1f9283ffa6f665e7710fac588a16d7a714cb214 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 2 Jul 2014 14:22:28 +0100 Subject: [PATCH 09/26] Fixup tuple issue --- hooks/cinder_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/cinder_hooks.py b/hooks/cinder_hooks.py index 295e4ade..0ca94187 100755 --- a/hooks/cinder_hooks.py +++ b/hooks/cinder_hooks.py @@ -276,7 +276,7 @@ def ha_joined(): iface = get_iface_for_address(vip) if iface is not None: vip_key = 'res_cinder_{}_vip'.format(iface) - resources[vip_key] = 'ocf:heartbeat:IPaddr2', + resources[vip_key] = 'ocf:heartbeat:IPaddr2' resource_params[vip_key] = ( 'params ip="{vip}" cidr_netmask="{netmask}"' ' nic="{iface}"'.format(vip=vip, From a39798db9551169df99fd81f35a26f519ec93a0f Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 3 Jul 2014 10:57:07 +0100 Subject: [PATCH 10/26] Use new IP helpers for endpoint registration --- .coverage | Bin 5938 -> 6422 bytes hooks/charmhelpers/contrib/network/ip.py | 6 ++-- .../charmhelpers/contrib/openstack/context.py | 2 +- hooks/cinder_hooks.py | 34 +++++++----------- unit_tests/test_cinder_hooks.py | 1 + unit_tests/test_cluster_hooks.py | 14 +++++--- 6 files changed, 27 insertions(+), 30 deletions(-) diff --git a/.coverage b/.coverage index de1f6d92fb98346873c29d8c0810468d0c44995d..8003cf7829ec7053c4580ec20ef3cd22a159a7f0 100644 GIT binary patch delta 1981 zcmZ{lcW_i?6ve%dXTp-~l5CbVN)iGRT1Wzd^aR9+$73+KDBx!ZvY2QR@=W50WfmJE zI<`;Iv0xi}0};D2>e$e+VHvE59SsVK%2@AvO9uVJ|MtCicfR|b^SkFhRKBKaU5TWv z#lwmC5?dLbZ&yf6PSIZra$CDv5}jij8k*ahI~y9x+n0NpCcZ#F)!L7=C~SAhNx2c@ z^(})Si~^KlEM}oD5ua_Zh}76^y+Ur5@v?1qul(eAOu%GRV}@4L;&9By5de+^aTMxM zkGY6r9*)Ivn2!Zm2!j)_2uqMa6HZ18+OP~BPDKY!!|6B!XW?ulu>$Aee5}NUScQwQ z8W-adT!zbWB`mJO)mVdTa4oLK4N2UHwYUkl;x^olJ8&oN!o9c;58y$p!+Jc7jd%o) z;&D8IC-F3%!6rP5=kPpUz>9bZFXI(##%p*3Z{lri!8>>t?_n!G#7FoT+wd7a#~1h# zU*T(fgKx0|-{E_7;|KhRUD%D}9{hq|u@`^fFCHKip({L4*jpG9_7P?XGlgMcUtyLo zTNo9_gt@{3VWDt*(~!h#D`)$DlVt=z|PoDi9H5A&MO2qCbi;6y+F)8nt*aTCrU1x(e6f zX4QU^s=HZr-Gc4^qvFm$!=Lao_TV=-|zk@JL5gv{a7>QA+Kqaaa=y5n0 z0lao+TK_62u5%T}4h3)}HYgjfC&HZ;ZdinQvj z7UZXDmn7O-mMw2hv;}mOnRp`bq$o45aJcbC_^3GOg##6oJe6FfPQ>fAq_*F`r5b6x zQU2YJ@mU+cw4<|ev48$(`3hpTZ_n24F0E%llk&1i>bE2H&#i9&ab z@ha@v@Y2Fc<5iXE;jSK^ro4ugf+46##82>TscLTb&lqdGaekL+3bT)LUt|X`#}wy0 z2ez%)(5 zRJL_#vMmoX?Hrt|spd!0qj*dM>8&76y8V<$MTFv?N{MJNnkJae3xq3$tAuw6?-t%G zNp2FpE_|bhiuZ&c^q3wv&fUVFg?og*2!9ps75*;#L-?ogFX7+91042p=sHYu*vDbS zVUEK*hlLIYJFIkA?eG|fOB^O0u5!5Ap>_DQ!%YseUUc}9!rsk+548y(Ta+{di=Fx(11oPEyXG7NtbSu z8+1vm$5VJU73jtwUrmcv!y3>p|-QE&ya;uG3W5YQu?hND2v}flO7i>`fUcl=b zYFn|}kNb2#))T@0tosk-hZ=8|4|yuJS|jQ9iHvaiVaBV~w{}r(vwL{Y+KuJK`ph=o K5&EI4tNb5`YG^0` delta 1786 zcmY+Fd2mf<7{=#4Ik`8>O>XuTSp*5$g&-mcBI5BjX}DohN1Ad;ipUAk-JLF~Bdy{q z-8UVK7DY={RhL2AQHrX)wY1gRDyUY5PWztQQ2p=zalU(=?^%9lN$&d0wMk@KfaPx2 zL}f;mt0qdJ8u?YGGHo_zUGs$chFR{G=Bb%Y510;Br%jFuGR)vq5P}ekRP;tED%`H> zjw2zO(@>$l)ny@u8EQS{2Y&=05Ds}h7*2#D91(~{CnTXevd{;EF%}chhzBtbi}5nv z#Cm*+udow)v0tt6jZMCY%eaEyaTR}{oqdR%#EHb+iF1ex$n7eZw->0{VK$ZEYTk#C$<5?`gLM(#95!*QI%Dg21j_yuQh7Uysto(uR57tw}GUjHoj z{=_w0#|`|2zgZ*JiG7J}#CGBU;y_{taS(AZaR_lJaX4`VaTIYhaSU-RaU5|xaTjj@ zNyN#-U5QhOyLt2IL7Yb1lQ@GovywQQxHoYgaUbHo#QDVii2GZi@gU-%#HGZ;h|7q} ziANBRBxYhcSVcUFxSDtj@mS&-VuReO*yh$F3^P)#v^5QhGR){y=&-@w(K3fLD-Q9} ztsC}Py7S%DdL}q ze;|fHxkY5T zDt2~`>u#7mtSxHrfv+qvT0F@{j@wn`UF}w2v zgvm01>FQEsl}c~~hEJX8Zf>lbDO8wYT|C_|Gt~BwNqVMXW~r>uNOjq6X-TAPRUB5S z=NM+LdM<2_r?+9|{U^dMGD0P$6bUE85+k7$$T+RRHA)aOFbfakk&daoF8i_8vkY50 zdT*8a8rpH4i-?Pf2NTy5HxN%DZY6%2cs}t`;>HWF_l-a@>KcsH?_f6MuU zUh@wVAH8Y*apDuiC%q}$ia?xiBfd)9PJBbgqj9*#$r|Ts42?%?T&;1f#!qQHPveCe zFVa|PyjbH+8gJG$w`-g?ktPyMh1t`x8k#Hv`xs_l%NHN1vIR>El9A#}8Zrfs*>2ZZ zZ%9I8&+UcFH_U!F?bKxf7GXog(kP*FoOc0TycoC&pR?+#y`sFEqPifyvLJkK1K^ zh^$YNE%NrO7a3-;Rq+3)KilE2ml)>Y_gWq8r1nNx1$6vxpF`A|nDGHa4YO2EwzOpa E1LtfsUjP6A diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 43851e9c..38b7cb37 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -98,7 +98,7 @@ def is_address_in_network(network, address): def _get_for_address(address, key): """Retrieve an attribute of or the physical interface that the IP address provided could be bound to. - + :param address (str): An individual IPv4 or IPv6 address without a net mask or subnet prefix. For example, '192.168.1.1'. :param key: 'iface' for the physical interface name or an attribute @@ -125,7 +125,7 @@ def get_iface_for_address(address): :param address (str): An individual IPv4 or IPv6 address without a net mask or subnet prefix. For example, '192.168.1.1'. - :returns str: Interface name or None if address is not bindable. + :returns str: Interface name or None if address is not bindable. """ return _get_for_address(address, 'iface') @@ -137,6 +137,6 @@ def get_netmask_for_address(address): :param address (str): An individual IPv4 or IPv6 address without a net mask or subnet prefix. For example, '192.168.1.1'. :returns str: Netmask of configured interface or None if address is - not bindable. + not bindable. """ return _get_for_address(address, 'netmask') diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 9da9c1ff..af6a38dd 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -148,7 +148,7 @@ class SharedDBContext(OSContextGenerator): if self.relation_prefix is not None: hostname_key = "{}_hostname".format(self.relation_prefix) else: - hostname_key = "hostname" + hostname_key = "hostname" access_hostname = get_address_in_network(access_network, unit_get('private-address')) set_hostname = relation_get(attribute=hostname_key, diff --git a/hooks/cinder_hooks.py b/hooks/cinder_hooks.py index 0ca94187..6b25bbb2 100755 --- a/hooks/cinder_hooks.py +++ b/hooks/cinder_hooks.py @@ -35,7 +35,6 @@ from charmhelpers.core.hookenv import ( unit_get, log, ERROR, - local_unit, ) from charmhelpers.fetch import apt_install, apt_update @@ -47,7 +46,6 @@ from charmhelpers.contrib.openstack.utils import ( from charmhelpers.contrib.storage.linux.ceph import ensure_ceph_keyring from charmhelpers.contrib.hahelpers.cluster import ( - canonical_url, eligible_leader, is_leader, get_hacluster_config, @@ -55,9 +53,12 @@ from charmhelpers.contrib.hahelpers.cluster import ( from charmhelpers.payload.execd import execd_preinstall from charmhelpers.contrib.network.ip import ( - get_address_in_network, get_iface_for_address, - get_netmask_for_address + get_netmask_for_address, +) +from charmhelpers.contrib.openstack.ip import ( + canonical_url, + PUBLIC, INTERNAL, ADMIN ) hooks = Hooks() @@ -181,31 +182,20 @@ def identity_joined(rid=None): if not eligible_leader(CLUSTER_RES): return - conf = config() - port = conf['api-listening-port'] public_url = '{}:{}/v1/$(tenant_id)s'.format( - canonical_url( - CONFIGS, - address=get_address_in_network(conf.get('os-public-network'), - unit_get('public-address'))), - port + canonical_url(CONFIGS, PUBLIC), + config('api-listening-port') ) internal_url = '{}:{}/v1/$(tenant_id)s'.format( - canonical_url( - CONFIGS, - address=get_address_in_network(conf.get('os-internal-network'), - unit_get('private-address'))), - port + canonical_url(CONFIGS, INTERNAL), + config('api-listening-port') ) admin_url = '{}:{}/v1/$(tenant_id)s'.format( - canonical_url( - CONFIGS, - address=get_address_in_network(conf.get('os-admin-network'), - unit_get('private-address'))), - port + canonical_url(CONFIGS, ADMIN), + config('api-listening-port') ) settings = { - 'region': conf['region'], + 'region': config('region'), 'service': 'cinder', 'public_url': public_url, 'internal_url': internal_url, diff --git a/unit_tests/test_cinder_hooks.py b/unit_tests/test_cinder_hooks.py index a2c9b323..d1a9a41a 100644 --- a/unit_tests/test_cinder_hooks.py +++ b/unit_tests/test_cinder_hooks.py @@ -300,6 +300,7 @@ class TestJoinedHooks(CharmTestCase): def test_identity_service_joined(self): 'It properly requests unclustered endpoint via identity-service' self.unit_get.return_value = 'cindernode1' + self.config.side_effect = self.test_config.get self.canonical_url.return_value = 'http://cindernode1' hooks.hooks.execute(['hooks/identity-service-relation-joined']) expected = { diff --git a/unit_tests/test_cluster_hooks.py b/unit_tests/test_cluster_hooks.py index aa7151b9..da978e07 100644 --- a/unit_tests/test_cluster_hooks.py +++ b/unit_tests/test_cluster_hooks.py @@ -51,7 +51,10 @@ TO_PATCH = [ # charmhelpers.contrib.hahelpers.cluster_utils 'eligible_leader', 'get_hacluster_config', - 'is_leader' + 'is_leader', + # charmhelpers.contrib.network.ip + 'get_iface_for_address', + 'get_netmask_for_address' ] @@ -96,19 +99,22 @@ class TestClusterHooks(CharmTestCase): 'vip_cidr': '19', } self.get_hacluster_config.return_value = conf + self.get_iface_for_address.return_value = 'eth101' + self.get_netmask_for_address.return_value = '255.255.224.0' hooks.hooks.execute(['hooks/ha-relation-joined']) ex_args = { 'corosync_mcastport': '37373', 'init_services': {'res_cinder_haproxy': 'haproxy'}, 'resource_params': { - 'res_cinder_vip': - 'params ip="192.168.25.163" cidr_netmask="19" nic="eth101"', + 'res_cinder_eth101_vip': + 'params ip="192.168.25.163" cidr_netmask="255.255.224.0"' + ' nic="eth101"', 'res_cinder_haproxy': 'op monitor interval="5s"' }, 'corosync_bindiface': 'eth100', 'clones': {'cl_cinder_haproxy': 'res_cinder_haproxy'}, 'resources': { - 'res_cinder_vip': 'ocf:heartbeat:IPaddr2', + 'res_cinder_eth101_vip': 'ocf:heartbeat:IPaddr2', 'res_cinder_haproxy': 'lsb:haproxy' } } From 4d1f199d9efa9f84e07ead58c52fec70fdb71109 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 3 Jul 2014 10:57:23 +0100 Subject: [PATCH 11/26] Drop coverage db --- .coverage | Bin 6422 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .coverage diff --git a/.coverage b/.coverage deleted file mode 100644 index 8003cf7829ec7053c4580ec20ef3cd22a159a7f0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6422 zcmcJTd3aRi6~;Xm2n0gHCK&deuw@8a*!RWodPISXpj_i*a>ERn%*>r7BvQZ)q_jS2 zMN1Kjwbg3r<^xu-rL?tflxi)aMXR=2v@T$!uB~Fv_uaXf;j#WBm%n)63HQtS&U@bX zcfPBP^$wMpg`s#fYKPMCq+^&Rp?IU646e5OH;yZ>EU$1{nQfwxn4NN3uXoy%nYDp% zyul8v2{zcNg#1__l?(+!;b5{M6-e53cG8Z8?9nkhy*8e#4}>DIT01HKC0?I$e=2N8 z6LwPm5|5>mk(xj_=>1D|btD!^S67!Onwb~=!MXpv=J z>eG33b~+Re1e2j~q|weKqv~QiTD~_hB`Q^_6G>*80@20>bxyJ6l$3$c5v3@@ILyal z+pIc+!nA6uwQAM(J6TR=C6Lx4k+vvAPdQ~cMxq>JFcFoQidmQ=r!K_#sKO!u7l3i0 zme5j|SdL4u0xNMTE`x>3aRq8&BaAhOB8E5|B#}ZRns6n)hV|Hpt8gu@!)9EME%-XN z;v4uTZp2Oa7H+{d+=|=qZEVNw*oiwtq`PnzzK^?c4}OSyu^T_ePw_MS9QR`{9>6c~ zAok&5{1W@|YaGCD@jE<%NAVaQ#}jxG2XP2b<4^cA4&zxI!E-o@7x7oTgumhMcn$x+ zF}#j9@K3ynx9~RJ!EwBcf8jlx#0U5gAK?>xic@?JF%h>WZbRITxRAI#aR=fe;*P{6 z#GQyc6L%r*M%E-l5icQLN^BA@CtgW>8SyIO zYT_%1gTx`?I0Qr^mvYg&3Ji|Vh=bh!I zP5y~}ET^wJ(Qh0&{lt)+2rE7Ivz-2F20Z8JE}k(Qqs28UF-6qH5-Y2b#?2B5yCe$s zYTr15SH!AMDnAyl6su$uw@LC&rQJ43x$j82xoLGs66qLDNYZ?kO%w?XHxH%| zPt}e;UmN|!+Sx7QF!5UA>xs7z-%flN@qNU5h#w-}NBl7He&T0{4--F2e1!OU;-kbb z62CRT8scvPp=_wHw12V~%OZ)41g;h`xX1Qg{kh@JqGO9c4zYZ|tCp_~wVYu}G)5M%6odB` zj~ysN86|!?S$uTfm%!CBfvc4wAPdzZqM+nwNYJWIreTf9DW<_Hjf}w6TEVMk0jzZz zu{LPPx*FGL(Aty*ErnS(XwbS@P|F3a?IOY*GF1w#?$dDf6M?HeGGqJjD~(jY5u|zq zzZax(f$9+cD2c5=>UsP{oc#sCsh3*->bOp7g-ss|IQ>Tn8mI8NN^X~(3W(ckcIv7z zsjo*z13fg7Xdn9j(5S*gqw&NOG$2hPmf>&#N$AK$BbkyGD3V!`yjkXP(F)B)wZyi< zAzrO#C_=o3xSlvl93zesCy0~889iQ2#OsJR5N{;DhWG~JTXl48C*DE4llTteUBq`1 ze~Q4llqw!#Ez;3cbwdx8g zEN85@0v3oBcN7yIj4`%(xlbp0?;$z;MiT1aah5Y)l~|w=xELjQN!kPFWubI{5|-a^ zmwm)3-%zn35N}NBKl4R5!7?v6gX-NSTh(nhrFA z(VGj~3VdAPc5!=(Y~a(g8}~(`#AVr?+uOHu(V+J3I#Fsp8vM`hT40X{ry2aB-{HldiOrAcIqO%P|KQEqIc>p z9`95WZzA5TdB^qG-NXlopCW$R!<^@cU+^gAghCiTN&G(X2gDx|e?iv%S_b|_L=Bs<~`(4JHvHY_Z hSkA(;ap(D|lYid%{=HRu>7M`jD$7|UU1 From 339362199e85c54b3652f4ef45cbae5b4471873e Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 3 Jul 2014 10:58:58 +0100 Subject: [PATCH 12/26] Add missing helper --- hooks/charmhelpers/contrib/openstack/ip.py | 71 ++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 hooks/charmhelpers/contrib/openstack/ip.py diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py new file mode 100644 index 00000000..b3fc0c43 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -0,0 +1,71 @@ +from charmhelpers.core.hookenv import ( + config, + unit_get, +) + +from charmhelpers.contrib.network.ip import ( + get_address_in_network, + is_address_in_network, +) + +from charmhelpers.contrib.hahelpers.cluster import is_clustered + +PUBLIC = 'public' +INTERNAL = 'int' +ADMIN = 'admin' + +_address_map = { + PUBLIC: { + 'config': 'os-public-network', + 'fallback': 'public-address' + }, + INTERNAL: { + 'config': 'os-internal-network', + 'fallback': 'private-address' + }, + ADMIN: { + 'config': 'os-admin-network', + 'fallback': 'private-address' + } +} + + +def canonical_url(configs, endpoint_type=PUBLIC): + ''' + Returns the correct HTTP URL to this host given the state of HTTPS + configuration, hacluster and charm configuration. + + :configs OSTemplateRenderer: A config tempating object to inspect for + a complete https context. + :endpoint_type str: The endpoint type to resolve. + + :returns str: Base URL for services on the current service unit. + ''' + scheme = 'http' + if 'https' in configs.complete_contexts(): + scheme = 'https' + return '%s://%s' % (scheme, resolve_address(endpoint_type)) + + +def resolve_address(endpoint_type=PUBLIC): + resolved_address = None + if is_clustered(): + if config(_address_map[endpoint_type]['config']) is None: + # Assume vip is simple and pass back directly + resolved_address = config('vip') + else: + for vip in config('vip').split(): + if is_address_in_network( + config(_address_map[endpoint_type]['config']), + vip): + resolved_address = vip + else: + resolved_address = get_address_in_network( + config(_address_map[endpoint_type]['config']), + unit_get(_address_map[endpoint_type]['fallback']) + ) + if resolved_address is None: + raise ValueError('Unable to resolve a suitable IP address' + ' based on charm state and configuration') + else: + return resolved_address From 9e1f8cb6182fd2194212e84ad3de80a528a2895b Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 3 Jul 2014 12:28:47 +0100 Subject: [PATCH 13/26] Resync helpers for IPv6 haproxy --- .../charmhelpers/contrib/openstack/templates/haproxy.cfg | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index 56ed913e..0eb56617 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -27,7 +27,13 @@ listen stats :8888 {% if units -%} {% for service, ports in service_ports.iteritems() -%} -listen {{ service }} 0.0.0.0:{{ ports[0] }} +listen {{ service }}_ipv4 0.0.0.0:{{ ports[0] }} + balance roundrobin + {% for unit, address in units.iteritems() -%} + server {{ unit }} {{ address }}:{{ ports[1] }} check + {% endfor %} + +listen {{ service }}_ipv6 :::{{ ports[0] }} balance roundrobin {% for unit, address in units.iteritems() -%} server {{ unit }} {{ address }}:{{ ports[1] }} check From 3797f4103b71e5562060c0dd2a7a71cb2597b835 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 3 Jul 2014 13:56:00 +0100 Subject: [PATCH 14/26] Use internal network if provided for loadbalancing to other services --- .../contrib/openstack/templates/haproxy.cfg | 1 - hooks/cinder_hooks.py | 12 ++++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index 0eb56617..a95eddd1 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -32,7 +32,6 @@ listen {{ service }}_ipv4 0.0.0.0:{{ ports[0] }} {% for unit, address in units.iteritems() -%} server {{ unit }} {{ address }}:{{ ports[1] }} check {% endfor %} - listen {{ service }}_ipv6 :::{{ ports[0] }} balance roundrobin {% for unit, address in units.iteritems() -%} diff --git a/hooks/cinder_hooks.py b/hooks/cinder_hooks.py index 6b25bbb2..33a1102f 100755 --- a/hooks/cinder_hooks.py +++ b/hooks/cinder_hooks.py @@ -55,6 +55,7 @@ from charmhelpers.payload.execd import execd_preinstall from charmhelpers.contrib.network.ip import ( get_iface_for_address, get_netmask_for_address, + get_address_in_network ) from charmhelpers.contrib.openstack.ip import ( canonical_url, @@ -100,6 +101,9 @@ def config_changed(): CONFIGS.write_all() configure_https() + for rid in relation_ids('cluster'): + cluster_joined(relation_id=rid) + @hooks.hook('shared-db-relation-joined') def db_joined(): @@ -242,6 +246,14 @@ def ceph_changed(): replicas=_config['ceph-osd-replication-count']) +@hooks.hook('cluster-relation-joined') +def cluster_joined(relation_id=None): + address = get_address_in_network(config('os-internal-network'), + unit_get('private-addresss')) + relation_set(relation_id=relation_id, + relation_settings={'private-addresss': address}) + + @hooks.hook('cluster-relation-changed', 'cluster-relation-departed') @restart_on_change(restart_map(), stopstart=True) From 4bb6d6a6abc6d9c9029733b37d37236707b25272 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 3 Jul 2014 13:57:03 +0100 Subject: [PATCH 15/26] Spell address correctly --- hooks/cinder_hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/cinder_hooks.py b/hooks/cinder_hooks.py index 33a1102f..7d56f47c 100755 --- a/hooks/cinder_hooks.py +++ b/hooks/cinder_hooks.py @@ -249,9 +249,9 @@ def ceph_changed(): @hooks.hook('cluster-relation-joined') def cluster_joined(relation_id=None): address = get_address_in_network(config('os-internal-network'), - unit_get('private-addresss')) + unit_get('private-address')) relation_set(relation_id=relation_id, - relation_settings={'private-addresss': address}) + relation_settings={'private-address': address}) @hooks.hook('cluster-relation-changed', From 9444b47608f23e807eb4d6c40f9b1769b5e92f3f Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 3 Jul 2014 14:01:20 +0100 Subject: [PATCH 16/26] Resync helpers --- hooks/charmhelpers/contrib/openstack/context.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index af6a38dd..1e8cfc72 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -400,7 +400,9 @@ class HAProxyContext(OSContextGenerator): cluster_hosts = {} l_unit = local_unit().replace('/', '-') - cluster_hosts[l_unit] = unit_get('private-address') + cluster_hosts[l_unit] = \ + get_address_in_network(config('os-internal-network'), + unit_get('private-address')) for rid in relation_ids('cluster'): for unit in related_units(rid): From c2a6421803ea1c31bcd87d14804e21109cd94b3b Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 3 Jul 2014 15:28:27 +0100 Subject: [PATCH 17/26] Resync helpers --- hooks/charmhelpers/contrib/network/ip.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 38b7cb37..b665e6ce 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -28,7 +28,7 @@ def _validate_cidr(network): def get_address_in_network(network, fallback=None, fatal=False): """ - Get an IPv4 address within the network from the host. + Get an IPv4 or IPv6 address within the network from the host. :param network (str): CIDR presentation format. For example, '192.168.1.0/24'. @@ -51,13 +51,20 @@ def get_address_in_network(network, fallback=None, fatal=False): not_found_error_out() _validate_cidr(network) + network = netaddr.IPNetwork(network) for iface in netifaces.interfaces(): addresses = netifaces.ifaddresses(iface) - if netifaces.AF_INET in addresses: + if network.version == 4 and netifaces.AF_INET in addresses: addr = addresses[netifaces.AF_INET][0]['addr'] netmask = addresses[netifaces.AF_INET][0]['netmask'] cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) - if cidr in netaddr.IPNetwork(network): + if cidr in network: + return str(cidr.ip) + if network.version == 6 and netifaces.AF_INET6 in addresses: + addr = addresses[netifaces.AF_INET6][0]['addr'] + netmask = addresses[netifaces.AF_INET6][0]['netmask'] + cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + if cidr in network: return str(cidr.ip) if fallback is not None: From b43a068310e30b4d66634659d6e541ba65dc3caa Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 3 Jul 2014 15:32:05 +0100 Subject: [PATCH 18/26] Fixup handling --- hooks/charmhelpers/contrib/network/ip.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index b665e6ce..e14747d6 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -61,11 +61,12 @@ def get_address_in_network(network, fallback=None, fatal=False): if cidr in network: return str(cidr.ip) if network.version == 6 and netifaces.AF_INET6 in addresses: - addr = addresses[netifaces.AF_INET6][0]['addr'] - netmask = addresses[netifaces.AF_INET6][0]['netmask'] - cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) - if cidr in network: - return str(cidr.ip) + for addr in addresses[netifaces.AF_INET6]: + if 'fe80' not in addr['addr']: + netmask = addr['netmask'] + cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + if cidr in network: + return str(cidr.ip) if fallback is not None: return fallback From e3404b1efed0f5763f66dc24172aac8f4b65a895 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 3 Jul 2014 15:38:06 +0100 Subject: [PATCH 19/26] Resync helper --- hooks/charmhelpers/contrib/network/ip.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index e14747d6..ab283808 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -64,9 +64,10 @@ def get_address_in_network(network, fallback=None, fatal=False): for addr in addresses[netifaces.AF_INET6]: if 'fe80' not in addr['addr']: netmask = addr['netmask'] - cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + cidr = netaddr.IPAddress("%s/%s" % (addr['addr'], + netmask)) if cidr in network: - return str(cidr.ip) + return "[{}]".format(cidr.ip) if fallback is not None: return fallback From 84b598a70da2df511bd6121173699a1509ed15a4 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 3 Jul 2014 15:39:34 +0100 Subject: [PATCH 20/26] Resync helpers --- hooks/charmhelpers/contrib/network/ip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index ab283808..2b917bad 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -64,7 +64,7 @@ def get_address_in_network(network, fallback=None, fatal=False): for addr in addresses[netifaces.AF_INET6]: if 'fe80' not in addr['addr']: netmask = addr['netmask'] - cidr = netaddr.IPAddress("%s/%s" % (addr['addr'], + cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], netmask)) if cidr in network: return "[{}]".format(cidr.ip) From 896c1cc04fb1da3294cc3813c798de5e28ea6ede Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 3 Jul 2014 16:16:16 +0100 Subject: [PATCH 21/26] Only use [] formatting for URL's --- hooks/charmhelpers/contrib/network/ip.py | 12 +++++++++++- hooks/charmhelpers/contrib/openstack/ip.py | 6 +++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 2b917bad..b530fc42 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -67,7 +67,7 @@ def get_address_in_network(network, fallback=None, fatal=False): cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], netmask)) if cidr in network: - return "[{}]".format(cidr.ip) + return str(cidr.ip) if fallback is not None: return fallback @@ -78,6 +78,16 @@ def get_address_in_network(network, fallback=None, fatal=False): return None +def is_ipv6(address): + '''Determine whether provided address is IPv6 or not''' + try: + address = netaddr.IPAddress(address) + except netaddr.AddrFormatError: + return False + else: + return address.version == 6 + + def is_address_in_network(network, address): """ Determine whether the provided address is within a network range. diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py index b3fc0c43..7e7a536f 100644 --- a/hooks/charmhelpers/contrib/openstack/ip.py +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -6,6 +6,7 @@ from charmhelpers.core.hookenv import ( from charmhelpers.contrib.network.ip import ( get_address_in_network, is_address_in_network, + is_ipv6, ) from charmhelpers.contrib.hahelpers.cluster import is_clustered @@ -44,7 +45,10 @@ def canonical_url(configs, endpoint_type=PUBLIC): scheme = 'http' if 'https' in configs.complete_contexts(): scheme = 'https' - return '%s://%s' % (scheme, resolve_address(endpoint_type)) + address = resolve_address(endpoint_type) + if is_ipv6(address): + address = "[{}]".format(address) + return '%s://%s' % (scheme, address) def resolve_address(endpoint_type=PUBLIC): From d799047b07c4b1a463e38f7b054365baa9d258f2 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 4 Jul 2014 10:26:47 +0100 Subject: [PATCH 22/26] Use vip group to determine cluster leadership --- hooks/cinder_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index 378287fc..42b1345c 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -86,7 +86,7 @@ SCHEDULER_PACKAGES = ['cinder-scheduler'] DEFAULT_LOOPBACK_SIZE = '5G' # Cluster resource used to determine leadership when hacluster'd -CLUSTER_RES = 'res_cinder_vip' +CLUSTER_RES = 'grp_cinder_vips' class CinderCharmError(Exception): From bb8dd0e7434801af9d50735161c81cee15b1bd5a Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 4 Jul 2014 11:37:47 +0100 Subject: [PATCH 23/26] Resync helpers --- hooks/charmhelpers/contrib/network/ip.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index b530fc42..e0f9eb66 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -83,10 +83,11 @@ def is_ipv6(address): try: address = netaddr.IPAddress(address) except netaddr.AddrFormatError: + # probably a hostname - so not an address at all! return False else: return address.version == 6 - + def is_address_in_network(network, address): """ @@ -127,7 +128,7 @@ def _get_for_address(address, key): address = netaddr.IPAddress(address) for iface in netifaces.interfaces(): addresses = netifaces.ifaddresses(iface) - if netifaces.AF_INET in addresses: + if address.version == 4 and netifaces.AF_INET in addresses: addr = addresses[netifaces.AF_INET][0]['addr'] netmask = addresses[netifaces.AF_INET][0]['netmask'] cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) @@ -136,6 +137,16 @@ def _get_for_address(address, key): return iface else: return addresses[netifaces.AF_INET][0][key] + if address.version == 6 and netifaces.AF_INET6 in addresses: + for addr in addresses[netifaces.AF_INET6]: + if 'fe80' not in addr['addr']: + cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], + addr['netmask'])) + if address in cidr: + if key == 'iface': + return iface + else: + return addr[key] return None From 1d674b74caf184a7c83721dde5eeb2ba4cfd536d Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 15 Jul 2014 16:40:45 +0100 Subject: [PATCH 24/26] Tidy lint --- hooks/charmhelpers/contrib/openstack/context.py | 13 +++++++++---- hooks/cinder_hooks.py | 12 ++++++------ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 1e8cfc72..92c41b23 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -25,6 +25,7 @@ from charmhelpers.core.hookenv import ( unit_get, unit_private_ip, ERROR, + INFO ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -714,7 +715,7 @@ class SubordinateConfigContext(OSContextGenerator): self.interface = interface def __call__(self): - ctxt = {} + ctxt = {'sections': {}} for rid in relation_ids(self.interface): for unit in related_units(rid): sub_config = relation_get('subordinate_configuration', @@ -740,10 +741,14 @@ class SubordinateConfigContext(OSContextGenerator): sub_config = sub_config[self.config_file] for k, v in sub_config.iteritems(): - ctxt[k] = v + if k == 'sections': + for section, config_dict in v.iteritems(): + log("adding section '%s'" % (section)) + ctxt[k][section] = config_dict + else: + ctxt[k] = v - if not ctxt: - ctxt['sections'] = {} + log("%d section(s) found" % (len(ctxt['sections'])), level=INFO) return ctxt diff --git a/hooks/cinder_hooks.py b/hooks/cinder_hooks.py index 7d56f47c..7a30d4f4 100755 --- a/hooks/cinder_hooks.py +++ b/hooks/cinder_hooks.py @@ -102,7 +102,7 @@ def config_changed(): configure_https() for rid in relation_ids('cluster'): - cluster_joined(relation_id=rid) + cluster_joined(relation_id=rid) @hooks.hook('shared-db-relation-joined') @@ -272,7 +272,7 @@ def ha_joined(): resource_params = { 'res_cinder_haproxy': 'op monitor interval="5s"' } - + vip_group = [] for vip in config['vip'].split(): iface = get_iface_for_address(vip) @@ -280,10 +280,10 @@ def ha_joined(): vip_key = 'res_cinder_{}_vip'.format(iface) resources[vip_key] = 'ocf:heartbeat:IPaddr2' resource_params[vip_key] = ( - 'params ip="{vip}" cidr_netmask="{netmask}"' - ' nic="{iface}"'.format(vip=vip, - iface=iface, - netmask=get_netmask_for_address(vip)) + 'params ip="{vip}" cidr_netmask="{netmask}"' + ' nic="{iface}"'.format(vip=vip, + iface=iface, + netmask=get_netmask_for_address(vip)) ) vip_group.append(vip_key) From 2c68c2e81a185fad6755f18e702f78433fab9313 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 16 Jul 2014 14:35:25 +0100 Subject: [PATCH 25/26] Resync helpers, drop surplus vip config --- config.yaml | 14 +++++--------- hooks/charmhelpers/contrib/hahelpers/cluster.py | 4 ++-- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/config.yaml b/config.yaml index d21d24f9..c816a216 100644 --- a/config.yaml +++ b/config.yaml @@ -102,15 +102,11 @@ options: # HA configuration settings vip: type: string - description: "Virtual IP to use to front cinder API in ha configuration" - vip_iface: - type: string - default: eth0 - description: "Network Interface where to place the Virtual IP" - vip_cidr: - type: int - default: 24 - description: "Netmask that will be used for the Virtual IP" + description: | + Virtual IP(s) to use to front API services in HA configuration. + . + If multiple networks are being used, a VIP should be provided for each + network, separated by spaces. ha-bindiface: type: string default: eth0 diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index dd89f347..5e9ff01e 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -146,12 +146,12 @@ def get_hacluster_config(): Obtains all relevant configuration from charm configuration required for initiating a relation to hacluster: - ha-bindiface, ha-mcastport, vip, vip_iface, vip_cidr + ha-bindiface, ha-mcastport, vip returns: dict: A dict containing settings keyed by setting name. raises: HAIncompleteConfig if settings are missing. ''' - settings = ['ha-bindiface', 'ha-mcastport', 'vip', 'vip_iface', 'vip_cidr'] + settings = ['ha-bindiface', 'ha-mcastport', 'vip'] conf = {} for setting in settings: conf[setting] = config_get(setting) From b138b92047710e560cc21545f3306fb311841720 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 24 Jul 2014 11:25:17 +0100 Subject: [PATCH 26/26] Resync to trunk helpers --- charm-helpers.yaml | 2 +- .../charmhelpers/contrib/hahelpers/cluster.py | 4 +-- hooks/charmhelpers/contrib/network/ip.py | 30 +++++-------------- hooks/charmhelpers/core/host.py | 4 +++ 4 files changed, 14 insertions(+), 26 deletions(-) diff --git a/charm-helpers.yaml b/charm-helpers.yaml index de3d3853..e64ed4ed 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -1,4 +1,4 @@ -branch: lp:~james-page/charm-helpers/network-splits +branch: lp:charm-helpers destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 5e9ff01e..505de6b2 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -163,7 +163,7 @@ def get_hacluster_config(): return conf -def canonical_url(configs, vip_setting='vip', address=None): +def canonical_url(configs, vip_setting='vip'): ''' Returns the correct HTTP URL to this host given the state of HTTPS configuration and hacluster. @@ -180,5 +180,5 @@ def canonical_url(configs, vip_setting='vip', address=None): if is_clustered(): addr = config_get(vip_setting) else: - addr = address or unit_get('private-address') + addr = unit_get('private-address') return '%s://%s' % (scheme, addr) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index e0f9eb66..0972e91a 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -1,5 +1,7 @@ import sys +from functools import partial + from charmhelpers.fetch import apt_install from charmhelpers.core.hookenv import ( ERROR, log, @@ -62,10 +64,9 @@ def get_address_in_network(network, fallback=None, fatal=False): return str(cidr.ip) if network.version == 6 and netifaces.AF_INET6 in addresses: for addr in addresses[netifaces.AF_INET6]: - if 'fe80' not in addr['addr']: - netmask = addr['netmask'] + if not addr['addr'].startswith('fe80'): cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], - netmask)) + addr['netmask'])) if cidr in network: return str(cidr.ip) @@ -139,7 +140,7 @@ def _get_for_address(address, key): return addresses[netifaces.AF_INET][0][key] if address.version == 6 and netifaces.AF_INET6 in addresses: for addr in addresses[netifaces.AF_INET6]: - if 'fe80' not in addr['addr']: + if not addr['addr'].startswith('fe80'): cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], addr['netmask'])) if address in cidr: @@ -150,23 +151,6 @@ def _get_for_address(address, key): return None -def get_iface_for_address(address): - """Determine the physical interface to which an IP address could be bound +get_iface_for_address = partial(_get_for_address, key='iface') - :param address (str): An individual IPv4 or IPv6 address without a net - mask or subnet prefix. For example, '192.168.1.1'. - :returns str: Interface name or None if address is not bindable. - """ - return _get_for_address(address, 'iface') - - -def get_netmask_for_address(address): - """Determine the netmask of the physical interface to which and IP address - could be bound - - :param address (str): An individual IPv4 or IPv6 address without a net - mask or subnet prefix. For example, '192.168.1.1'. - :returns str: Netmask of configured interface or None if address is - not bindable. - """ - return _get_for_address(address, 'netmask') +get_netmask_for_address = partial(_get_for_address, key='netmask') diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 8b617a42..d934f940 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -322,6 +322,10 @@ def cmp_pkgrevno(package, revno, pkgcache=None): import apt_pkg if not pkgcache: apt_pkg.init() + # Force Apt to build its cache in memory. That way we avoid race + # conditions with other applications building the cache in the same + # place. + apt_pkg.config.set("Dir::Cache::pkgcache", "") pkgcache = apt_pkg.Cache() pkg = pkgcache[package] return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)