From 5d952bcb110de8e100645c1a4cb7d5b373b7c0c5 Mon Sep 17 00:00:00 2001 From: Hui Xiang Date: Fri, 19 Sep 2014 20:01:10 +0800 Subject: [PATCH] Sync with lp:~xianghui/charm-helpers/format-ipv6 --- hooks/charmhelpers/contrib/network/ip.py | 110 +++++++++++++++--- .../charmhelpers/contrib/openstack/context.py | 26 +++-- .../contrib/peerstorage/__init__.py | 100 +++++++++++----- hooks/charmhelpers/core/hookenv.py | 11 ++ hooks/charmhelpers/fetch/archiveurl.py | 40 +++++++ 5 files changed, 240 insertions(+), 47 deletions(-) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 7edbcc4..b859a09 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -1,10 +1,11 @@ +import glob import sys from functools import partial from charmhelpers.fetch import apt_install from charmhelpers.core.hookenv import ( - ERROR, log, config, + ERROR, log, ) try: @@ -156,19 +157,102 @@ get_iface_for_address = partial(_get_for_address, key='iface') get_netmask_for_address = partial(_get_for_address, key='netmask') -def get_ipv6_addr(iface="eth0"): +def format_ipv6_addr(address): + """ + IPv6 needs to be wrapped with [] in url link to parse correctly. + """ + if is_ipv6(address): + address = "[%s]" % address + else: + log("Not an valid ipv6 address: %s" % address, + level=ERROR) + address = None + return address + + +def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, fatal=True, exc_list=None): + """ + Return the assigned IP address for a given interface, if any, or []. + """ + # Extract nic if passed /dev/ethX + if '/' in iface: + iface = iface.split('/')[-1] + if not exc_list: + exc_list = [] try: - iface_addrs = netifaces.ifaddresses(iface) - if netifaces.AF_INET6 not in iface_addrs: - raise Exception("Interface '%s' doesn't have an ipv6 address." % iface) + inet_num = getattr(netifaces, inet_type) + except AttributeError: + raise Exception('Unknown inet type ' + str(inet_type)) - addresses = netifaces.ifaddresses(iface)[netifaces.AF_INET6] - ipv6_addr = [a['addr'] for a in addresses if not a['addr'].startswith('fe80') - and config('vip') != a['addr']] - if not ipv6_addr: - raise Exception("Interface '%s' doesn't have global ipv6 address." % iface) + interfaces = netifaces.interfaces() + if inc_aliases: + ifaces = [] + for _iface in interfaces: + if iface == _iface or _iface.split(':')[0] == iface: + ifaces.append(_iface) + if fatal and not ifaces: + raise Exception("Invalid interface '%s'" % iface) + ifaces.sort() + else: + if iface not in interfaces: + if fatal: + raise Exception("%s not found " % (iface)) + else: + return [] + else: + ifaces = [iface] - return ipv6_addr[0] + addresses = [] + for netiface in ifaces: + net_info = netifaces.ifaddresses(netiface) + if inet_num in net_info: + for entry in net_info[inet_num]: + if 'addr' in entry and entry['addr'] not in exc_list: + addresses.append(entry['addr']) + if fatal and not addresses: + raise Exception("Interface '%s' doesn't have any %s addresses." % (iface, inet_type)) + return addresses - except ValueError: - raise ValueError("Invalid interface '%s'" % iface) +get_ipv4_addr = partial(get_iface_addr, inet_type='AF_INET') + + +def get_ipv6_addr(iface='eth0', inc_aliases=False, fatal=True, exc_list=None): + """ + Return the assigned IPv6 address for a given interface, if any, or []. + """ + addresses = get_iface_addr(iface=iface, inet_type='AF_INET6', + inc_aliases=inc_aliases, fatal=fatal, + exc_list=exc_list) + remotly_addressable = [] + for address in addresses: + if not address.startswith('fe80'): + remotly_addressable.append(address) + if fatal and not remotly_addressable: + raise Exception("Interface '%s' doesn't have global ipv6 address." % iface) + return remotly_addressable + + +def get_bridges(vnic_dir='/sys/devices/virtual/net'): + """ + Return a list of bridges on the system or [] + """ + b_rgex = vnic_dir + '/*/bridge' + return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_rgex)] + + +def get_bridge_nics(bridge, vnic_dir='/sys/devices/virtual/net'): + """ + Return a list of nics comprising a given bridge on the system or [] + """ + brif_rgex = "%s/%s/brif/*" % (vnic_dir, bridge) + return [x.split('/')[-1] for x in glob.glob(brif_rgex)] + + +def is_bridge_member(nic): + """ + Check if a given nic is a member of a bridge + """ + for bridge in get_bridges(): + if nic in get_bridge_nics(bridge): + return True + return False diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index d41b74a..6fffa20 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -47,6 +47,7 @@ from charmhelpers.contrib.openstack.neutron import ( from charmhelpers.contrib.network.ip import ( get_address_in_network, get_ipv6_addr, + format_ipv6_addr, ) CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' @@ -168,8 +169,10 @@ class SharedDBContext(OSContextGenerator): for rid in relation_ids('shared-db'): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) + host = rdata.get('db_host') + host = format_ipv6_addr(host) or host ctxt = { - 'database_host': rdata.get('db_host'), + 'database_host': host, 'database': self.database, 'database_user': self.user, 'database_password': rdata.get(password_setting), @@ -245,9 +248,12 @@ class IdentityServiceContext(OSContextGenerator): for rid in relation_ids('identity-service'): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) + serv_host = rdata.get('service_host') + serv_host = format_ipv6_addr(serv_host) or serv_host + ctxt = { 'service_port': rdata.get('service_port'), - 'service_host': rdata.get('service_host'), + 'service_host': serv_host, 'auth_host': rdata.get('auth_host'), 'auth_port': rdata.get('auth_port'), 'admin_tenant_name': rdata.get('service_tenant'), @@ -297,11 +303,13 @@ class AMQPContext(OSContextGenerator): for unit in related_units(rid): if relation_get('clustered', rid=rid, unit=unit): ctxt['clustered'] = True - ctxt['rabbitmq_host'] = relation_get('vip', rid=rid, - unit=unit) + vip = relation_get('vip', rid=rid, unit=unit) + vip = format_ipv6_addr(vip) or vip + ctxt['rabbitmq_host'] = vip else: - ctxt['rabbitmq_host'] = relation_get('private-address', - rid=rid, unit=unit) + host = relation_get('private-address', rid=rid, unit=unit) + host = format_ipv6_addr(host) or host + ctxt['rabbitmq_host'] = host ctxt.update({ 'rabbitmq_user': username, 'rabbitmq_password': relation_get('password', rid=rid, @@ -340,8 +348,9 @@ class AMQPContext(OSContextGenerator): and len(related_units(rid)) > 1: rabbitmq_hosts = [] for unit in related_units(rid): - rabbitmq_hosts.append(relation_get('private-address', - rid=rid, unit=unit)) + host = relation_get('private-address', rid=rid, unit=unit) + host = format_ipv6_addr(host) or host + rabbitmq_hosts.append(host) ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts) if not context_complete(ctxt): return {} @@ -370,6 +379,7 @@ class CephContext(OSContextGenerator): ceph_addr = \ relation_get('ceph-public-address', rid=rid, unit=unit) or \ relation_get('private-address', rid=rid, unit=unit) + ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr mon_hosts.append(ceph_addr) ctxt = { diff --git a/hooks/charmhelpers/contrib/peerstorage/__init__.py b/hooks/charmhelpers/contrib/peerstorage/__init__.py index e0eeab5..11e4fea 100644 --- a/hooks/charmhelpers/contrib/peerstorage/__init__.py +++ b/hooks/charmhelpers/contrib/peerstorage/__init__.py @@ -1,44 +1,44 @@ +from charmhelpers.core.hookenv import relation_id as current_relation_id from charmhelpers.core.hookenv import ( + is_relation_made, relation_ids, relation_get, local_unit, relation_set, ) + """ This helper provides functions to support use of a peer relation for basic key/value storage, with the added benefit that all storage -can be replicated across peer units, so this is really useful for -services that issue usernames/passwords to remote services. +can be replicated across peer units. -def shared_db_changed() - # Only the lead unit should create passwords - if not is_leader(): - return - username = relation_get('username') - key = '{}.password'.format(username) - # Attempt to retrieve any existing password for this user - password = peer_retrieve(key) - if password is None: - # New user, create password and store - password = pwgen(length=64) - peer_store(key, password) - create_access(username, password) - relation_set(password=password) +Requirement to use: +To use this, the "peer_echo()" method has to be called form the peer +relation's relation-changed hook: -def cluster_changed() - # Echo any relation data other that *-address - # back onto the peer relation so all units have - # all *.password keys stored on their local relation - # for later retrieval. +@hooks.hook("cluster-relation-changed") # Adapt the to your peer relation name +def cluster_relation_changed(): peer_echo() +Once this is done, you can use peer storage from anywhere: + +@hooks.hook("some-hook") +def some_hook(): + # You can store and retrieve key/values this way: + if is_relation_made("cluster"): # from charmhelpers.core.hookenv + # There are peers available so we can work with peer storage + peer_store("mykey", "myvalue") + value = peer_retrieve("mykey") + print value + else: + print "No peers joind the relation, cannot share key/values :(" """ def peer_retrieve(key, relation_name='cluster'): - """ Retrieve a named key from peer relation relation_name """ + """Retrieve a named key from peer relation `relation_name`.""" cluster_rels = relation_ids(relation_name) if len(cluster_rels) > 0: cluster_rid = cluster_rels[0] @@ -49,8 +49,26 @@ def peer_retrieve(key, relation_name='cluster'): 'peer relation {}'.format(relation_name)) +def peer_retrieve_by_prefix(prefix, relation_name='cluster', delimiter='_', + inc_list=None, exc_list=None): + """ Retrieve k/v pairs given a prefix and filter using {inc,exc}_list """ + inc_list = inc_list if inc_list else [] + exc_list = exc_list if exc_list else [] + peerdb_settings = peer_retrieve('-', relation_name=relation_name) + matched = {} + for k, v in peerdb_settings.items(): + full_prefix = prefix + delimiter + if k.startswith(full_prefix): + new_key = k.replace(full_prefix, '') + if new_key in exc_list: + continue + if new_key in inc_list or len(inc_list) == 0: + matched[new_key] = v + return matched + + def peer_store(key, value, relation_name='cluster'): - """ Store the key/value pair on the named peer relation relation_name """ + """Store the key/value pair on the named peer relation `relation_name`.""" cluster_rels = relation_ids(relation_name) if len(cluster_rels) > 0: cluster_rid = cluster_rels[0] @@ -62,10 +80,10 @@ def peer_store(key, value, relation_name='cluster'): def peer_echo(includes=None): - """Echo filtered attributes back onto the same relation for storage + """Echo filtered attributes back onto the same relation for storage. - Note that this helper must only be called within a peer relation - changed hook + This is a requirement to use the peerstorage module - it needs to be called + from the peer relation's changed hook. """ rdata = relation_get() echo_data = {} @@ -81,3 +99,33 @@ def peer_echo(includes=None): echo_data[attribute] = value if len(echo_data) > 0: relation_set(relation_settings=echo_data) + + +def peer_store_and_set(relation_id=None, peer_relation_name='cluster', + peer_store_fatal=False, relation_settings=None, + delimiter='_', **kwargs): + """Store passed-in arguments both in argument relation and in peer storage. + + It functions like doing relation_set() and peer_store() at the same time, + with the same data. + + @param relation_id: the id of the relation to store the data on. Defaults + to the current relation. + @param peer_store_fatal: Set to True, the function will raise an exception + should the peer sotrage not be avialable.""" + + relation_settings = relation_settings if relation_settings else {} + relation_set(relation_id=relation_id, + relation_settings=relation_settings, + **kwargs) + if is_relation_made(peer_relation_name): + for key, value in dict(kwargs.items() + + relation_settings.items()).iteritems(): + key_prefix = relation_id or current_relation_id() + peer_store(key_prefix + delimiter + key, + value, + relation_name=peer_relation_name) + else: + if peer_store_fatal: + raise ValueError('Unable to detect ' + 'peer relation {}'.format(peer_relation_name)) diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index f396e03..324987e 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -203,6 +203,17 @@ class Config(dict): if os.path.exists(self.path): self.load_previous() + def __getitem__(self, key): + """For regular dict lookups, check the current juju config first, + then the previous (saved) copy. This ensures that user-saved values + will be returned by a dict lookup. + + """ + try: + return dict.__getitem__(self, key) + except KeyError: + return (self._prev_dict or {})[key] + def load_previous(self, path=None): """Load previous copy of config from disk. diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index 87e7071..1b11fa0 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -1,6 +1,8 @@ import os import urllib2 +from urllib import urlretrieve import urlparse +import hashlib from charmhelpers.fetch import ( BaseFetchHandler, @@ -12,7 +14,17 @@ from charmhelpers.payload.archive import ( ) from charmhelpers.core.host import mkdir +""" +This class is a plugin for charmhelpers.fetch.install_remote. +It grabs, validates and installs remote archives fetched over "http", "https", "ftp" or "file" protocols. The contents of the archive are installed in $CHARM_DIR/fetched/. + +Example usage: +install_remote("https://example.com/some/archive.tar.gz") +# Installs the contents of archive.tar.gz in $CHARM_DIR/fetched/. + +See charmhelpers.fetch.archiveurl.get_archivehandler for supported archive types. +""" class ArchiveUrlFetchHandler(BaseFetchHandler): """Handler for archives via generic URLs""" def can_handle(self, source): @@ -61,3 +73,31 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): except OSError as e: raise UnhandledSource(e.strerror) return extract(dld_file) + + # Mandatory file validation via Sha1 or MD5 hashing. + def download_and_validate(self, url, hashsum, validate="sha1"): + if validate == 'sha1' and len(hashsum) != 40: + raise ValueError("HashSum must be = 40 characters when using sha1" + " validation") + if validate == 'md5' and len(hashsum) != 32: + raise ValueError("HashSum must be = 32 characters when using md5" + " validation") + tempfile, headers = urlretrieve(url) + self.validate_file(tempfile, hashsum, validate) + return tempfile + + # Predicate method that returns status of hash matching expected hash. + def validate_file(self, source, hashsum, vmethod='sha1'): + if vmethod != 'sha1' and vmethod != 'md5': + raise ValueError("Validation Method not supported") + + if vmethod == 'md5': + m = hashlib.md5() + if vmethod == 'sha1': + m = hashlib.sha1() + with open(source) as f: + for line in f: + m.update(line) + if hashsum != m.hexdigest(): + msg = "Hash Mismatch on {} expected {} got {}" + raise ValueError(msg.format(source, hashsum, m.hexdigest()))