Sync with lp:~xianghui/charm-helpers/format-ipv6

hooks/charmhelpers/contrib/network/ip.py

@@ -1,10 +1,11 @@
+import glob
 import sys
 
 from functools import partial
 
 from charmhelpers.fetch import apt_install
 from charmhelpers.core.hookenv import (
-    ERROR, log, config,
+    ERROR, log,
 )
 
 try:
@@ -156,19 +157,102 @@ get_iface_for_address = partial(_get_for_address, key='iface')
 get_netmask_for_address = partial(_get_for_address, key='netmask')
 
 
-def get_ipv6_addr(iface="eth0"):
+def format_ipv6_addr(address):
+    """
+    IPv6 needs to be wrapped with [] in url link to parse correctly.
+    """
+    if is_ipv6(address):
+        address = "[%s]" % address
+    else:
+        log("Not an valid ipv6 address: %s" % address,
+            level=ERROR)
+        address = None
+    return address
+
+
+def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, fatal=True, exc_list=None):
+    """
+    Return the assigned IP address for a given interface, if any, or [].
+    """
+    # Extract nic if passed /dev/ethX
+    if '/' in iface:
+        iface = iface.split('/')[-1]
+    if not exc_list:
+        exc_list = []
     try:
-        iface_addrs = netifaces.ifaddresses(iface)
+        inet_num = getattr(netifaces, inet_type)
-        if netifaces.AF_INET6 not in iface_addrs:
+    except AttributeError:
-            raise Exception("Interface '%s' doesn't have an ipv6 address." % iface)
+        raise Exception('Unknown inet type ' + str(inet_type))
 
-        addresses = netifaces.ifaddresses(iface)[netifaces.AF_INET6]
+    interfaces = netifaces.interfaces()
-        ipv6_addr = [a['addr'] for a in addresses if not a['addr'].startswith('fe80')
+    if inc_aliases:
-                     and config('vip') != a['addr']]
+        ifaces = []
-        if not ipv6_addr:
+        for _iface in interfaces:
-            raise Exception("Interface '%s' doesn't have global ipv6 address." % iface)
+            if iface == _iface or _iface.split(':')[0] == iface:
+                ifaces.append(_iface)
+        if fatal and not ifaces:
+            raise Exception("Invalid interface '%s'" % iface)
+        ifaces.sort()
+    else:
+        if iface not in interfaces:
+            if fatal:
+                raise Exception("%s not found " % (iface))
+            else:
+                return []
+        else:
+            ifaces = [iface]
 
-        return ipv6_addr[0]
+    addresses = []
+    for netiface in ifaces:
+        net_info = netifaces.ifaddresses(netiface)
+        if inet_num in net_info:
+            for entry in net_info[inet_num]:
+                if 'addr' in entry and entry['addr'] not in exc_list:
+                    addresses.append(entry['addr'])
+    if fatal and not addresses:
+        raise Exception("Interface '%s' doesn't have any %s addresses." % (iface, inet_type))
+    return addresses
 
-    except ValueError:
+get_ipv4_addr = partial(get_iface_addr, inet_type='AF_INET')
-        raise ValueError("Invalid interface '%s'" % iface)
+
+
+def get_ipv6_addr(iface='eth0', inc_aliases=False, fatal=True, exc_list=None):
+    """
+    Return the assigned IPv6 address for a given interface, if any, or [].
+    """
+    addresses = get_iface_addr(iface=iface, inet_type='AF_INET6',
+                               inc_aliases=inc_aliases, fatal=fatal,
+                               exc_list=exc_list)
+    remotly_addressable = []
+    for address in addresses:
+        if not address.startswith('fe80'):
+            remotly_addressable.append(address)
+    if fatal and not remotly_addressable:
+        raise Exception("Interface '%s' doesn't have global ipv6 address." % iface)
+    return remotly_addressable
+
+
+def get_bridges(vnic_dir='/sys/devices/virtual/net'):
+    """
+    Return a list of bridges on the system or []
+    """
+    b_rgex = vnic_dir + '/*/bridge'
+    return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_rgex)]
+
+
+def get_bridge_nics(bridge, vnic_dir='/sys/devices/virtual/net'):
+    """
+    Return a list of nics comprising a given bridge on the system or []
+    """
+    brif_rgex = "%s/%s/brif/*" % (vnic_dir, bridge)
+    return [x.split('/')[-1] for x in glob.glob(brif_rgex)]
+
+
+def is_bridge_member(nic):
+    """
+    Check if a given nic is a member of a bridge
+    """
+    for bridge in get_bridges():
+        if nic in get_bridge_nics(bridge):
+            return True
+    return False

hooks/charmhelpers/contrib/openstack/context.py

@@ -47,6 +47,7 @@ from charmhelpers.contrib.openstack.neutron import (
 from charmhelpers.contrib.network.ip import (
     get_address_in_network,
     get_ipv6_addr,
+    format_ipv6_addr,
 )
 
 CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
@@ -168,8 +169,10 @@ class SharedDBContext(OSContextGenerator):
         for rid in relation_ids('shared-db'):
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
+                host = rdata.get('db_host')
+                host = format_ipv6_addr(host) or host
                 ctxt = {
-                    'database_host': rdata.get('db_host'),
+                    'database_host': host,
                     'database': self.database,
                     'database_user': self.user,
                     'database_password': rdata.get(password_setting),
@@ -245,9 +248,12 @@ class IdentityServiceContext(OSContextGenerator):
         for rid in relation_ids('identity-service'):
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
+                serv_host = rdata.get('service_host')
+                serv_host = format_ipv6_addr(serv_host) or serv_host
+
                 ctxt = {
                     'service_port': rdata.get('service_port'),
-                    'service_host': rdata.get('service_host'),
+                    'service_host': serv_host,
                     'auth_host': rdata.get('auth_host'),
                     'auth_port': rdata.get('auth_port'),
                     'admin_tenant_name': rdata.get('service_tenant'),
@@ -297,11 +303,13 @@ class AMQPContext(OSContextGenerator):
             for unit in related_units(rid):
                 if relation_get('clustered', rid=rid, unit=unit):
                     ctxt['clustered'] = True
-                    ctxt['rabbitmq_host'] = relation_get('vip', rid=rid,
+                    vip = relation_get('vip', rid=rid, unit=unit)
-                                                         unit=unit)
+                    vip = format_ipv6_addr(vip) or vip
+                    ctxt['rabbitmq_host'] = vip
                 else:
-                    ctxt['rabbitmq_host'] = relation_get('private-address',
+                    host = relation_get('private-address', rid=rid, unit=unit)
-                                                         rid=rid, unit=unit)
+                    host = format_ipv6_addr(host) or host
+                    ctxt['rabbitmq_host'] = host
                 ctxt.update({
                     'rabbitmq_user': username,
                     'rabbitmq_password': relation_get('password', rid=rid,
@@ -340,8 +348,9 @@ class AMQPContext(OSContextGenerator):
                     and len(related_units(rid)) > 1:
                 rabbitmq_hosts = []
                 for unit in related_units(rid):
-                    rabbitmq_hosts.append(relation_get('private-address',
+                    host = relation_get('private-address', rid=rid, unit=unit)
-                                                       rid=rid, unit=unit))
+                    host = format_ipv6_addr(host) or host
+                    rabbitmq_hosts.append(host)
                 ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts)
         if not context_complete(ctxt):
             return {}
@@ -370,6 +379,7 @@ class CephContext(OSContextGenerator):
                 ceph_addr = \
                     relation_get('ceph-public-address', rid=rid, unit=unit) or \
                     relation_get('private-address', rid=rid, unit=unit)
+                ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr
                 mon_hosts.append(ceph_addr)
 
         ctxt = {

hooks/charmhelpers/contrib/peerstorage/__init__.py

@@ -1,44 +1,44 @@
+from charmhelpers.core.hookenv import relation_id as current_relation_id
 from charmhelpers.core.hookenv import (
+    is_relation_made,
     relation_ids,
     relation_get,
     local_unit,
     relation_set,
 )
 
+
 """
 This helper provides functions to support use of a peer relation
 for basic key/value storage, with the added benefit that all storage
-can be replicated across peer units, so this is really useful for
+can be replicated across peer units.
-services that issue usernames/passwords to remote services.
 
-def shared_db_changed()
+Requirement to use:
-    # Only the lead unit should create passwords
-    if not is_leader():
-        return
-    username = relation_get('username')
-    key = '{}.password'.format(username)
-    # Attempt to retrieve any existing password for this user
-    password = peer_retrieve(key)
-    if password is None:
-        # New user, create password and store
-        password = pwgen(length=64)
-        peer_store(key, password)
-    create_access(username, password)
-    relation_set(password=password)
 
+To use this, the "peer_echo()" method has to be called form the peer
+relation's relation-changed hook:
 
-def cluster_changed()
+@hooks.hook("cluster-relation-changed") # Adapt the to your peer relation name
-    # Echo any relation data other that *-address
+def cluster_relation_changed():
-    # back onto the peer relation so all units have
-    # all *.password keys stored on their local relation
-    # for later retrieval.
     peer_echo()
 
+Once this is done, you can use peer storage from anywhere:
+
+@hooks.hook("some-hook")
+def some_hook():
+    # You can store and retrieve key/values this way:
+    if is_relation_made("cluster"):  # from charmhelpers.core.hookenv
+        # There are peers available so we can work with peer storage
+        peer_store("mykey", "myvalue")
+        value = peer_retrieve("mykey")
+        print value
+    else:
+        print "No peers joind the relation, cannot share key/values :("
 """
 
 
 def peer_retrieve(key, relation_name='cluster'):
-    """ Retrieve a named key from peer relation relation_name """
+    """Retrieve a named key from peer relation `relation_name`."""
     cluster_rels = relation_ids(relation_name)
     if len(cluster_rels) > 0:
         cluster_rid = cluster_rels[0]
@@ -49,8 +49,26 @@ def peer_retrieve(key, relation_name='cluster'):
                          'peer relation {}'.format(relation_name))
 
 
+def peer_retrieve_by_prefix(prefix, relation_name='cluster', delimiter='_',
+                            inc_list=None, exc_list=None):
+    """ Retrieve k/v pairs given a prefix and filter using {inc,exc}_list """
+    inc_list = inc_list if inc_list else []
+    exc_list = exc_list if exc_list else []
+    peerdb_settings = peer_retrieve('-', relation_name=relation_name)
+    matched = {}
+    for k, v in peerdb_settings.items():
+        full_prefix = prefix + delimiter
+        if k.startswith(full_prefix):
+            new_key = k.replace(full_prefix, '')
+            if new_key in exc_list:
+                continue
+            if new_key in inc_list or len(inc_list) == 0:
+                matched[new_key] = v
+    return matched
+
+
 def peer_store(key, value, relation_name='cluster'):
-    """ Store the key/value pair on the named peer relation relation_name """
+    """Store the key/value pair on the named peer relation `relation_name`."""
     cluster_rels = relation_ids(relation_name)
     if len(cluster_rels) > 0:
         cluster_rid = cluster_rels[0]
@@ -62,10 +80,10 @@ def peer_store(key, value, relation_name='cluster'):
 
 
 def peer_echo(includes=None):
-    """Echo filtered attributes back onto the same relation for storage
+    """Echo filtered attributes back onto the same relation for storage.
 
-    Note that this helper must only be called within a peer relation
+    This is a requirement to use the peerstorage module - it needs to be called
-    changed hook
+    from the peer relation's changed hook.
     """
     rdata = relation_get()
     echo_data = {}
@@ -81,3 +99,33 @@ def peer_echo(includes=None):
                     echo_data[attribute] = value
     if len(echo_data) > 0:
         relation_set(relation_settings=echo_data)
+
+
+def peer_store_and_set(relation_id=None, peer_relation_name='cluster',
+                       peer_store_fatal=False, relation_settings=None,
+                       delimiter='_', **kwargs):
+    """Store passed-in arguments both in argument relation and in peer storage.
+
+    It functions like doing relation_set() and peer_store() at the same time,
+    with the same data.
+
+    @param relation_id: the id of the relation to store the data on. Defaults
+                        to the current relation.
+    @param peer_store_fatal: Set to True, the function will raise an exception
+                             should the peer sotrage not be avialable."""
+
+    relation_settings = relation_settings if relation_settings else {}
+    relation_set(relation_id=relation_id,
+                 relation_settings=relation_settings,
+                 **kwargs)
+    if is_relation_made(peer_relation_name):
+        for key, value in dict(kwargs.items() +
+                               relation_settings.items()).iteritems():
+            key_prefix = relation_id or current_relation_id()
+            peer_store(key_prefix + delimiter + key,
+                       value,
+                       relation_name=peer_relation_name)
+    else:
+        if peer_store_fatal:
+            raise ValueError('Unable to detect '
+                             'peer relation {}'.format(peer_relation_name))

hooks/charmhelpers/core/hookenv.py

@@ -203,6 +203,17 @@ class Config(dict):
         if os.path.exists(self.path):
             self.load_previous()
 
+    def __getitem__(self, key):
+        """For regular dict lookups, check the current juju config first,
+        then the previous (saved) copy. This ensures that user-saved values
+        will be returned by a dict lookup.
+
+        """
+        try:
+            return dict.__getitem__(self, key)
+        except KeyError:
+            return (self._prev_dict or {})[key]
+
     def load_previous(self, path=None):
         """Load previous copy of config from disk.
 

hooks/charmhelpers/fetch/archiveurl.py

@@ -1,6 +1,8 @@
 import os
 import urllib2
+from urllib import urlretrieve
 import urlparse
+import hashlib
 
 from charmhelpers.fetch import (
     BaseFetchHandler,
@@ -12,7 +14,17 @@ from charmhelpers.payload.archive import (
 )
 from charmhelpers.core.host import mkdir
 
+"""
+This class is a plugin for charmhelpers.fetch.install_remote.
 
+It grabs, validates and installs remote archives fetched over "http", "https", "ftp" or "file" protocols. The contents of the archive are installed in $CHARM_DIR/fetched/.
+
+Example usage:
+install_remote("https://example.com/some/archive.tar.gz")
+# Installs the contents of archive.tar.gz in $CHARM_DIR/fetched/.
+
+See charmhelpers.fetch.archiveurl.get_archivehandler for supported archive types.
+"""
 class ArchiveUrlFetchHandler(BaseFetchHandler):
     """Handler for archives via generic URLs"""
     def can_handle(self, source):
@@ -61,3 +73,31 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
         except OSError as e:
             raise UnhandledSource(e.strerror)
         return extract(dld_file)
+
+    # Mandatory file validation via Sha1 or MD5 hashing.
+    def download_and_validate(self, url, hashsum, validate="sha1"):
+        if validate == 'sha1' and len(hashsum) != 40:
+            raise ValueError("HashSum must be = 40 characters when using sha1"
+                             " validation")
+        if validate == 'md5' and len(hashsum) != 32:
+            raise ValueError("HashSum must be = 32 characters when using md5"
+                             " validation")
+        tempfile, headers = urlretrieve(url)
+        self.validate_file(tempfile, hashsum, validate)
+        return tempfile
+
+    # Predicate method that returns status of hash matching expected hash.
+    def validate_file(self, source, hashsum, vmethod='sha1'):
+        if vmethod != 'sha1' and vmethod != 'md5':
+            raise ValueError("Validation Method not supported")
+
+        if vmethod == 'md5':
+            m = hashlib.md5()
+        if vmethod == 'sha1':
+            m = hashlib.sha1()
+        with open(source) as f:
+            for line in f:
+                m.update(line)
+        if hashsum != m.hexdigest():
+            msg = "Hash Mismatch on {} expected {} got {}"
+            raise ValueError(msg.format(source, hashsum, m.hexdigest()))