openstack/charm-swift-proxy
Code Issues Proposed changes

Sync with lp:~xianghui/charm-helpers/format-ipv6

Browse Source
This commit is contained in:
Hui Xiang
2014-09-19 20:01:10 +08:00
parent 60e1c2f4b9
commit 5d952bcb11
5 changed files with 240 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
hooks/charmhelpers/contrib/network/ip.py
View File

@@ -1,10 +1,11 @@
import glob
import sys
from functools import partial
from charmhelpers.fetch import apt_install
from charmhelpers.core.hookenv import (
ERROR, log, config,
ERROR, log,
)
try:
@@ -156,19 +157,102 @@ get_iface_for_address = partial(_get_for_address, key='iface')
get_netmask_for_address = partial(_get_for_address, key='netmask')
def get_ipv6_addr(iface="eth0"):
def format_ipv6_addr(address):
"""
IPv6 needs to be wrapped with [] in url link to parse correctly.
"""
if is_ipv6(address):
address = "[%s]" % address
else:
log("Not an valid ipv6 address: %s" % address,
level=ERROR)
address = None
return address
def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, fatal=True, exc_list=None):
"""
Return the assigned IP address for a given interface, if any, or [].
"""
# Extract nic if passed /dev/ethX
if '/' in iface:
iface = iface.split('/')[-1]
if not exc_list:
exc_list = []
try:
iface_addrs = netifaces.ifaddresses(iface)
if netifaces.AF_INET6 not in iface_addrs:
raise Exception("Interface '%s' doesn't have an ipv6 address." % iface)
inet_num = getattr(netifaces, inet_type)
except AttributeError:
raise Exception('Unknown inet type ' + str(inet_type))
addresses = netifaces.ifaddresses(iface)[netifaces.AF_INET6]
ipv6_addr = [a['addr'] for a in addresses if not a['addr'].startswith('fe80')
and config('vip') != a['addr']]
if not ipv6_addr:
raise Exception("Interface '%s' doesn't have global ipv6 address." % iface)
interfaces = netifaces.interfaces()
if inc_aliases:
ifaces = []
for _iface in interfaces:
if iface == _iface or _iface.split(':')[0] == iface:
ifaces.append(_iface)
if fatal and not ifaces:
raise Exception("Invalid interface '%s'" % iface)
ifaces.sort()
else:
if iface not in interfaces:
if fatal:
raise Exception("%s not found " % (iface))
else:
return []
else:
ifaces = [iface]
return ipv6_addr[0]
addresses = []
for netiface in ifaces:
net_info = netifaces.ifaddresses(netiface)
if inet_num in net_info:
for entry in net_info[inet_num]:
if 'addr' in entry and entry['addr'] not in exc_list:
addresses.append(entry['addr'])
if fatal and not addresses:
raise Exception("Interface '%s' doesn't have any %s addresses." % (iface, inet_type))
return addresses
except ValueError:
raise ValueError("Invalid interface '%s'" % iface)
get_ipv4_addr = partial(get_iface_addr, inet_type='AF_INET')
def get_ipv6_addr(iface='eth0', inc_aliases=False, fatal=True, exc_list=None):
"""
Return the assigned IPv6 address for a given interface, if any, or [].
"""
addresses = get_iface_addr(iface=iface, inet_type='AF_INET6',
inc_aliases=inc_aliases, fatal=fatal,
exc_list=exc_list)
remotly_addressable = []
for address in addresses:
if not address.startswith('fe80'):
remotly_addressable.append(address)
if fatal and not remotly_addressable:
raise Exception("Interface '%s' doesn't have global ipv6 address." % iface)
return remotly_addressable
def get_bridges(vnic_dir='/sys/devices/virtual/net'):
"""
Return a list of bridges on the system or []
"""
b_rgex = vnic_dir + '/*/bridge'
return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_rgex)]
def get_bridge_nics(bridge, vnic_dir='/sys/devices/virtual/net'):
"""
Return a list of nics comprising a given bridge on the system or []
"""
brif_rgex = "%s/%s/brif/*" % (vnic_dir, bridge)
return [x.split('/')[-1] for x in glob.glob(brif_rgex)]
def is_bridge_member(nic):
"""
Check if a given nic is a member of a bridge
"""
for bridge in get_bridges():
if nic in get_bridge_nics(bridge):
return True
return False

26
hooks/charmhelpers/contrib/openstack/context.py
View File

@@ -47,6 +47,7 @@ from charmhelpers.contrib.openstack.neutron import (
from charmhelpers.contrib.network.ip import (
get_address_in_network,
get_ipv6_addr,
format_ipv6_addr,
)
CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
@@ -168,8 +169,10 @@ class SharedDBContext(OSContextGenerator):
for rid in relation_ids('shared-db'):
for unit in related_units(rid):
rdata = relation_get(rid=rid, unit=unit)
host = rdata.get('db_host')
host = format_ipv6_addr(host) or host
ctxt = {
'database_host': rdata.get('db_host'),
'database_host': host,
'database': self.database,
'database_user': self.user,
'database_password': rdata.get(password_setting),
@@ -245,9 +248,12 @@ class IdentityServiceContext(OSContextGenerator):
for rid in relation_ids('identity-service'):
for unit in related_units(rid):
rdata = relation_get(rid=rid, unit=unit)
serv_host = rdata.get('service_host')
serv_host = format_ipv6_addr(serv_host) or serv_host
ctxt = {
'service_port': rdata.get('service_port'),
'service_host': rdata.get('service_host'),
'service_host': serv_host,
'auth_host': rdata.get('auth_host'),
'auth_port': rdata.get('auth_port'),
'admin_tenant_name': rdata.get('service_tenant'),
@@ -297,11 +303,13 @@ class AMQPContext(OSContextGenerator):
for unit in related_units(rid):
if relation_get('clustered', rid=rid, unit=unit):
ctxt['clustered'] = True
ctxt['rabbitmq_host'] = relation_get('vip', rid=rid,
unit=unit)
vip = relation_get('vip', rid=rid, unit=unit)
vip = format_ipv6_addr(vip) or vip
ctxt['rabbitmq_host'] = vip
else:
ctxt['rabbitmq_host'] = relation_get('private-address',
rid=rid, unit=unit)
host = relation_get('private-address', rid=rid, unit=unit)
host = format_ipv6_addr(host) or host
ctxt['rabbitmq_host'] = host
ctxt.update({
'rabbitmq_user': username,
'rabbitmq_password': relation_get('password', rid=rid,
@@ -340,8 +348,9 @@ class AMQPContext(OSContextGenerator):
and len(related_units(rid)) > 1:
rabbitmq_hosts = []
for unit in related_units(rid):
rabbitmq_hosts.append(relation_get('private-address',
rid=rid, unit=unit))
host = relation_get('private-address', rid=rid, unit=unit)
host = format_ipv6_addr(host) or host
rabbitmq_hosts.append(host)
ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts)
if not context_complete(ctxt):
return {}
@@ -370,6 +379,7 @@ class CephContext(OSContextGenerator):
ceph_addr = \
relation_get('ceph-public-address', rid=rid, unit=unit) or \
relation_get('private-address', rid=rid, unit=unit)
ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr
mon_hosts.append(ceph_addr)
ctxt = {

100
hooks/charmhelpers/contrib/peerstorage/__init__.py
View File

@@ -1,44 +1,44 @@
from charmhelpers.core.hookenv import relation_id as current_relation_id
from charmhelpers.core.hookenv import (
is_relation_made,
relation_ids,
relation_get,
local_unit,
relation_set,
)
"""
This helper provides functions to support use of a peer relation
for basic key/value storage, with the added benefit that all storage
can be replicated across peer units, so this is really useful for
services that issue usernames/passwords to remote services.
can be replicated across peer units.
def shared_db_changed()
# Only the lead unit should create passwords
if not is_leader():
return
username = relation_get('username')
key = '{}.password'.format(username)
# Attempt to retrieve any existing password for this user
password = peer_retrieve(key)
if password is None:
# New user, create password and store
password = pwgen(length=64)
peer_store(key, password)
create_access(username, password)
relation_set(password=password)
Requirement to use:
To use this, the "peer_echo()" method has to be called form the peer
relation's relation-changed hook:
def cluster_changed()
# Echo any relation data other that *-address
# back onto the peer relation so all units have
# all *.password keys stored on their local relation
# for later retrieval.
@hooks.hook("cluster-relation-changed") # Adapt the to your peer relation name
def cluster_relation_changed():
peer_echo()
Once this is done, you can use peer storage from anywhere:
@hooks.hook("some-hook")
def some_hook():
# You can store and retrieve key/values this way:
if is_relation_made("cluster"): # from charmhelpers.core.hookenv
# There are peers available so we can work with peer storage
peer_store("mykey", "myvalue")
value = peer_retrieve("mykey")
print value
else:
print "No peers joind the relation, cannot share key/values :("
"""
def peer_retrieve(key, relation_name='cluster'):
""" Retrieve a named key from peer relation relation_name """
"""Retrieve a named key from peer relation `relation_name`."""
cluster_rels = relation_ids(relation_name)
if len(cluster_rels) > 0:
cluster_rid = cluster_rels[0]
@@ -49,8 +49,26 @@ def peer_retrieve(key, relation_name='cluster'):
'peer relation {}'.format(relation_name))
def peer_retrieve_by_prefix(prefix, relation_name='cluster', delimiter='_',
inc_list=None, exc_list=None):
""" Retrieve k/v pairs given a prefix and filter using {inc,exc}_list """
inc_list = inc_list if inc_list else []
exc_list = exc_list if exc_list else []
peerdb_settings = peer_retrieve('-', relation_name=relation_name)
matched = {}
for k, v in peerdb_settings.items():
full_prefix = prefix + delimiter
if k.startswith(full_prefix):
new_key = k.replace(full_prefix, '')
if new_key in exc_list:
continue
if new_key in inc_list or len(inc_list) == 0:
matched[new_key] = v
return matched
def peer_store(key, value, relation_name='cluster'):
""" Store the key/value pair on the named peer relation relation_name """
"""Store the key/value pair on the named peer relation `relation_name`."""
cluster_rels = relation_ids(relation_name)
if len(cluster_rels) > 0:
cluster_rid = cluster_rels[0]
@@ -62,10 +80,10 @@ def peer_store(key, value, relation_name='cluster'):
def peer_echo(includes=None):
"""Echo filtered attributes back onto the same relation for storage
"""Echo filtered attributes back onto the same relation for storage.
Note that this helper must only be called within a peer relation
changed hook
This is a requirement to use the peerstorage module - it needs to be called
from the peer relation's changed hook.
"""
rdata = relation_get()
echo_data = {}
@@ -81,3 +99,33 @@ def peer_echo(includes=None):
echo_data[attribute] = value
if len(echo_data) > 0:
relation_set(relation_settings=echo_data)
def peer_store_and_set(relation_id=None, peer_relation_name='cluster',
peer_store_fatal=False, relation_settings=None,
delimiter='_', **kwargs):
"""Store passed-in arguments both in argument relation and in peer storage.
It functions like doing relation_set() and peer_store() at the same time,
with the same data.
@param relation_id: the id of the relation to store the data on. Defaults
to the current relation.
@param peer_store_fatal: Set to True, the function will raise an exception
should the peer sotrage not be avialable."""
relation_settings = relation_settings if relation_settings else {}
relation_set(relation_id=relation_id,
relation_settings=relation_settings,
**kwargs)
if is_relation_made(peer_relation_name):
for key, value in dict(kwargs.items() +
relation_settings.items()).iteritems():
key_prefix = relation_id or current_relation_id()
peer_store(key_prefix + delimiter + key,
value,
relation_name=peer_relation_name)
else:
if peer_store_fatal:
raise ValueError('Unable to detect '
'peer relation {}'.format(peer_relation_name))

11
hooks/charmhelpers/core/hookenv.py
View File

@@ -203,6 +203,17 @@ class Config(dict):
if os.path.exists(self.path):
self.load_previous()
def __getitem__(self, key):
"""For regular dict lookups, check the current juju config first,
then the previous (saved) copy. This ensures that user-saved values
will be returned by a dict lookup.
"""
try:
return dict.__getitem__(self, key)
except KeyError:
return (self._prev_dict or {})[key]
def load_previous(self, path=None):
"""Load previous copy of config from disk.

40
hooks/charmhelpers/fetch/archiveurl.py
View File

@@ -1,6 +1,8 @@
import os
import urllib2
from urllib import urlretrieve
import urlparse
import hashlib
from charmhelpers.fetch import (
BaseFetchHandler,
@@ -12,7 +14,17 @@ from charmhelpers.payload.archive import (
)
from charmhelpers.core.host import mkdir
"""
This class is a plugin for charmhelpers.fetch.install_remote.
It grabs, validates and installs remote archives fetched over "http", "https", "ftp" or "file" protocols. The contents of the archive are installed in $CHARM_DIR/fetched/.
Example usage:
install_remote("https://example.com/some/archive.tar.gz")
# Installs the contents of archive.tar.gz in $CHARM_DIR/fetched/.
See charmhelpers.fetch.archiveurl.get_archivehandler for supported archive types.
"""
class ArchiveUrlFetchHandler(BaseFetchHandler):
"""Handler for archives via generic URLs"""
def can_handle(self, source):
@@ -61,3 +73,31 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
except OSError as e:
raise UnhandledSource(e.strerror)
return extract(dld_file)
# Mandatory file validation via Sha1 or MD5 hashing.
def download_and_validate(self, url, hashsum, validate="sha1"):
if validate == 'sha1' and len(hashsum) != 40:
raise ValueError("HashSum must be = 40 characters when using sha1"
" validation")
if validate == 'md5' and len(hashsum) != 32:
raise ValueError("HashSum must be = 32 characters when using md5"
" validation")
tempfile, headers = urlretrieve(url)
self.validate_file(tempfile, hashsum, validate)
return tempfile
# Predicate method that returns status of hash matching expected hash.
def validate_file(self, source, hashsum, vmethod='sha1'):
if vmethod != 'sha1' and vmethod != 'md5':
raise ValueError("Validation Method not supported")
if vmethod == 'md5':
m = hashlib.md5()
if vmethod == 'sha1':
m = hashlib.sha1()
with open(source) as f:
for line in f:
m.update(line)
if hashsum != m.hexdigest():
msg = "Hash Mismatch on {} expected {} got {}"
raise ValueError(msg.format(source, hashsum, m.hexdigest()))