From 23a3e01b563821d8874ef23e88a3f4dfc4af5fe3 Mon Sep 17 00:00:00 2001
From: Edward Hope-Morley <edward.hope-morley@canonical.com>
Date: Thu, 26 Mar 2015 10:36:48 -0700
Subject: [PATCH 1/3] synced charm-helpers

---
 charm-helpers.yaml                            |   2 +-
 .../charmhelpers/contrib/hahelpers/cluster.py |  14 +-
 .../charmhelpers/contrib/openstack/context.py | 205 +++++++++++++++++-
 .../charmhelpers/contrib/openstack/neutron.py |  70 ++++++
 .../templates/section-keystone-authtoken      |   9 +
 .../openstack/templates/section-rabbitmq-oslo |  22 ++
 hooks/charmhelpers/core/hookenv.py            |  98 +++++++++
 hooks/charmhelpers/core/host.py               |   6 +-
 hooks/charmhelpers/core/services/helpers.py   |   4 +-
 9 files changed, 422 insertions(+), 8 deletions(-)
 create mode 100644 hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken
 create mode 100644 hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo

diff --git a/charm-helpers.yaml b/charm-helpers.yaml
index 6da8c15..b699795 100644
--- a/charm-helpers.yaml
+++ b/charm-helpers.yaml
@@ -1,4 +1,4 @@
-branch: lp:charm-helpers
+branch: lp:~le-charmers/charm-helpers/leadership-election
 destination: hooks/charmhelpers
 include:
     - core
diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py
index 9333efc..ebec379 100644
--- a/hooks/charmhelpers/contrib/hahelpers/cluster.py
+++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py
@@ -44,6 +44,7 @@ from charmhelpers.core.hookenv import (
     ERROR,
     WARNING,
     unit_get,
+    is_leader as juju_is_leader
 )
 from charmhelpers.core.decorators import (
     retry_on_exception,
@@ -66,12 +67,21 @@ def is_elected_leader(resource):
     Returns True if the charm executing this is the elected cluster leader.
 
     It relies on two mechanisms to determine leadership:
-        1. If the charm is part of a corosync cluster, call corosync to
+        1. If juju is sufficiently new and leadership election is supported,
+        the is_leader command will be used.
+        2. If the charm is part of a corosync cluster, call corosync to
         determine leadership.
-        2. If the charm is not part of a corosync cluster, the leader is
+        3. If the charm is not part of a corosync cluster, the leader is
         determined as being "the alive unit with the lowest unit numer". In
         other words, the oldest surviving unit.
     """
+    try:
+        return juju_is_leader()
+    except NotImplementedError:
+        log('Juju leadership election feature not enabled'
+            ', using fallback support',
+            level=WARNING)
+
     if is_clustered():
         if not is_crm_leader(resource):
             log('Deferring action to CRM leader.', level=INFO)
diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py
index 2d9a95c..45e6579 100644
--- a/hooks/charmhelpers/contrib/openstack/context.py
+++ b/hooks/charmhelpers/contrib/openstack/context.py
@@ -16,6 +16,7 @@
 
 import json
 import os
+import re
 import time
 from base64 import b64decode
 from subprocess import check_call
@@ -46,8 +47,11 @@ from charmhelpers.core.hookenv import (
 )
 
 from charmhelpers.core.sysctl import create as sysctl_create
+from charmhelpers.core.strutils import bool_from_string
 
 from charmhelpers.core.host import (
+    list_nics,
+    get_nic_hwaddr,
     mkdir,
     write_file,
 )
@@ -64,16 +68,22 @@ from charmhelpers.contrib.hahelpers.apache import (
 )
 from charmhelpers.contrib.openstack.neutron import (
     neutron_plugin_attribute,
+    parse_data_port_mappings,
+)
+from charmhelpers.contrib.openstack.ip import (
+    resolve_address,
+    INTERNAL,
 )
 from charmhelpers.contrib.network.ip import (
     get_address_in_network,
+    get_ipv4_addr,
     get_ipv6_addr,
     get_netmask_for_address,
     format_ipv6_addr,
     is_address_in_network,
+    is_bridge_member,
 )
 from charmhelpers.contrib.openstack.utils import get_host_ip
-
 CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
 ADDRESS_TYPES = ['admin', 'internal', 'public']
 
@@ -727,7 +737,14 @@ class ApacheSSLContext(OSContextGenerator):
                 'endpoints': [],
                 'ext_ports': []}
 
-        for cn in self.canonical_names():
+        cns = self.canonical_names()
+        if cns:
+            for cn in cns:
+                self.configure_cert(cn)
+        else:
+            # Expect cert/key provided in config (currently assumed that ca
+            # uses ip for cn)
+            cn = resolve_address(endpoint_type=INTERNAL)
             self.configure_cert(cn)
 
         addresses = self.get_network_addresses()
@@ -883,6 +900,48 @@ class NeutronContext(OSContextGenerator):
         return ctxt
 
 
+class NeutronPortContext(OSContextGenerator):
+    NIC_PREFIXES = ['eth', 'bond']
+
+    def resolve_ports(self, ports):
+        """Resolve NICs not yet bound to bridge(s)
+
+        If hwaddress provided then returns resolved hwaddress otherwise NIC.
+        """
+        if not ports:
+            return None
+
+        hwaddr_to_nic = {}
+        hwaddr_to_ip = {}
+        for nic in list_nics(self.NIC_PREFIXES):
+            hwaddr = get_nic_hwaddr(nic)
+            hwaddr_to_nic[hwaddr] = nic
+            addresses = get_ipv4_addr(nic, fatal=False)
+            addresses += get_ipv6_addr(iface=nic, fatal=False)
+            hwaddr_to_ip[hwaddr] = addresses
+
+        resolved = []
+        mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I)
+        for entry in ports:
+            if re.match(mac_regex, entry):
+                # NIC is in known NICs and does NOT hace an IP address
+                if entry in hwaddr_to_nic and not hwaddr_to_ip[entry]:
+                    # If the nic is part of a bridge then don't use it
+                    if is_bridge_member(hwaddr_to_nic[entry]):
+                        continue
+
+                    # Entry is a MAC address for a valid interface that doesn't
+                    # have an IP address assigned yet.
+                    resolved.append(hwaddr_to_nic[entry])
+            else:
+                # If the passed entry is not a MAC address, assume it's a valid
+                # interface, and that the user put it there on purpose (we can
+                # trust it to be the real external network).
+                resolved.append(entry)
+
+        return resolved
+
+
 class OSConfigFlagContext(OSContextGenerator):
     """Provides support for user-defined config flags.
 
@@ -1104,3 +1163,145 @@ class SysctlContext(OSContextGenerator):
             sysctl_create(sysctl_dict,
                           '/etc/sysctl.d/50-{0}.conf'.format(charm_name()))
         return {'sysctl': sysctl_dict}
+
+
+class NeutronAPIContext(OSContextGenerator):
+    '''
+    Inspects current neutron-plugin-api relation for neutron settings. Return
+    defaults if it is not present.
+    '''
+    interfaces = ['neutron-plugin-api']
+
+    def __call__(self):
+        self.neutron_defaults = {
+            'l2_population': {
+                'rel_key': 'l2-population',
+                'default': False,
+            },
+            'overlay_network_type': {
+                'rel_key': 'overlay-network-type',
+                'default': 'gre',
+            },
+            'neutron_security_groups': {
+                'rel_key': 'neutron-security-groups',
+                'default': False,
+            },
+            'network_device_mtu': {
+                'rel_key': 'network-device-mtu',
+                'default': None,
+            },
+            'enable_dvr': {
+                'rel_key': 'enable-dvr',
+                'default': False,
+            },
+            'enable_l3ha': {
+                'rel_key': 'enable-l3ha',
+                'default': False,
+            },
+        }
+        ctxt = self.get_neutron_options({})
+        for rid in relation_ids('neutron-plugin-api'):
+            for unit in related_units(rid):
+                rdata = relation_get(rid=rid, unit=unit)
+                if 'l2-population' in rdata:
+                    ctxt.update(self.get_neutron_options(rdata))
+
+        return ctxt
+
+    def get_neutron_options(self, rdata):
+        settings = {}
+        for nkey in self.neutron_defaults.keys():
+            defv = self.neutron_defaults[nkey]['default']
+            rkey = self.neutron_defaults[nkey]['rel_key']
+            if rkey in rdata.keys():
+                if type(defv) is bool:
+                    settings[nkey] = bool_from_string(rdata[rkey])
+                else:
+                    settings[nkey] = rdata[rkey]
+            else:
+                settings[nkey] = defv
+        return settings
+
+
+class ExternalPortContext(NeutronPortContext):
+
+    def __call__(self):
+        ctxt = {}
+        ports = config('ext-port')
+        if ports:
+            ports = [p.strip() for p in ports.split()]
+            ports = self.resolve_ports(ports)
+            if ports:
+                ctxt = {"ext_port": ports[0]}
+                napi_settings = NeutronAPIContext()()
+                mtu = napi_settings.get('network_device_mtu')
+                if mtu:
+                    ctxt['ext_port_mtu'] = mtu
+
+        return ctxt
+
+
+class DataPortContext(NeutronPortContext):
+
+    def __call__(self):
+        ports = config('data-port')
+        if ports:
+            portmap = parse_data_port_mappings(ports)
+            ports = portmap.values()
+            resolved = self.resolve_ports(ports)
+            normalized = {get_nic_hwaddr(port): port for port in resolved
+                          if port not in ports}
+            normalized.update({port: port for port in resolved
+                               if port in ports})
+            if resolved:
+                return {bridge: normalized[port] for bridge, port in
+                        six.iteritems(portmap) if port in normalized.keys()}
+
+        return None
+
+
+class PhyNICMTUContext(DataPortContext):
+
+    def __call__(self):
+        ctxt = {}
+        mappings = super(PhyNICMTUContext, self).__call__()
+        if mappings and mappings.values():
+            ports = mappings.values()
+            napi_settings = NeutronAPIContext()()
+            mtu = napi_settings.get('network_device_mtu')
+            if mtu:
+                ctxt["devs"] = '\\n'.join(ports)
+                ctxt['mtu'] = mtu
+
+        return ctxt
+
+
+class NetworkServiceContext(OSContextGenerator):
+
+    def __init__(self, rel_name='quantum-network-service'):
+        self.rel_name = rel_name
+        self.interfaces = [rel_name]
+
+    def __call__(self):
+        for rid in relation_ids(self.rel_name):
+            for unit in related_units(rid):
+                rdata = relation_get(rid=rid, unit=unit)
+                ctxt = {
+                    'keystone_host': rdata.get('keystone_host'),
+                    'service_port': rdata.get('service_port'),
+                    'auth_port': rdata.get('auth_port'),
+                    'service_tenant': rdata.get('service_tenant'),
+                    'service_username': rdata.get('service_username'),
+                    'service_password': rdata.get('service_password'),
+                    'quantum_host': rdata.get('quantum_host'),
+                    'quantum_port': rdata.get('quantum_port'),
+                    'quantum_url': rdata.get('quantum_url'),
+                    'region': rdata.get('region'),
+                    'service_protocol':
+                    rdata.get('service_protocol') or 'http',
+                    'auth_protocol':
+                    rdata.get('auth_protocol') or 'http',
+                }
+                if context_complete(ctxt):
+                    return ctxt
+        return {}
diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py
index 902757f..f885105 100644
--- a/hooks/charmhelpers/contrib/openstack/neutron.py
+++ b/hooks/charmhelpers/contrib/openstack/neutron.py
@@ -16,6 +16,7 @@
 
 # Various utilies for dealing with Neutron and the renaming from Quantum.
 
+import six
 from subprocess import check_output
 
 from charmhelpers.core.hookenv import (
@@ -237,3 +238,72 @@ def network_manager():
     else:
         # ensure accurate naming for all releases post-H
         return 'neutron'
+
+
+def parse_mappings(mappings):
+    parsed = {}
+    if mappings:
+        mappings = mappings.split(' ')
+        for m in mappings:
+            p = m.partition(':')
+            if p[1] == ':':
+                parsed[p[0].strip()] = p[2].strip()
+
+    return parsed
+
+
+def parse_bridge_mappings(mappings):
+    """Parse bridge mappings.
+
+    Mappings must be a space-delimited list of provider:bridge mappings.
+
+    Returns dict of the form {provider:bridge}.
+    """
+    return parse_mappings(mappings)
+
+
+def parse_data_port_mappings(mappings, default_bridge='br-data'):
+    """Parse data port mappings.
+
+    Mappings must be a space-delimited list of bridge:port mappings.
+
+    Returns dict of the form {bridge:port}.
+    """
+    _mappings = parse_mappings(mappings)
+    if not _mappings:
+        if not mappings:
+            return {}
+
+        # For backwards-compatibility we need to support port-only provided in
+        # config.
+        _mappings = {default_bridge: mappings.split(' ')[0]}
+
+    bridges = _mappings.keys()
+    ports = _mappings.values()
+    if len(set(bridges)) != len(bridges):
+        raise Exception("It is not allowed to have more than one port "
+                        "configured on the same bridge")
+
+    if len(set(ports)) != len(ports):
+        raise Exception("It is not allowed to have the same port configured "
+                        "on more than one bridge")
+
+    return _mappings
+
+
+def parse_vlan_range_mappings(mappings):
+    """Parse vlan range mappings.
+
+    Mappings must be a space-delimited list of provider:start:end mappings.
+
+    Returns dict of the form {provider: (start, end)}.
+    """
+    _mappings = parse_mappings(mappings)
+    if not _mappings:
+        return {}
+
+    mappings = {}
+    for p, r in six.iteritems(_mappings):
+        mappings[p] = tuple(r.split(':'))
+
+    return mappings
diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken
new file mode 100644
index 0000000..2a37edd
--- /dev/null
+++ b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken
@@ -0,0 +1,9 @@
+{% if auth_host -%}
+[keystone_authtoken]
+identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/{{ auth_admin_prefix }}
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }}
+admin_tenant_name = {{ admin_tenant_name }}
+admin_user = {{ admin_user }}
+admin_password = {{ admin_password }}
+signing_dir = {{ signing_dir }}
+{% endif -%}
diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo b/hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo
new file mode 100644
index 0000000..b444c9c
--- /dev/null
+++ b/hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo
@@ -0,0 +1,22 @@
+{% if rabbitmq_host or rabbitmq_hosts -%}
+[oslo_messaging_rabbit]
+rabbit_userid = {{ rabbitmq_user }}
+rabbit_virtual_host = {{ rabbitmq_virtual_host }}
+rabbit_password = {{ rabbitmq_password }}
+{% if rabbitmq_hosts -%}
+rabbit_hosts = {{ rabbitmq_hosts }}
+{% if rabbitmq_ha_queues -%}
+rabbit_ha_queues = True
+rabbit_durable_queues = False
+{% endif -%}
+{% else -%}
+rabbit_host = {{ rabbitmq_host }}
+{% endif -%}
+{% if rabbit_ssl_port -%}
+rabbit_use_ssl = True
+rabbit_port = {{ rabbit_ssl_port }}
+{% if rabbit_ssl_ca -%}
+kombu_ssl_ca_certs = {{ rabbit_ssl_ca }}
+{% endif -%}
+{% endif -%}
+{% endif -%}
diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py
index cf552b3..60820cc 100644
--- a/hooks/charmhelpers/core/hookenv.py
+++ b/hooks/charmhelpers/core/hookenv.py
@@ -352,6 +352,17 @@ def relation_set(relation_id=None, relation_settings=None, **kwargs):
     flush(local_unit())
 
 
+def relation_clear(r_id=None):
+    ''' Clears any relation data already set on relation r_id '''
+    settings = relation_get(rid=r_id,
+                            unit=local_unit())
+    for setting in settings:
+        if setting not in ['public-address', 'private-address']:
+            settings[setting] = None
+    relation_set(relation_id=r_id,
+                 **settings)
+
+
 @cached
 def relation_ids(reltype=None):
     """A list of relation_ids"""
@@ -566,3 +577,90 @@ class Hooks(object):
 def charm_dir():
     """Return the root directory of the current charm"""
     return os.environ.get('CHARM_DIR')
+
+
+def translate_exc(from_exc, to_exc):
+    def inner_translate_exc1(f):
+        def inner_translate_exc2(*args, **kwargs):
+            try:
+                return f(*args, **kwargs)
+            except from_exc:
+                raise to_exc
+
+        return inner_translate_exc2
+
+    return inner_translate_exc1
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def is_leader():
+    """Does the current unit hold the juju leadership
+
+    Uses juju to determine whether the current unit is the leader of its peers
+    """
+    try:
+        leader = json.loads(
+            subprocess.check_output(['is-leader', '--format=json'])
+        )
+        return (leader is True)
+    except ValueError:
+        raise NotImplementedError
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def leader_get(attribute=None):
+    """Juju leader get value(s)"""
+    cmd = ['leader-get', '--format=json'] + [attribute or '-']
+    try:
+        ret = json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+        log("Juju leader-get '%s' = '%s'" % (attribute, ret), level=DEBUG)
+        return ret
+    except ValueError:
+        return None
+    except CalledProcessError as e:
+        if e.returncode == 2:
+            return None
+
+        raise
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def leader_set(settings=None, **kwargs):
+    """Juju leader set value(s)"""
+    log("Juju leader-set '%s'" % (settings), level=DEBUG)
+    cmd = ['leader-set']
+    settings = settings or {}
+    settings.update(kwargs)
+    for k, v in settings.iteritems():
+        if v is None:
+            cmd.append('{}='.format(k))
+        else:
+            cmd.append('{}={}'.format(k, v))
+
+    subprocess.check_call(cmd)
+
+
+@cached
+def action_get(key=None):
+    """Gets the value of an action parameter, or all key/value param pairs"""
+    cmd = ['action-get']
+    if key is not None:
+        cmd.append(key)
+    cmd.append('--format=json')
+    action_data = json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+    return action_data
+
+
+def action_set(values):
+    """Sets the values to be returned after the action finishes"""
+    cmd = ['action-set']
+    for k, v in list(values.items()):
+        cmd.append('{}={}'.format(k, v))
+    subprocess.check_call(cmd)
+
+
+def action_fail(message):
+    """Sets the action status to failed and sets the error message.
+
+    The results set by action_set are preserved."""
+    subprocess.check_call(['action-fail', message])
diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py
index b771c61..830822a 100644
--- a/hooks/charmhelpers/core/host.py
+++ b/hooks/charmhelpers/core/host.py
@@ -339,12 +339,16 @@ def lsb_release():
 def pwgen(length=None):
     """Generate a random pasword."""
     if length is None:
+        # A random length is ok to use a weak PRNG
         length = random.choice(range(35, 45))
     alphanumeric_chars = [
         l for l in (string.ascii_letters + string.digits)
         if l not in 'l0QD1vAEIOUaeiou']
+    # Use a crypto-friendly PRNG (e.g. /dev/urandom) for making the
+    # actual password
+    random_generator = random.SystemRandom()
     random_chars = [
-        random.choice(alphanumeric_chars) for _ in range(length)]
+        random_generator.choice(alphanumeric_chars) for _ in range(length)]
     return(''.join(random_chars))
 
 
diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py
index 15b2166..3eb5fb4 100644
--- a/hooks/charmhelpers/core/services/helpers.py
+++ b/hooks/charmhelpers/core/services/helpers.py
@@ -139,7 +139,7 @@ class MysqlRelation(RelationContext):
 
     def __init__(self, *args, **kwargs):
         self.required_keys = ['host', 'user', 'password', 'database']
-        super(HttpRelation).__init__(self, *args, **kwargs)
+        RelationContext.__init__(self, *args, **kwargs)
 
 
 class HttpRelation(RelationContext):
@@ -154,7 +154,7 @@ class HttpRelation(RelationContext):
 
     def __init__(self, *args, **kwargs):
         self.required_keys = ['host', 'port']
-        super(HttpRelation).__init__(self, *args, **kwargs)
+        RelationContext.__init__(self, *args, **kwargs)
 
     def provide_data(self):
         return {

From 6f7c6c235cb5718961988663dbef2ce5f5b05509 Mon Sep 17 00:00:00 2001
From: Edward Hope-Morley <edward.hope-morley@canonical.com>
Date: Wed, 1 Apr 2015 14:50:29 +0100
Subject: [PATCH 2/3] synced l/e ch

---
 .../contrib/openstack/amulet/deployment.py    |  27 ++-
 .../charmhelpers/contrib/openstack/context.py |   9 +-
 .../contrib/openstack/templates/git.upstart   |  13 ++
 .../templates/{zeromq => section-zeromq}      |   4 +-
 hooks/charmhelpers/contrib/openstack/utils.py | 179 ++++++++++++------
 hooks/charmhelpers/core/unitdata.py           |   2 +-
 6 files changed, 163 insertions(+), 71 deletions(-)
 create mode 100644 hooks/charmhelpers/contrib/openstack/templates/git.upstart
 rename hooks/charmhelpers/contrib/openstack/templates/{zeromq => section-zeromq} (66%)

diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
index 0cfeaa4..0e0db56 100644
--- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -15,6 +15,7 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import six
+from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
@@ -100,12 +101,34 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            """
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
-         self.trusty_icehouse) = range(6)
+         self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8)
         releases = {
             ('precise', None): self.precise_essex,
             ('precise', 'cloud:precise-folsom'): self.precise_folsom,
             ('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
             ('precise', 'cloud:precise-havana'): self.precise_havana,
             ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
-            ('trusty', None): self.trusty_icehouse}
+            ('trusty', None): self.trusty_icehouse,
+            ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
+            ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo}
         return releases[(self.series, self.openstack)]
+
+    def _get_openstack_release_string(self):
+        """Get openstack release string.
+
+           Return a string representing the openstack release.
+           """
+        releases = OrderedDict([
+            ('precise', 'essex'),
+            ('quantal', 'folsom'),
+            ('raring', 'grizzly'),
+            ('saucy', 'havana'),
+            ('trusty', 'icehouse'),
+            ('utopic', 'juno'),
+            ('vivid', 'kilo'),
+        ])
+        if self.openstack:
+            os_origin = self.openstack.split(':')[1]
+            return os_origin.split('%s-' % self.series)[1].split('/')[0]
+        else:
+            return releases[self.series]
diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py
index 45e6579..dd51bfb 100644
--- a/hooks/charmhelpers/contrib/openstack/context.py
+++ b/hooks/charmhelpers/contrib/openstack/context.py
@@ -320,14 +320,15 @@ def db_ssl(rdata, ctxt, ssl_dir):
 
 
 class IdentityServiceContext(OSContextGenerator):
-    interfaces = ['identity-service']
 
-    def __init__(self, service=None, service_user=None):
+    def __init__(self, service=None, service_user=None, rel_name='identity-service'):
         self.service = service
         self.service_user = service_user
+        self.rel_name = rel_name
+        self.interfaces = [self.rel_name]
 
     def __call__(self):
-        log('Generating template context for identity-service', level=DEBUG)
+        log('Generating template context for ' + self.rel_name, level=DEBUG)
         ctxt = {}
 
         if self.service and self.service_user:
@@ -341,7 +342,7 @@ class IdentityServiceContext(OSContextGenerator):
 
             ctxt['signing_dir'] = cachedir
 
-        for rid in relation_ids('identity-service'):
+        for rid in relation_ids(self.rel_name):
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 serv_host = rdata.get('service_host')
diff --git a/hooks/charmhelpers/contrib/openstack/templates/git.upstart b/hooks/charmhelpers/contrib/openstack/templates/git.upstart
new file mode 100644
index 0000000..da94ad1
--- /dev/null
+++ b/hooks/charmhelpers/contrib/openstack/templates/git.upstart
@@ -0,0 +1,13 @@
+description "{{ service_description }}"
+author "Juju {{ service_name }} Charm <juju@localhost>"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+
+exec start-stop-daemon --start --chuid {{ user_name }} \
+            --chdir {{ start_dir }} --name {{ process_name }} \
+            --exec {{ executable_name }} -- \
+            --config-file={{ config_file }} \
+            --log-file={{ log_file }}
diff --git a/hooks/charmhelpers/contrib/openstack/templates/zeromq b/hooks/charmhelpers/contrib/openstack/templates/section-zeromq
similarity index 66%
rename from hooks/charmhelpers/contrib/openstack/templates/zeromq
rename to hooks/charmhelpers/contrib/openstack/templates/section-zeromq
index 0695eef..95f1a76 100644
--- a/hooks/charmhelpers/contrib/openstack/templates/zeromq
+++ b/hooks/charmhelpers/contrib/openstack/templates/section-zeromq
@@ -3,12 +3,12 @@
 rpc_backend = zmq
 rpc_zmq_host = {{ zmq_host }}
 {% if zmq_redis_address -%}
-rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_redis.MatchMakerRedis
+rpc_zmq_matchmaker = redis
 matchmaker_heartbeat_freq = 15
 matchmaker_heartbeat_ttl = 30
 [matchmaker_redis]
 host = {{ zmq_redis_address }}
 {% else -%}
-rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_ring.MatchMakerRing
+rpc_zmq_matchmaker = ring
 {% endif -%}
 {% endif -%}
diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index 4f110c6..78c5e2d 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -30,6 +30,10 @@ import yaml
 
 from charmhelpers.contrib.network import ip
 
+from charmhelpers.core import (
+    unitdata,
+)
+
 from charmhelpers.core.hookenv import (
     config,
     log as juju_log,
@@ -330,6 +334,21 @@ def configure_installation_source(rel):
         error_out("Invalid openstack-release specified: %s" % rel)
 
 
+def config_value_changed(option):
+    """
+    Determine if config value changed since last call to this function.
+    """
+    hook_data = unitdata.HookData()
+    with hook_data():
+        db = unitdata.kv()
+        current = config(option)
+        saved = db.get(option)
+        db.set(option, current)
+        if saved is None:
+            return False
+        return current != saved
+
+
 def save_script_rc(script_path="scripts/scriptrc", **env_vars):
     """
     Write an rc file in the charm-delivered directory containing
@@ -469,82 +488,95 @@ def os_requires_version(ostack_release, pkg):
 
 
 def git_install_requested():
-    """Returns true if openstack-origin-git is specified."""
-    return config('openstack-origin-git') != "None"
+    """
+    Returns true if openstack-origin-git is specified.
+    """
+    return config('openstack-origin-git') is not None
 
 
 requirements_dir = None
 
 
-def git_clone_and_install(file_name, core_project):
-    """Clone/install all OpenStack repos specified in yaml config file."""
-    global requirements_dir
+def git_clone_and_install(projects_yaml, core_project):
+    """
+    Clone/install all specified OpenStack repositories.
 
-    if file_name == "None":
+    The expected format of projects_yaml is:
+        repositories:
+          - {name: keystone,
+             repository: 'git://git.openstack.org/openstack/keystone.git',
+             branch: 'stable/icehouse'}
+          - {name: requirements,
+             repository: 'git://git.openstack.org/openstack/requirements.git',
+             branch: 'stable/icehouse'}
+        directory: /mnt/openstack-git
+
+        The directory key is optional.
+    """
+    global requirements_dir
+    parent_dir = '/mnt/openstack-git'
+
+    if not projects_yaml:
         return
 
-    yaml_file = os.path.join(charm_dir(), file_name)
+    projects = yaml.load(projects_yaml)
+    _git_validate_projects_yaml(projects, core_project)
 
-    # clone/install the requirements project first
-    installed = _git_clone_and_install_subset(yaml_file,
-                                              whitelist=['requirements'])
-    if 'requirements' not in installed:
-        error_out('requirements git repository must be specified')
+    if 'directory' in projects.keys():
+        parent_dir = projects['directory']
 
-    # clone/install all other projects except requirements and the core project
-    blacklist = ['requirements', core_project]
-    _git_clone_and_install_subset(yaml_file, blacklist=blacklist,
-                                  update_requirements=True)
-
-    # clone/install the core project
-    whitelist = [core_project]
-    installed = _git_clone_and_install_subset(yaml_file, whitelist=whitelist,
-                                              update_requirements=True)
-    if core_project not in installed:
-        error_out('{} git repository must be specified'.format(core_project))
+    for p in projects['repositories']:
+        repo = p['repository']
+        branch = p['branch']
+        if p['name'] == 'requirements':
+            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+                                                     update_requirements=False)
+            requirements_dir = repo_dir
+        else:
+            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+                                                     update_requirements=True)
 
 
-def _git_clone_and_install_subset(yaml_file, whitelist=[], blacklist=[],
-                                  update_requirements=False):
-    """Clone/install subset of OpenStack repos specified in yaml config file."""
-    global requirements_dir
-    installed = []
+def _git_validate_projects_yaml(projects, core_project):
+    """
+    Validate the projects yaml.
+    """
+    _git_ensure_key_exists('repositories', projects)
 
-    with open(yaml_file, 'r') as fd:
-        projects = yaml.load(fd)
-        for proj, val in projects.items():
-            # The project subset is chosen based on the following 3 rules:
-            # 1) If project is in blacklist, we don't clone/install it, period.
-            # 2) If whitelist is empty, we clone/install everything else.
-            # 3) If whitelist is not empty, we clone/install everything in the
-            #    whitelist.
-            if proj in blacklist:
-                continue
-            if whitelist and proj not in whitelist:
-                continue
-            repo = val['repository']
-            branch = val['branch']
-            repo_dir = _git_clone_and_install_single(repo, branch,
-                                                     update_requirements)
-            if proj == 'requirements':
-                requirements_dir = repo_dir
-            installed.append(proj)
-    return installed
+    for project in projects['repositories']:
+        _git_ensure_key_exists('name', project.keys())
+        _git_ensure_key_exists('repository', project.keys())
+        _git_ensure_key_exists('branch', project.keys())
+
+    if projects['repositories'][0]['name'] != 'requirements':
+        error_out('{} git repo must be specified first'.format('requirements'))
+
+    if projects['repositories'][-1]['name'] != core_project:
+        error_out('{} git repo must be specified last'.format(core_project))
 
 
-def _git_clone_and_install_single(repo, branch, update_requirements=False):
-    """Clone and install a single git repository."""
-    dest_parent_dir = "/mnt/openstack-git/"
-    dest_dir = os.path.join(dest_parent_dir, os.path.basename(repo))
+def _git_ensure_key_exists(key, keys):
+    """
+    Ensure that key exists in keys.
+    """
+    if key not in keys:
+        error_out('openstack-origin-git key \'{}\' is missing'.format(key))
 
-    if not os.path.exists(dest_parent_dir):
-        juju_log('Host dir not mounted at {}. '
-                 'Creating directory there instead.'.format(dest_parent_dir))
-        os.mkdir(dest_parent_dir)
+
+def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements):
+    """
+    Clone and install a single git repository.
+    """
+    dest_dir = os.path.join(parent_dir, os.path.basename(repo))
+
+    if not os.path.exists(parent_dir):
+        juju_log('Directory already exists at {}. '
+                 'No need to create directory.'.format(parent_dir))
+        os.mkdir(parent_dir)
 
     if not os.path.exists(dest_dir):
         juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
-        repo_dir = install_remote(repo, dest=dest_parent_dir, branch=branch)
+        repo_dir = install_remote(repo, dest=parent_dir, branch=branch)
     else:
         repo_dir = dest_dir
 
@@ -561,16 +593,39 @@ def _git_clone_and_install_single(repo, branch, update_requirements=False):
 
 
 def _git_update_requirements(package_dir, reqs_dir):
-    """Update from global requirements.
+    """
+    Update from global requirements.
 
-       Update an OpenStack git directory's requirements.txt and
-       test-requirements.txt from global-requirements.txt."""
+    Update an OpenStack git directory's requirements.txt and
+    test-requirements.txt from global-requirements.txt.
+    """
     orig_dir = os.getcwd()
     os.chdir(reqs_dir)
-    cmd = "python update.py {}".format(package_dir)
+    cmd = ['python', 'update.py', package_dir]
     try:
-        subprocess.check_call(cmd.split(' '))
+        subprocess.check_call(cmd)
     except subprocess.CalledProcessError:
         package = os.path.basename(package_dir)
         error_out("Error updating {} from global-requirements.txt".format(package))
     os.chdir(orig_dir)
+
+
+def git_src_dir(projects_yaml, project):
+    """
+    Return the directory where the specified project's source is located.
+    """
+    parent_dir = '/mnt/openstack-git'
+
+    if not projects_yaml:
+        return
+
+    projects = yaml.load(projects_yaml)
+
+    if 'directory' in projects.keys():
+        parent_dir = projects['directory']
+
+    for p in projects['repositories']:
+        if p['name'] == project:
+            return os.path.join(parent_dir, os.path.basename(p['repository']))
+
+    return None
diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py
index 3000134..406a35c 100644
--- a/hooks/charmhelpers/core/unitdata.py
+++ b/hooks/charmhelpers/core/unitdata.py
@@ -443,7 +443,7 @@ class HookData(object):
         data = hookenv.execution_environment()
         self.conf = conf_delta = self.kv.delta(data['conf'], 'config')
         self.rels = rels_delta = self.kv.delta(data['rels'], 'rels')
-        self.kv.set('env', data['env'])
+        self.kv.set('env', dict(data['env']))
         self.kv.set('unit', data['unit'])
         self.kv.set('relid', data.get('relid'))
         return conf_delta, rels_delta

From 774e2ac0ca74a584e31f6f7b5eb3396ab9f59eb9 Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Thu, 4 Jun 2015 09:45:25 +0100
Subject: [PATCH 3/3] Resync le charm helpers

---
 charm-helpers.yaml                            |   2 +-
 .../charmhelpers/contrib/hahelpers/cluster.py |  25 ++++
 .../charmhelpers/contrib/openstack/neutron.py |  15 +-
 hooks/charmhelpers/contrib/openstack/utils.py |  81 +++++++---
 hooks/charmhelpers/contrib/python/packages.py |  33 ++++-
 hooks/charmhelpers/core/hookenv.py            | 139 +++++++++---------
 hooks/charmhelpers/core/services/base.py      |  39 +++--
 hooks/charmhelpers/fetch/giturl.py            |  12 +-
 8 files changed, 237 insertions(+), 109 deletions(-)

diff --git a/charm-helpers.yaml b/charm-helpers.yaml
index b699795..6da8c15 100644
--- a/charm-helpers.yaml
+++ b/charm-helpers.yaml
@@ -1,4 +1,4 @@
-branch: lp:~le-charmers/charm-helpers/leadership-election
+branch: lp:charm-helpers
 destination: hooks/charmhelpers
 include:
     - core
diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py
index ebec379..5790b46 100644
--- a/hooks/charmhelpers/contrib/hahelpers/cluster.py
+++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py
@@ -53,6 +53,8 @@ from charmhelpers.core.strutils import (
     bool_from_string,
 )
 
+DC_RESOURCE_NAME = 'DC'
+
 
 class HAIncompleteConfig(Exception):
     pass
@@ -105,6 +107,27 @@ def is_clustered():
     return False
 
 
+def is_crm_dc():
+    """
+    Determine leadership by querying the pacemaker Designated Controller
+    """
+    cmd = ['crm', 'status']
+    try:
+        status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+        if not isinstance(status, six.text_type):
+            status = six.text_type(status, "utf-8")
+    except subprocess.CalledProcessError:
+        return False
+    current_dc = ''
+    for line in status.split('\n'):
+        if line.startswith('Current DC'):
+            # Current DC: juju-lytrusty-machine-2 (168108163) - partition with quorum
+            current_dc = line.split(':')[1].split()[0]
+    if current_dc == get_unit_hostname():
+        return True
+    return False
+
+
 @retry_on_exception(5, base_delay=2, exc_type=CRMResourceNotFound)
 def is_crm_leader(resource, retry=False):
     """
@@ -114,6 +137,8 @@ def is_crm_leader(resource, retry=False):
     We allow this operation to be retried to avoid the possibility of getting a
     false negative. See LP #1396246 for more info.
     """
+    if resource == DC_RESOURCE_NAME:
+        return is_crm_dc()
     cmd = ['crm', 'resource', 'show', resource]
     try:
         status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py
index 02c92e9..b3aa3d4 100644
--- a/hooks/charmhelpers/contrib/openstack/neutron.py
+++ b/hooks/charmhelpers/contrib/openstack/neutron.py
@@ -256,11 +256,14 @@ def network_manager():
 def parse_mappings(mappings):
     parsed = {}
     if mappings:
-        mappings = mappings.split(' ')
+        mappings = mappings.split()
         for m in mappings:
             p = m.partition(':')
-            if p[1] == ':':
-                parsed[p[0].strip()] = p[2].strip()
+            key = p[0].strip()
+            if p[1]:
+                parsed[key] = p[2].strip()
+            else:
+                parsed[key] = ''
 
     return parsed
 
@@ -283,13 +286,13 @@ def parse_data_port_mappings(mappings, default_bridge='br-data'):
     Returns dict of the form {bridge:port}.
     """
     _mappings = parse_mappings(mappings)
-    if not _mappings:
+    if not _mappings or list(_mappings.values()) == ['']:
         if not mappings:
             return {}
 
         # For backwards-compatibility we need to support port-only provided in
         # config.
-        _mappings = {default_bridge: mappings.split(' ')[0]}
+        _mappings = {default_bridge: mappings.split()[0]}
 
     bridges = _mappings.keys()
     ports = _mappings.values()
@@ -309,6 +312,8 @@ def parse_vlan_range_mappings(mappings):
 
     Mappings must be a space-delimited list of provider:start:end mappings.
 
+    The start:end range is optional and may be omitted.
+
     Returns dict of the form {provider: (start, end)}.
     """
     _mappings = parse_mappings(mappings)
diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index f90a028..d795a35 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -53,9 +53,13 @@ from charmhelpers.contrib.network.ip import (
     get_ipv6_addr
 )
 
+from charmhelpers.contrib.python.packages import (
+    pip_create_virtualenv,
+    pip_install,
+)
+
 from charmhelpers.core.host import lsb_release, mounts, umount
 from charmhelpers.fetch import apt_install, apt_cache, install_remote
-from charmhelpers.contrib.python.packages import pip_install
 from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
 from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
 
@@ -497,7 +501,17 @@ def git_install_requested():
 requirements_dir = None
 
 
-def git_clone_and_install(projects_yaml, core_project):
+def _git_yaml_load(projects_yaml):
+    """
+    Load the specified yaml into a dictionary.
+    """
+    if not projects_yaml:
+        return None
+
+    return yaml.load(projects_yaml)
+
+
+def git_clone_and_install(projects_yaml, core_project, depth=1):
     """
     Clone/install all specified OpenStack repositories.
 
@@ -510,23 +524,22 @@ def git_clone_and_install(projects_yaml, core_project):
              repository: 'git://git.openstack.org/openstack/requirements.git',
              branch: 'stable/icehouse'}
         directory: /mnt/openstack-git
-        http_proxy: http://squid.internal:3128
-        https_proxy: https://squid.internal:3128
+        http_proxy: squid-proxy-url
+        https_proxy: squid-proxy-url
 
         The directory, http_proxy, and https_proxy keys are optional.
     """
     global requirements_dir
     parent_dir = '/mnt/openstack-git'
+    http_proxy = None
 
-    if not projects_yaml:
-        return
-
-    projects = yaml.load(projects_yaml)
+    projects = _git_yaml_load(projects_yaml)
     _git_validate_projects_yaml(projects, core_project)
 
     old_environ = dict(os.environ)
 
     if 'http_proxy' in projects.keys():
+        http_proxy = projects['http_proxy']
         os.environ['http_proxy'] = projects['http_proxy']
     if 'https_proxy' in projects.keys():
         os.environ['https_proxy'] = projects['https_proxy']
@@ -534,15 +547,19 @@ def git_clone_and_install(projects_yaml, core_project):
     if 'directory' in projects.keys():
         parent_dir = projects['directory']
 
+    pip_create_virtualenv(os.path.join(parent_dir, 'venv'))
+
     for p in projects['repositories']:
         repo = p['repository']
         branch = p['branch']
         if p['name'] == 'requirements':
-            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+            repo_dir = _git_clone_and_install_single(repo, branch, depth,
+                                                     parent_dir, http_proxy,
                                                      update_requirements=False)
             requirements_dir = repo_dir
         else:
-            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+            repo_dir = _git_clone_and_install_single(repo, branch, depth,
+                                                     parent_dir, http_proxy,
                                                      update_requirements=True)
 
     os.environ = old_environ
@@ -574,7 +591,8 @@ def _git_ensure_key_exists(key, keys):
         error_out('openstack-origin-git key \'{}\' is missing'.format(key))
 
 
-def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements):
+def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
+                                  update_requirements):
     """
     Clone and install a single git repository.
     """
@@ -587,7 +605,8 @@ def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements)
 
     if not os.path.exists(dest_dir):
         juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
-        repo_dir = install_remote(repo, dest=parent_dir, branch=branch)
+        repo_dir = install_remote(repo, dest=parent_dir, branch=branch,
+                                  depth=depth)
     else:
         repo_dir = dest_dir
 
@@ -598,7 +617,12 @@ def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements)
         _git_update_requirements(repo_dir, requirements_dir)
 
     juju_log('Installing git repo from dir: {}'.format(repo_dir))
-    pip_install(repo_dir)
+    if http_proxy:
+        pip_install(repo_dir, proxy=http_proxy,
+                    venv=os.path.join(parent_dir, 'venv'))
+    else:
+        pip_install(repo_dir,
+                    venv=os.path.join(parent_dir, 'venv'))
 
     return repo_dir
 
@@ -621,16 +645,27 @@ def _git_update_requirements(package_dir, reqs_dir):
     os.chdir(orig_dir)
 
 
+def git_pip_venv_dir(projects_yaml):
+    """
+    Return the pip virtualenv path.
+    """
+    parent_dir = '/mnt/openstack-git'
+
+    projects = _git_yaml_load(projects_yaml)
+
+    if 'directory' in projects.keys():
+        parent_dir = projects['directory']
+
+    return os.path.join(parent_dir, 'venv')
+
+
 def git_src_dir(projects_yaml, project):
     """
     Return the directory where the specified project's source is located.
     """
     parent_dir = '/mnt/openstack-git'
 
-    if not projects_yaml:
-        return
-
-    projects = yaml.load(projects_yaml)
+    projects = _git_yaml_load(projects_yaml)
 
     if 'directory' in projects.keys():
         parent_dir = projects['directory']
@@ -640,3 +675,15 @@ def git_src_dir(projects_yaml, project):
             return os.path.join(parent_dir, os.path.basename(p['repository']))
 
     return None
+
+
+def git_yaml_value(projects_yaml, key):
+    """
+    Return the value in projects_yaml for the specified key.
+    """
+    projects = _git_yaml_load(projects_yaml)
+
+    if key in projects.keys():
+        return projects[key]
+
+    return None
diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py
index 8659516..07b0c1d 100644
--- a/hooks/charmhelpers/contrib/python/packages.py
+++ b/hooks/charmhelpers/contrib/python/packages.py
@@ -17,8 +17,11 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
+import subprocess
+
 from charmhelpers.fetch import apt_install, apt_update
-from charmhelpers.core.hookenv import log
+from charmhelpers.core.hookenv import charm_dir, log
 
 try:
     from pip import main as pip_execute
@@ -51,11 +54,15 @@ def pip_install_requirements(requirements, **options):
     pip_execute(command)
 
 
-def pip_install(package, fatal=False, upgrade=False, **options):
+def pip_install(package, fatal=False, upgrade=False, venv=None, **options):
     """Install a python package"""
-    command = ["install"]
+    if venv:
+        venv_python = os.path.join(venv, 'bin/pip')
+        command = [venv_python, "install"]
+    else:
+        command = ["install"]
 
-    available_options = ('proxy', 'src', 'log', "index-url", )
+    available_options = ('proxy', 'src', 'log', 'index-url', )
     for option in parse_options(options, available_options):
         command.append(option)
 
@@ -69,7 +76,10 @@ def pip_install(package, fatal=False, upgrade=False, **options):
 
     log("Installing {} package with options: {}".format(package,
                                                         command))
-    pip_execute(command)
+    if venv:
+        subprocess.check_call(command)
+    else:
+        pip_execute(command)
 
 
 def pip_uninstall(package, **options):
@@ -94,3 +104,16 @@ def pip_list():
     """Returns the list of current python installed packages
     """
     return pip_execute(["list"])
+
+
+def pip_create_virtualenv(path=None):
+    """Create an isolated Python environment."""
+    apt_install('python-virtualenv')
+
+    if path:
+        venv_path = path
+    else:
+        venv_path = os.path.join(charm_dir(), 'venv')
+
+    if not os.path.exists(venv_path):
+        subprocess.check_call(['virtualenv', venv_path])
diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py
index 8c97eac..117429f 100644
--- a/hooks/charmhelpers/core/hookenv.py
+++ b/hooks/charmhelpers/core/hookenv.py
@@ -28,6 +28,7 @@ import yaml
 import subprocess
 import sys
 import errno
+import tempfile
 from subprocess import CalledProcessError
 
 import six
@@ -362,14 +363,34 @@ def relation_set(relation_id=None, relation_settings=None, **kwargs):
     """Set relation information for the current unit"""
     relation_settings = relation_settings if relation_settings else {}
     relation_cmd_line = ['relation-set']
+    accepts_file = "--file" in subprocess.check_output(
+        relation_cmd_line + ["--help"], universal_newlines=True)
     if relation_id is not None:
         relation_cmd_line.extend(('-r', relation_id))
-    for k, v in (list(relation_settings.items()) + list(kwargs.items())):
-        if v is None:
-            relation_cmd_line.append('{}='.format(k))
-        else:
-            relation_cmd_line.append('{}={}'.format(k, v))
-    subprocess.check_call(relation_cmd_line)
+    settings = relation_settings.copy()
+    settings.update(kwargs)
+    for key, value in settings.items():
+        # Force value to be a string: it always should, but some call
+        # sites pass in things like dicts or numbers.
+        if value is not None:
+            settings[key] = "{}".format(value)
+    if accepts_file:
+        # --file was introduced in Juju 1.23.2. Use it by default if
+        # available, since otherwise we'll break if the relation data is
+        # too big. Ideally we should tell relation-set to read the data from
+        # stdin, but that feature is broken in 1.23.2: Bug #1454678.
+        with tempfile.NamedTemporaryFile(delete=False) as settings_file:
+            settings_file.write(yaml.safe_dump(settings).encode("utf-8"))
+        subprocess.check_call(
+            relation_cmd_line + ["--file", settings_file.name])
+        os.remove(settings_file.name)
+    else:
+        for key, value in settings.items():
+            if value is None:
+                relation_cmd_line.append('{}='.format(key))
+            else:
+                relation_cmd_line.append('{}={}'.format(key, value))
+        subprocess.check_call(relation_cmd_line)
     # Flush cache of any relation-gets for local unit
     flush(local_unit())
 
@@ -606,67 +627,6 @@ def charm_dir():
     return os.environ.get('CHARM_DIR')
 
 
-def translate_exc(from_exc, to_exc):
-    def inner_translate_exc1(f):
-        def inner_translate_exc2(*args, **kwargs):
-            try:
-                return f(*args, **kwargs)
-            except from_exc:
-                raise to_exc
-
-        return inner_translate_exc2
-
-    return inner_translate_exc1
-
-
-@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
-def is_leader():
-    """Does the current unit hold the juju leadership
-
-    Uses juju to determine whether the current unit is the leader of its peers
-    """
-    try:
-        leader = json.loads(
-            subprocess.check_output(['is-leader', '--format=json'])
-        )
-        return (leader is True)
-    except ValueError:
-        raise NotImplementedError
-
-
-@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
-def leader_get(attribute=None):
-    """Juju leader get value(s)"""
-    cmd = ['leader-get', '--format=json'] + [attribute or '-']
-    try:
-        ret = json.loads(subprocess.check_output(cmd).decode('UTF-8'))
-        log("Juju leader-get '%s' = '%s'" % (attribute, ret), level=DEBUG)
-        return ret
-    except ValueError:
-        return None
-    except CalledProcessError as e:
-        if e.returncode == 2:
-            return None
-
-        raise
-
-
-@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
-def leader_set(settings=None, **kwargs):
-    """Juju leader set value(s)"""
-    log("Juju leader-set '%s'" % (settings), level=DEBUG)
-    cmd = ['leader-set']
-    settings = settings or {}
-    settings.update(kwargs)
-    for k, v in settings.iteritems():
-        if v is None:
-            cmd.append('{}='.format(k))
-        else:
-            cmd.append('{}={}'.format(k, v))
-
-    subprocess.check_call(cmd)
-
-
 @cached
 def action_get(key=None):
     """Gets the value of an action parameter, or all key/value param pairs"""
@@ -737,3 +697,48 @@ def status_get():
             return 'unknown'
         else:
             raise
+
+
+def translate_exc(from_exc, to_exc):
+    def inner_translate_exc1(f):
+        def inner_translate_exc2(*args, **kwargs):
+            try:
+                return f(*args, **kwargs)
+            except from_exc:
+                raise to_exc
+
+        return inner_translate_exc2
+
+    return inner_translate_exc1
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def is_leader():
+    """Does the current unit hold the juju leadership
+
+    Uses juju to determine whether the current unit is the leader of its peers
+    """
+    cmd = ['is-leader', '--format=json']
+    return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def leader_get(attribute=None):
+    """Juju leader get value(s)"""
+    cmd = ['leader-get', '--format=json'] + [attribute or '-']
+    return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def leader_set(settings=None, **kwargs):
+    """Juju leader set value(s)"""
+    log("Juju leader-set '%s'" % (settings), level=DEBUG)
+    cmd = ['leader-set']
+    settings = settings or {}
+    settings.update(kwargs)
+    for k, v in settings.iteritems():
+        if v is None:
+            cmd.append('{}='.format(k))
+        else:
+            cmd.append('{}={}'.format(k, v))
+    subprocess.check_call(cmd)
diff --git a/hooks/charmhelpers/core/services/base.py b/hooks/charmhelpers/core/services/base.py
index 2ff5338..98d344e 100644
--- a/hooks/charmhelpers/core/services/base.py
+++ b/hooks/charmhelpers/core/services/base.py
@@ -15,8 +15,8 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import re
 import json
+from inspect import getargspec
 from collections import Iterable, OrderedDict
 
 from charmhelpers.core import host
@@ -132,8 +132,8 @@ class ServiceManager(object):
         if hook_name == 'stop':
             self.stop_services()
         else:
-            self.provide_data()
             self.reconfigure_services()
+            self.provide_data()
         cfg = hookenv.config()
         if cfg.implicit_save:
             cfg.save()
@@ -145,15 +145,36 @@ class ServiceManager(object):
         A provider must have a `name` attribute, which indicates which relation
         to set data on, and a `provide_data()` method, which returns a dict of
         data to set.
+
+        The `provide_data()` method can optionally accept two parameters:
+
+          * ``remote_service`` The name of the remote service that the data will
+            be provided to.  The `provide_data()` method will be called once
+            for each connected service (not unit).  This allows the method to
+            tailor its data to the given service.
+          * ``service_ready`` Whether or not the service definition had all of
+            its requirements met, and thus the ``data_ready`` callbacks run.
+
+        Note that the ``provided_data`` methods are now called **after** the
+        ``data_ready`` callbacks are run.  This gives the ``data_ready`` callbacks
+        a chance to generate any data necessary for the providing to the remote
+        services.
         """
-        hook_name = hookenv.hook_name()
-        for service in self.services.values():
+        for service_name, service in self.services.items():
+            service_ready = self.is_ready(service_name)
             for provider in service.get('provided_data', []):
-                if re.match(r'{}-relation-(joined|changed)'.format(provider.name), hook_name):
-                    data = provider.provide_data()
-                    _ready = provider._is_ready(data) if hasattr(provider, '_is_ready') else data
-                    if _ready:
-                        hookenv.relation_set(None, data)
+                for relid in hookenv.relation_ids(provider.name):
+                    units = hookenv.related_units(relid)
+                    if not units:
+                        continue
+                    remote_service = units[0].split('/')[0]
+                    argspec = getargspec(provider.provide_data)
+                    if len(argspec.args) > 1:
+                        data = provider.provide_data(remote_service, service_ready)
+                    else:
+                        data = provider.provide_data()
+                    if data:
+                        hookenv.relation_set(relid, data)
 
     def reconfigure_services(self, *service_names):
         """
diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py
index 93aae87..ddc25b7 100644
--- a/hooks/charmhelpers/fetch/giturl.py
+++ b/hooks/charmhelpers/fetch/giturl.py
@@ -45,14 +45,16 @@ class GitUrlFetchHandler(BaseFetchHandler):
         else:
             return True
 
-    def clone(self, source, dest, branch):
+    def clone(self, source, dest, branch, depth=None):
         if not self.can_handle(source):
             raise UnhandledSource("Cannot handle {}".format(source))
 
-        repo = Repo.clone_from(source, dest)
-        repo.git.checkout(branch)
+        if depth:
+            Repo.clone_from(source, dest, branch=branch, depth=depth)
+        else:
+            Repo.clone_from(source, dest, branch=branch)
 
-    def install(self, source, branch="master", dest=None):
+    def install(self, source, branch="master", dest=None, depth=None):
         url_parts = self.parse_url(source)
         branch_name = url_parts.path.strip("/").split("/")[-1]
         if dest:
@@ -63,7 +65,7 @@ class GitUrlFetchHandler(BaseFetchHandler):
         if not os.path.exists(dest_dir):
             mkdir(dest_dir, perms=0o755)
         try:
-            self.clone(source, dest_dir, branch)
+            self.clone(source, dest_dir, branch, depth)
         except GitCommandError as e:
             raise UnhandledSource(e.message)
         except OSError as e: