From efe50867cd922aebcfc47d6af9c99a42a5f6f3c9 Mon Sep 17 00:00:00 2001
From: James Page <james.page@ubuntu.com>
Date: Thu, 12 Mar 2015 12:44:14 +0000
Subject: [PATCH 01/88] Resync helpers

---
 .../charmhelpers/contrib/openstack/context.py | 47 +++++++++++++
 .../charmhelpers/contrib/openstack/neutron.py | 70 +++++++++++++++++++
 hooks/charmhelpers/core/hookenv.py            | 26 +++++++
 hooks/charmhelpers/core/host.py               |  6 +-
 4 files changed, 148 insertions(+), 1 deletion(-)

diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py
index 2d9a95cd..f69d91be 100644
--- a/hooks/charmhelpers/contrib/openstack/context.py
+++ b/hooks/charmhelpers/contrib/openstack/context.py
@@ -16,6 +16,7 @@
 
 import json
 import os
+import re
 import time
 from base64 import b64decode
 from subprocess import check_call
@@ -48,6 +49,8 @@ from charmhelpers.core.hookenv import (
 from charmhelpers.core.sysctl import create as sysctl_create
 
 from charmhelpers.core.host import (
+    list_nics,
+    get_nic_hwaddr,
     mkdir,
     write_file,
 )
@@ -67,10 +70,12 @@ from charmhelpers.contrib.openstack.neutron import (
 )
 from charmhelpers.contrib.network.ip import (
     get_address_in_network,
+    get_ipv4_addr,
     get_ipv6_addr,
     get_netmask_for_address,
     format_ipv6_addr,
     is_address_in_network,
+    is_bridge_member,
 )
 from charmhelpers.contrib.openstack.utils import get_host_ip
 
@@ -883,6 +888,48 @@ class NeutronContext(OSContextGenerator):
         return ctxt
 
 
+class NeutronPortContext(OSContextGenerator):
+    NIC_PREFIXES = ['eth', 'bond']
+
+    def resolve_ports(self, ports):
+        """Resolve NICs not yet bound to bridge(s)
+
+        If hwaddress provided then returns resolved hwaddress otherwise NIC.
+        """
+        if not ports:
+            return None
+
+        hwaddr_to_nic = {}
+        hwaddr_to_ip = {}
+        for nic in list_nics(self.NIC_PREFIXES):
+            hwaddr = get_nic_hwaddr(nic)
+            hwaddr_to_nic[hwaddr] = nic
+            addresses = get_ipv4_addr(nic, fatal=False)
+            addresses += get_ipv6_addr(iface=nic, fatal=False)
+            hwaddr_to_ip[hwaddr] = addresses
+
+        resolved = []
+        mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I)
+        for entry in ports:
+            if re.match(mac_regex, entry):
+                # NIC is in known NICs and does NOT hace an IP address
+                if entry in hwaddr_to_nic and not hwaddr_to_ip[entry]:
+                    # If the nic is part of a bridge then don't use it
+                    if is_bridge_member(hwaddr_to_nic[entry]):
+                        continue
+
+                    # Entry is a MAC address for a valid interface that doesn't
+                    # have an IP address assigned yet.
+                    resolved.append(hwaddr_to_nic[entry])
+            else:
+                # If the passed entry is not a MAC address, assume it's a valid
+                # interface, and that the user put it there on purpose (we can
+                # trust it to be the real external network).
+                resolved.append(entry)
+
+        return resolved
+
+
 class OSConfigFlagContext(OSContextGenerator):
     """Provides support for user-defined config flags.
 
diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py
index 902757fe..f8851050 100644
--- a/hooks/charmhelpers/contrib/openstack/neutron.py
+++ b/hooks/charmhelpers/contrib/openstack/neutron.py
@@ -16,6 +16,7 @@
 
 # Various utilies for dealing with Neutron and the renaming from Quantum.
 
+import six
 from subprocess import check_output
 
 from charmhelpers.core.hookenv import (
@@ -237,3 +238,72 @@ def network_manager():
     else:
         # ensure accurate naming for all releases post-H
         return 'neutron'
+
+
+def parse_mappings(mappings):
+    parsed = {}
+    if mappings:
+        mappings = mappings.split(' ')
+        for m in mappings:
+            p = m.partition(':')
+            if p[1] == ':':
+                parsed[p[0].strip()] = p[2].strip()
+
+    return parsed
+
+
+def parse_bridge_mappings(mappings):
+    """Parse bridge mappings.
+
+    Mappings must be a space-delimited list of provider:bridge mappings.
+
+    Returns dict of the form {provider:bridge}.
+    """
+    return parse_mappings(mappings)
+
+
+def parse_data_port_mappings(mappings, default_bridge='br-data'):
+    """Parse data port mappings.
+
+    Mappings must be a space-delimited list of bridge:port mappings.
+
+    Returns dict of the form {bridge:port}.
+    """
+    _mappings = parse_mappings(mappings)
+    if not _mappings:
+        if not mappings:
+            return {}
+
+        # For backwards-compatibility we need to support port-only provided in
+        # config.
+        _mappings = {default_bridge: mappings.split(' ')[0]}
+
+    bridges = _mappings.keys()
+    ports = _mappings.values()
+    if len(set(bridges)) != len(bridges):
+        raise Exception("It is not allowed to have more than one port "
+                        "configured on the same bridge")
+
+    if len(set(ports)) != len(ports):
+        raise Exception("It is not allowed to have the same port configured "
+                        "on more than one bridge")
+
+    return _mappings
+
+
+def parse_vlan_range_mappings(mappings):
+    """Parse vlan range mappings.
+
+    Mappings must be a space-delimited list of provider:start:end mappings.
+
+    Returns dict of the form {provider: (start, end)}.
+    """
+    _mappings = parse_mappings(mappings)
+    if not _mappings:
+        return {}
+
+    mappings = {}
+    for p, r in six.iteritems(_mappings):
+        mappings[p] = tuple(r.split(':'))
+
+    return mappings
diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py
index cf552b39..715dd4c5 100644
--- a/hooks/charmhelpers/core/hookenv.py
+++ b/hooks/charmhelpers/core/hookenv.py
@@ -566,3 +566,29 @@ class Hooks(object):
 def charm_dir():
     """Return the root directory of the current charm"""
     return os.environ.get('CHARM_DIR')
+
+
+@cached
+def action_get(key=None):
+    """Gets the value of an action parameter, or all key/value param pairs"""
+    cmd = ['action-get']
+    if key is not None:
+        cmd.append(key)
+    cmd.append('--format=json')
+    action_data = json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+    return action_data
+
+
+def action_set(values):
+    """Sets the values to be returned after the action finishes"""
+    cmd = ['action-set']
+    for k, v in list(values.items()):
+        cmd.append('{}={}'.format(k, v))
+    subprocess.check_call(cmd)
+
+
+def action_fail(message):
+    """Sets the action status to failed and sets the error message.
+
+    The results set by action_set are preserved."""
+    subprocess.check_call(['action-fail', message])
diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py
index b771c611..830822af 100644
--- a/hooks/charmhelpers/core/host.py
+++ b/hooks/charmhelpers/core/host.py
@@ -339,12 +339,16 @@ def lsb_release():
 def pwgen(length=None):
     """Generate a random pasword."""
     if length is None:
+        # A random length is ok to use a weak PRNG
         length = random.choice(range(35, 45))
     alphanumeric_chars = [
         l for l in (string.ascii_letters + string.digits)
         if l not in 'l0QD1vAEIOUaeiou']
+    # Use a crypto-friendly PRNG (e.g. /dev/urandom) for making the
+    # actual password
+    random_generator = random.SystemRandom()
     random_chars = [
-        random.choice(alphanumeric_chars) for _ in range(length)]
+        random_generator.choice(alphanumeric_chars) for _ in range(length)]
     return(''.join(random_chars))
 
 

From b3b8dbdb5a98ad46905171346bbfcbd83a27f51a Mon Sep 17 00:00:00 2001
From: Edward Hope-Morley <edward.hope-morley@canonical.com>
Date: Thu, 26 Mar 2015 10:37:01 -0700
Subject: [PATCH 02/88] synced charm-helpers

---
 charm-helpers-hooks.yaml                      |   2 +-
 charm-helpers-tests.yaml                      |   2 +-
 .../charmhelpers/contrib/hahelpers/cluster.py |  14 +-
 .../charmhelpers/contrib/openstack/context.py | 158 +++++++++++++++++-
 .../templates/section-keystone-authtoken      |   9 +
 .../openstack/templates/section-rabbitmq-oslo |  22 +++
 hooks/charmhelpers/core/hookenv.py            |  72 ++++++++
 hooks/charmhelpers/core/services/helpers.py   |   4 +-
 8 files changed, 275 insertions(+), 8 deletions(-)
 create mode 100644 hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken
 create mode 100644 hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo

diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml
index 917cf211..7216e1bc 100644
--- a/charm-helpers-hooks.yaml
+++ b/charm-helpers-hooks.yaml
@@ -1,4 +1,4 @@
-branch: lp:charm-helpers
+branch: lp:~le-charmers/charm-helpers/leadership-election
 destination: hooks/charmhelpers
 include:
     - core
diff --git a/charm-helpers-tests.yaml b/charm-helpers-tests.yaml
index 48b12f6f..cad8e7cb 100644
--- a/charm-helpers-tests.yaml
+++ b/charm-helpers-tests.yaml
@@ -1,4 +1,4 @@
-branch: lp:charm-helpers
+branch: lp:~le-charmers/charm-helpers/leadership-election
 destination: tests/charmhelpers
 include:
     - contrib.amulet
diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py
index 9333efc3..ebec3790 100644
--- a/hooks/charmhelpers/contrib/hahelpers/cluster.py
+++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py
@@ -44,6 +44,7 @@ from charmhelpers.core.hookenv import (
     ERROR,
     WARNING,
     unit_get,
+    is_leader as juju_is_leader
 )
 from charmhelpers.core.decorators import (
     retry_on_exception,
@@ -66,12 +67,21 @@ def is_elected_leader(resource):
     Returns True if the charm executing this is the elected cluster leader.
 
     It relies on two mechanisms to determine leadership:
-        1. If the charm is part of a corosync cluster, call corosync to
+        1. If juju is sufficiently new and leadership election is supported,
+        the is_leader command will be used.
+        2. If the charm is part of a corosync cluster, call corosync to
         determine leadership.
-        2. If the charm is not part of a corosync cluster, the leader is
+        3. If the charm is not part of a corosync cluster, the leader is
         determined as being "the alive unit with the lowest unit numer". In
         other words, the oldest surviving unit.
     """
+    try:
+        return juju_is_leader()
+    except NotImplementedError:
+        log('Juju leadership election feature not enabled'
+            ', using fallback support',
+            level=WARNING)
+
     if is_clustered():
         if not is_crm_leader(resource):
             log('Deferring action to CRM leader.', level=INFO)
diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py
index f69d91be..45e65790 100644
--- a/hooks/charmhelpers/contrib/openstack/context.py
+++ b/hooks/charmhelpers/contrib/openstack/context.py
@@ -47,6 +47,7 @@ from charmhelpers.core.hookenv import (
 )
 
 from charmhelpers.core.sysctl import create as sysctl_create
+from charmhelpers.core.strutils import bool_from_string
 
 from charmhelpers.core.host import (
     list_nics,
@@ -67,6 +68,11 @@ from charmhelpers.contrib.hahelpers.apache import (
 )
 from charmhelpers.contrib.openstack.neutron import (
     neutron_plugin_attribute,
+    parse_data_port_mappings,
+)
+from charmhelpers.contrib.openstack.ip import (
+    resolve_address,
+    INTERNAL,
 )
 from charmhelpers.contrib.network.ip import (
     get_address_in_network,
@@ -78,7 +84,6 @@ from charmhelpers.contrib.network.ip import (
     is_bridge_member,
 )
 from charmhelpers.contrib.openstack.utils import get_host_ip
-
 CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
 ADDRESS_TYPES = ['admin', 'internal', 'public']
 
@@ -732,7 +737,14 @@ class ApacheSSLContext(OSContextGenerator):
                 'endpoints': [],
                 'ext_ports': []}
 
-        for cn in self.canonical_names():
+        cns = self.canonical_names()
+        if cns:
+            for cn in cns:
+                self.configure_cert(cn)
+        else:
+            # Expect cert/key provided in config (currently assumed that ca
+            # uses ip for cn)
+            cn = resolve_address(endpoint_type=INTERNAL)
             self.configure_cert(cn)
 
         addresses = self.get_network_addresses()
@@ -1151,3 +1163,145 @@ class SysctlContext(OSContextGenerator):
             sysctl_create(sysctl_dict,
                           '/etc/sysctl.d/50-{0}.conf'.format(charm_name()))
         return {'sysctl': sysctl_dict}
+
+
+class NeutronAPIContext(OSContextGenerator):
+    '''
+    Inspects current neutron-plugin-api relation for neutron settings. Return
+    defaults if it is not present.
+    '''
+    interfaces = ['neutron-plugin-api']
+
+    def __call__(self):
+        self.neutron_defaults = {
+            'l2_population': {
+                'rel_key': 'l2-population',
+                'default': False,
+            },
+            'overlay_network_type': {
+                'rel_key': 'overlay-network-type',
+                'default': 'gre',
+            },
+            'neutron_security_groups': {
+                'rel_key': 'neutron-security-groups',
+                'default': False,
+            },
+            'network_device_mtu': {
+                'rel_key': 'network-device-mtu',
+                'default': None,
+            },
+            'enable_dvr': {
+                'rel_key': 'enable-dvr',
+                'default': False,
+            },
+            'enable_l3ha': {
+                'rel_key': 'enable-l3ha',
+                'default': False,
+            },
+        }
+        ctxt = self.get_neutron_options({})
+        for rid in relation_ids('neutron-plugin-api'):
+            for unit in related_units(rid):
+                rdata = relation_get(rid=rid, unit=unit)
+                if 'l2-population' in rdata:
+                    ctxt.update(self.get_neutron_options(rdata))
+
+        return ctxt
+
+    def get_neutron_options(self, rdata):
+        settings = {}
+        for nkey in self.neutron_defaults.keys():
+            defv = self.neutron_defaults[nkey]['default']
+            rkey = self.neutron_defaults[nkey]['rel_key']
+            if rkey in rdata.keys():
+                if type(defv) is bool:
+                    settings[nkey] = bool_from_string(rdata[rkey])
+                else:
+                    settings[nkey] = rdata[rkey]
+            else:
+                settings[nkey] = defv
+        return settings
+
+
+class ExternalPortContext(NeutronPortContext):
+
+    def __call__(self):
+        ctxt = {}
+        ports = config('ext-port')
+        if ports:
+            ports = [p.strip() for p in ports.split()]
+            ports = self.resolve_ports(ports)
+            if ports:
+                ctxt = {"ext_port": ports[0]}
+                napi_settings = NeutronAPIContext()()
+                mtu = napi_settings.get('network_device_mtu')
+                if mtu:
+                    ctxt['ext_port_mtu'] = mtu
+
+        return ctxt
+
+
+class DataPortContext(NeutronPortContext):
+
+    def __call__(self):
+        ports = config('data-port')
+        if ports:
+            portmap = parse_data_port_mappings(ports)
+            ports = portmap.values()
+            resolved = self.resolve_ports(ports)
+            normalized = {get_nic_hwaddr(port): port for port in resolved
+                          if port not in ports}
+            normalized.update({port: port for port in resolved
+                               if port in ports})
+            if resolved:
+                return {bridge: normalized[port] for bridge, port in
+                        six.iteritems(portmap) if port in normalized.keys()}
+
+        return None
+
+
+class PhyNICMTUContext(DataPortContext):
+
+    def __call__(self):
+        ctxt = {}
+        mappings = super(PhyNICMTUContext, self).__call__()
+        if mappings and mappings.values():
+            ports = mappings.values()
+            napi_settings = NeutronAPIContext()()
+            mtu = napi_settings.get('network_device_mtu')
+            if mtu:
+                ctxt["devs"] = '\\n'.join(ports)
+                ctxt['mtu'] = mtu
+
+        return ctxt
+
+
+class NetworkServiceContext(OSContextGenerator):
+
+    def __init__(self, rel_name='quantum-network-service'):
+        self.rel_name = rel_name
+        self.interfaces = [rel_name]
+
+    def __call__(self):
+        for rid in relation_ids(self.rel_name):
+            for unit in related_units(rid):
+                rdata = relation_get(rid=rid, unit=unit)
+                ctxt = {
+                    'keystone_host': rdata.get('keystone_host'),
+                    'service_port': rdata.get('service_port'),
+                    'auth_port': rdata.get('auth_port'),
+                    'service_tenant': rdata.get('service_tenant'),
+                    'service_username': rdata.get('service_username'),
+                    'service_password': rdata.get('service_password'),
+                    'quantum_host': rdata.get('quantum_host'),
+                    'quantum_port': rdata.get('quantum_port'),
+                    'quantum_url': rdata.get('quantum_url'),
+                    'region': rdata.get('region'),
+                    'service_protocol':
+                    rdata.get('service_protocol') or 'http',
+                    'auth_protocol':
+                    rdata.get('auth_protocol') or 'http',
+                }
+                if context_complete(ctxt):
+                    return ctxt
+        return {}
diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken
new file mode 100644
index 00000000..2a37edd5
--- /dev/null
+++ b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken
@@ -0,0 +1,9 @@
+{% if auth_host -%}
+[keystone_authtoken]
+identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/{{ auth_admin_prefix }}
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }}
+admin_tenant_name = {{ admin_tenant_name }}
+admin_user = {{ admin_user }}
+admin_password = {{ admin_password }}
+signing_dir = {{ signing_dir }}
+{% endif -%}
diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo b/hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo
new file mode 100644
index 00000000..b444c9c9
--- /dev/null
+++ b/hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo
@@ -0,0 +1,22 @@
+{% if rabbitmq_host or rabbitmq_hosts -%}
+[oslo_messaging_rabbit]
+rabbit_userid = {{ rabbitmq_user }}
+rabbit_virtual_host = {{ rabbitmq_virtual_host }}
+rabbit_password = {{ rabbitmq_password }}
+{% if rabbitmq_hosts -%}
+rabbit_hosts = {{ rabbitmq_hosts }}
+{% if rabbitmq_ha_queues -%}
+rabbit_ha_queues = True
+rabbit_durable_queues = False
+{% endif -%}
+{% else -%}
+rabbit_host = {{ rabbitmq_host }}
+{% endif -%}
+{% if rabbit_ssl_port -%}
+rabbit_use_ssl = True
+rabbit_port = {{ rabbit_ssl_port }}
+{% if rabbit_ssl_ca -%}
+kombu_ssl_ca_certs = {{ rabbit_ssl_ca }}
+{% endif -%}
+{% endif -%}
+{% endif -%}
diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py
index 715dd4c5..60820cc4 100644
--- a/hooks/charmhelpers/core/hookenv.py
+++ b/hooks/charmhelpers/core/hookenv.py
@@ -352,6 +352,17 @@ def relation_set(relation_id=None, relation_settings=None, **kwargs):
     flush(local_unit())
 
 
+def relation_clear(r_id=None):
+    ''' Clears any relation data already set on relation r_id '''
+    settings = relation_get(rid=r_id,
+                            unit=local_unit())
+    for setting in settings:
+        if setting not in ['public-address', 'private-address']:
+            settings[setting] = None
+    relation_set(relation_id=r_id,
+                 **settings)
+
+
 @cached
 def relation_ids(reltype=None):
     """A list of relation_ids"""
@@ -568,6 +579,67 @@ def charm_dir():
     return os.environ.get('CHARM_DIR')
 
 
+def translate_exc(from_exc, to_exc):
+    def inner_translate_exc1(f):
+        def inner_translate_exc2(*args, **kwargs):
+            try:
+                return f(*args, **kwargs)
+            except from_exc:
+                raise to_exc
+
+        return inner_translate_exc2
+
+    return inner_translate_exc1
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def is_leader():
+    """Does the current unit hold the juju leadership
+
+    Uses juju to determine whether the current unit is the leader of its peers
+    """
+    try:
+        leader = json.loads(
+            subprocess.check_output(['is-leader', '--format=json'])
+        )
+        return (leader is True)
+    except ValueError:
+        raise NotImplementedError
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def leader_get(attribute=None):
+    """Juju leader get value(s)"""
+    cmd = ['leader-get', '--format=json'] + [attribute or '-']
+    try:
+        ret = json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+        log("Juju leader-get '%s' = '%s'" % (attribute, ret), level=DEBUG)
+        return ret
+    except ValueError:
+        return None
+    except CalledProcessError as e:
+        if e.returncode == 2:
+            return None
+
+        raise
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def leader_set(settings=None, **kwargs):
+    """Juju leader set value(s)"""
+    log("Juju leader-set '%s'" % (settings), level=DEBUG)
+    cmd = ['leader-set']
+    settings = settings or {}
+    settings.update(kwargs)
+    for k, v in settings.iteritems():
+        if v is None:
+            cmd.append('{}='.format(k))
+        else:
+            cmd.append('{}={}'.format(k, v))
+
+    subprocess.check_call(cmd)
+
+
 @cached
 def action_get(key=None):
     """Gets the value of an action parameter, or all key/value param pairs"""
diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py
index 15b21664..3eb5fb44 100644
--- a/hooks/charmhelpers/core/services/helpers.py
+++ b/hooks/charmhelpers/core/services/helpers.py
@@ -139,7 +139,7 @@ class MysqlRelation(RelationContext):
 
     def __init__(self, *args, **kwargs):
         self.required_keys = ['host', 'user', 'password', 'database']
-        super(HttpRelation).__init__(self, *args, **kwargs)
+        RelationContext.__init__(self, *args, **kwargs)
 
 
 class HttpRelation(RelationContext):
@@ -154,7 +154,7 @@ class HttpRelation(RelationContext):
 
     def __init__(self, *args, **kwargs):
         self.required_keys = ['host', 'port']
-        super(HttpRelation).__init__(self, *args, **kwargs)
+        RelationContext.__init__(self, *args, **kwargs)
 
     def provide_data(self):
         return {

From a7e55540075d6518101c039672404788c7f14208 Mon Sep 17 00:00:00 2001
From: Edward Hope-Morley <edward.hope-morley@canonical.com>
Date: Wed, 1 Apr 2015 14:50:30 +0100
Subject: [PATCH 03/88] synced l/e ch

---
 .../contrib/openstack/amulet/deployment.py    |  27 ++-
 .../charmhelpers/contrib/openstack/context.py |   9 +-
 .../contrib/openstack/templates/git.upstart   |  13 ++
 .../templates/{zeromq => section-zeromq}      |   4 +-
 hooks/charmhelpers/contrib/openstack/utils.py | 179 ++++++++++++------
 hooks/charmhelpers/core/unitdata.py           |   2 +-
 .../contrib/openstack/amulet/deployment.py    |  27 ++-
 7 files changed, 188 insertions(+), 73 deletions(-)
 create mode 100644 hooks/charmhelpers/contrib/openstack/templates/git.upstart
 rename hooks/charmhelpers/contrib/openstack/templates/{zeromq => section-zeromq} (66%)

diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
index 0cfeaa4c..0e0db566 100644
--- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -15,6 +15,7 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import six
+from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
@@ -100,12 +101,34 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            """
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
-         self.trusty_icehouse) = range(6)
+         self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8)
         releases = {
             ('precise', None): self.precise_essex,
             ('precise', 'cloud:precise-folsom'): self.precise_folsom,
             ('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
             ('precise', 'cloud:precise-havana'): self.precise_havana,
             ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
-            ('trusty', None): self.trusty_icehouse}
+            ('trusty', None): self.trusty_icehouse,
+            ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
+            ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo}
         return releases[(self.series, self.openstack)]
+
+    def _get_openstack_release_string(self):
+        """Get openstack release string.
+
+           Return a string representing the openstack release.
+           """
+        releases = OrderedDict([
+            ('precise', 'essex'),
+            ('quantal', 'folsom'),
+            ('raring', 'grizzly'),
+            ('saucy', 'havana'),
+            ('trusty', 'icehouse'),
+            ('utopic', 'juno'),
+            ('vivid', 'kilo'),
+        ])
+        if self.openstack:
+            os_origin = self.openstack.split(':')[1]
+            return os_origin.split('%s-' % self.series)[1].split('/')[0]
+        else:
+            return releases[self.series]
diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py
index 45e65790..dd51bfbb 100644
--- a/hooks/charmhelpers/contrib/openstack/context.py
+++ b/hooks/charmhelpers/contrib/openstack/context.py
@@ -320,14 +320,15 @@ def db_ssl(rdata, ctxt, ssl_dir):
 
 
 class IdentityServiceContext(OSContextGenerator):
-    interfaces = ['identity-service']
 
-    def __init__(self, service=None, service_user=None):
+    def __init__(self, service=None, service_user=None, rel_name='identity-service'):
         self.service = service
         self.service_user = service_user
+        self.rel_name = rel_name
+        self.interfaces = [self.rel_name]
 
     def __call__(self):
-        log('Generating template context for identity-service', level=DEBUG)
+        log('Generating template context for ' + self.rel_name, level=DEBUG)
         ctxt = {}
 
         if self.service and self.service_user:
@@ -341,7 +342,7 @@ class IdentityServiceContext(OSContextGenerator):
 
             ctxt['signing_dir'] = cachedir
 
-        for rid in relation_ids('identity-service'):
+        for rid in relation_ids(self.rel_name):
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 serv_host = rdata.get('service_host')
diff --git a/hooks/charmhelpers/contrib/openstack/templates/git.upstart b/hooks/charmhelpers/contrib/openstack/templates/git.upstart
new file mode 100644
index 00000000..da94ad12
--- /dev/null
+++ b/hooks/charmhelpers/contrib/openstack/templates/git.upstart
@@ -0,0 +1,13 @@
+description "{{ service_description }}"
+author "Juju {{ service_name }} Charm <juju@localhost>"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+
+exec start-stop-daemon --start --chuid {{ user_name }} \
+            --chdir {{ start_dir }} --name {{ process_name }} \
+            --exec {{ executable_name }} -- \
+            --config-file={{ config_file }} \
+            --log-file={{ log_file }}
diff --git a/hooks/charmhelpers/contrib/openstack/templates/zeromq b/hooks/charmhelpers/contrib/openstack/templates/section-zeromq
similarity index 66%
rename from hooks/charmhelpers/contrib/openstack/templates/zeromq
rename to hooks/charmhelpers/contrib/openstack/templates/section-zeromq
index 0695eef1..95f1a76c 100644
--- a/hooks/charmhelpers/contrib/openstack/templates/zeromq
+++ b/hooks/charmhelpers/contrib/openstack/templates/section-zeromq
@@ -3,12 +3,12 @@
 rpc_backend = zmq
 rpc_zmq_host = {{ zmq_host }}
 {% if zmq_redis_address -%}
-rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_redis.MatchMakerRedis
+rpc_zmq_matchmaker = redis
 matchmaker_heartbeat_freq = 15
 matchmaker_heartbeat_ttl = 30
 [matchmaker_redis]
 host = {{ zmq_redis_address }}
 {% else -%}
-rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_ring.MatchMakerRing
+rpc_zmq_matchmaker = ring
 {% endif -%}
 {% endif -%}
diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index 4f110c63..78c5e2df 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -30,6 +30,10 @@ import yaml
 
 from charmhelpers.contrib.network import ip
 
+from charmhelpers.core import (
+    unitdata,
+)
+
 from charmhelpers.core.hookenv import (
     config,
     log as juju_log,
@@ -330,6 +334,21 @@ def configure_installation_source(rel):
         error_out("Invalid openstack-release specified: %s" % rel)
 
 
+def config_value_changed(option):
+    """
+    Determine if config value changed since last call to this function.
+    """
+    hook_data = unitdata.HookData()
+    with hook_data():
+        db = unitdata.kv()
+        current = config(option)
+        saved = db.get(option)
+        db.set(option, current)
+        if saved is None:
+            return False
+        return current != saved
+
+
 def save_script_rc(script_path="scripts/scriptrc", **env_vars):
     """
     Write an rc file in the charm-delivered directory containing
@@ -469,82 +488,95 @@ def os_requires_version(ostack_release, pkg):
 
 
 def git_install_requested():
-    """Returns true if openstack-origin-git is specified."""
-    return config('openstack-origin-git') != "None"
+    """
+    Returns true if openstack-origin-git is specified.
+    """
+    return config('openstack-origin-git') is not None
 
 
 requirements_dir = None
 
 
-def git_clone_and_install(file_name, core_project):
-    """Clone/install all OpenStack repos specified in yaml config file."""
-    global requirements_dir
+def git_clone_and_install(projects_yaml, core_project):
+    """
+    Clone/install all specified OpenStack repositories.
 
-    if file_name == "None":
+    The expected format of projects_yaml is:
+        repositories:
+          - {name: keystone,
+             repository: 'git://git.openstack.org/openstack/keystone.git',
+             branch: 'stable/icehouse'}
+          - {name: requirements,
+             repository: 'git://git.openstack.org/openstack/requirements.git',
+             branch: 'stable/icehouse'}
+        directory: /mnt/openstack-git
+
+        The directory key is optional.
+    """
+    global requirements_dir
+    parent_dir = '/mnt/openstack-git'
+
+    if not projects_yaml:
         return
 
-    yaml_file = os.path.join(charm_dir(), file_name)
+    projects = yaml.load(projects_yaml)
+    _git_validate_projects_yaml(projects, core_project)
 
-    # clone/install the requirements project first
-    installed = _git_clone_and_install_subset(yaml_file,
-                                              whitelist=['requirements'])
-    if 'requirements' not in installed:
-        error_out('requirements git repository must be specified')
+    if 'directory' in projects.keys():
+        parent_dir = projects['directory']
 
-    # clone/install all other projects except requirements and the core project
-    blacklist = ['requirements', core_project]
-    _git_clone_and_install_subset(yaml_file, blacklist=blacklist,
-                                  update_requirements=True)
-
-    # clone/install the core project
-    whitelist = [core_project]
-    installed = _git_clone_and_install_subset(yaml_file, whitelist=whitelist,
-                                              update_requirements=True)
-    if core_project not in installed:
-        error_out('{} git repository must be specified'.format(core_project))
+    for p in projects['repositories']:
+        repo = p['repository']
+        branch = p['branch']
+        if p['name'] == 'requirements':
+            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+                                                     update_requirements=False)
+            requirements_dir = repo_dir
+        else:
+            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+                                                     update_requirements=True)
 
 
-def _git_clone_and_install_subset(yaml_file, whitelist=[], blacklist=[],
-                                  update_requirements=False):
-    """Clone/install subset of OpenStack repos specified in yaml config file."""
-    global requirements_dir
-    installed = []
+def _git_validate_projects_yaml(projects, core_project):
+    """
+    Validate the projects yaml.
+    """
+    _git_ensure_key_exists('repositories', projects)
 
-    with open(yaml_file, 'r') as fd:
-        projects = yaml.load(fd)
-        for proj, val in projects.items():
-            # The project subset is chosen based on the following 3 rules:
-            # 1) If project is in blacklist, we don't clone/install it, period.
-            # 2) If whitelist is empty, we clone/install everything else.
-            # 3) If whitelist is not empty, we clone/install everything in the
-            #    whitelist.
-            if proj in blacklist:
-                continue
-            if whitelist and proj not in whitelist:
-                continue
-            repo = val['repository']
-            branch = val['branch']
-            repo_dir = _git_clone_and_install_single(repo, branch,
-                                                     update_requirements)
-            if proj == 'requirements':
-                requirements_dir = repo_dir
-            installed.append(proj)
-    return installed
+    for project in projects['repositories']:
+        _git_ensure_key_exists('name', project.keys())
+        _git_ensure_key_exists('repository', project.keys())
+        _git_ensure_key_exists('branch', project.keys())
+
+    if projects['repositories'][0]['name'] != 'requirements':
+        error_out('{} git repo must be specified first'.format('requirements'))
+
+    if projects['repositories'][-1]['name'] != core_project:
+        error_out('{} git repo must be specified last'.format(core_project))
 
 
-def _git_clone_and_install_single(repo, branch, update_requirements=False):
-    """Clone and install a single git repository."""
-    dest_parent_dir = "/mnt/openstack-git/"
-    dest_dir = os.path.join(dest_parent_dir, os.path.basename(repo))
+def _git_ensure_key_exists(key, keys):
+    """
+    Ensure that key exists in keys.
+    """
+    if key not in keys:
+        error_out('openstack-origin-git key \'{}\' is missing'.format(key))
 
-    if not os.path.exists(dest_parent_dir):
-        juju_log('Host dir not mounted at {}. '
-                 'Creating directory there instead.'.format(dest_parent_dir))
-        os.mkdir(dest_parent_dir)
+
+def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements):
+    """
+    Clone and install a single git repository.
+    """
+    dest_dir = os.path.join(parent_dir, os.path.basename(repo))
+
+    if not os.path.exists(parent_dir):
+        juju_log('Directory already exists at {}. '
+                 'No need to create directory.'.format(parent_dir))
+        os.mkdir(parent_dir)
 
     if not os.path.exists(dest_dir):
         juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
-        repo_dir = install_remote(repo, dest=dest_parent_dir, branch=branch)
+        repo_dir = install_remote(repo, dest=parent_dir, branch=branch)
     else:
         repo_dir = dest_dir
 
@@ -561,16 +593,39 @@ def _git_clone_and_install_single(repo, branch, update_requirements=False):
 
 
 def _git_update_requirements(package_dir, reqs_dir):
-    """Update from global requirements.
+    """
+    Update from global requirements.
 
-       Update an OpenStack git directory's requirements.txt and
-       test-requirements.txt from global-requirements.txt."""
+    Update an OpenStack git directory's requirements.txt and
+    test-requirements.txt from global-requirements.txt.
+    """
     orig_dir = os.getcwd()
     os.chdir(reqs_dir)
-    cmd = "python update.py {}".format(package_dir)
+    cmd = ['python', 'update.py', package_dir]
     try:
-        subprocess.check_call(cmd.split(' '))
+        subprocess.check_call(cmd)
     except subprocess.CalledProcessError:
         package = os.path.basename(package_dir)
         error_out("Error updating {} from global-requirements.txt".format(package))
     os.chdir(orig_dir)
+
+
+def git_src_dir(projects_yaml, project):
+    """
+    Return the directory where the specified project's source is located.
+    """
+    parent_dir = '/mnt/openstack-git'
+
+    if not projects_yaml:
+        return
+
+    projects = yaml.load(projects_yaml)
+
+    if 'directory' in projects.keys():
+        parent_dir = projects['directory']
+
+    for p in projects['repositories']:
+        if p['name'] == project:
+            return os.path.join(parent_dir, os.path.basename(p['repository']))
+
+    return None
diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py
index 3000134a..406a35c5 100644
--- a/hooks/charmhelpers/core/unitdata.py
+++ b/hooks/charmhelpers/core/unitdata.py
@@ -443,7 +443,7 @@ class HookData(object):
         data = hookenv.execution_environment()
         self.conf = conf_delta = self.kv.delta(data['conf'], 'config')
         self.rels = rels_delta = self.kv.delta(data['rels'], 'rels')
-        self.kv.set('env', data['env'])
+        self.kv.set('env', dict(data['env']))
         self.kv.set('unit', data['unit'])
         self.kv.set('relid', data.get('relid'))
         return conf_delta, rels_delta
diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
index 0cfeaa4c..0e0db566 100644
--- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -15,6 +15,7 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import six
+from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
@@ -100,12 +101,34 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            """
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
-         self.trusty_icehouse) = range(6)
+         self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8)
         releases = {
             ('precise', None): self.precise_essex,
             ('precise', 'cloud:precise-folsom'): self.precise_folsom,
             ('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
             ('precise', 'cloud:precise-havana'): self.precise_havana,
             ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
-            ('trusty', None): self.trusty_icehouse}
+            ('trusty', None): self.trusty_icehouse,
+            ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
+            ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo}
         return releases[(self.series, self.openstack)]
+
+    def _get_openstack_release_string(self):
+        """Get openstack release string.
+
+           Return a string representing the openstack release.
+           """
+        releases = OrderedDict([
+            ('precise', 'essex'),
+            ('quantal', 'folsom'),
+            ('raring', 'grizzly'),
+            ('saucy', 'havana'),
+            ('trusty', 'icehouse'),
+            ('utopic', 'juno'),
+            ('vivid', 'kilo'),
+        ])
+        if self.openstack:
+            os_origin = self.openstack.split(':')[1]
+            return os_origin.split('%s-' % self.series)[1].split('/')[0]
+        else:
+            return releases[self.series]

From 38b3b4292d1579e197cc35ff0c3f358686202778 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Wed, 22 Apr 2015 18:39:49 +0000
Subject: [PATCH 04/88] Sync charm-helpers

---
 .../contrib/openstack/amulet/deployment.py           |  2 +-
 hooks/charmhelpers/contrib/openstack/utils.py        | 12 ++++++++++--
 hooks/charmhelpers/contrib/python/packages.py        |  7 +++++--
 tests/charmhelpers/contrib/amulet/utils.py           |  6 +++++-
 .../contrib/openstack/amulet/deployment.py           |  2 +-
 5 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
index 4538e961..461a702f 100644
--- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -109,7 +109,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         # Must be ordered by OpenStack release (not by Ubuntu release):
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
-         self.trusty_icehouse, self.trusty_juno, self.utopic_juno, 
+         self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
          self.trusty_kilo, self.vivid_kilo) = range(10)
 
         releases = {
diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index f90a0289..f4824f99 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -517,6 +517,7 @@ def git_clone_and_install(projects_yaml, core_project):
     """
     global requirements_dir
     parent_dir = '/mnt/openstack-git'
+    http_proxy = None
 
     if not projects_yaml:
         return
@@ -527,6 +528,7 @@ def git_clone_and_install(projects_yaml, core_project):
     old_environ = dict(os.environ)
 
     if 'http_proxy' in projects.keys():
+        http_proxy = projects['http_proxy']
         os.environ['http_proxy'] = projects['http_proxy']
     if 'https_proxy' in projects.keys():
         os.environ['https_proxy'] = projects['https_proxy']
@@ -539,10 +541,12 @@ def git_clone_and_install(projects_yaml, core_project):
         branch = p['branch']
         if p['name'] == 'requirements':
             repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+                                                     http_proxy,
                                                      update_requirements=False)
             requirements_dir = repo_dir
         else:
             repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+                                                     http_proxy,
                                                      update_requirements=True)
 
     os.environ = old_environ
@@ -574,7 +578,8 @@ def _git_ensure_key_exists(key, keys):
         error_out('openstack-origin-git key \'{}\' is missing'.format(key))
 
 
-def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements):
+def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy,
+                                  update_requirements):
     """
     Clone and install a single git repository.
     """
@@ -598,7 +603,10 @@ def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements)
         _git_update_requirements(repo_dir, requirements_dir)
 
     juju_log('Installing git repo from dir: {}'.format(repo_dir))
-    pip_install(repo_dir)
+    if http_proxy:
+        pip_install(repo_dir, ignore=True, proxy=http_proxy)
+    else:
+        pip_install(repo_dir, ignore=True)
 
     return repo_dir
 
diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py
index 8659516b..ceaa5971 100644
--- a/hooks/charmhelpers/contrib/python/packages.py
+++ b/hooks/charmhelpers/contrib/python/packages.py
@@ -51,17 +51,20 @@ def pip_install_requirements(requirements, **options):
     pip_execute(command)
 
 
-def pip_install(package, fatal=False, upgrade=False, **options):
+def pip_install(package, fatal=False, upgrade=False, ignore=False, **options):
     """Install a python package"""
     command = ["install"]
 
-    available_options = ('proxy', 'src', 'log', "index-url", )
+    available_options = ('proxy', 'src', 'log', 'index-url', )
     for option in parse_options(options, available_options):
         command.append(option)
 
     if upgrade:
         command.append('--upgrade')
 
+    if ignore:
+        command.append('--ignore-installed')
+
     if isinstance(package, list):
         command.extend(package)
     else:
diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py
index 58233714..f61c2e8b 100644
--- a/tests/charmhelpers/contrib/amulet/utils.py
+++ b/tests/charmhelpers/contrib/amulet/utils.py
@@ -89,7 +89,11 @@ class AmuletUtils(object):
     def _get_config(self, unit, filename):
         """Get a ConfigParser object for parsing a unit's config file."""
         file_contents = unit.file_contents(filename)
-        config = ConfigParser.ConfigParser()
+
+        # NOTE(beisner):  by default, ConfigParser does not handle options
+        # with no value, such as the flags used in the mysql my.cnf file.
+        # https://bugs.python.org/issue7005
+        config = ConfigParser.ConfigParser(allow_no_value=True)
         config.readfp(io.StringIO(file_contents))
         return config
 
diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
index 4538e961..461a702f 100644
--- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -109,7 +109,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         # Must be ordered by OpenStack release (not by Ubuntu release):
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
-         self.trusty_icehouse, self.trusty_juno, self.utopic_juno, 
+         self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
          self.trusty_kilo, self.vivid_kilo) = range(10)
 
         releases = {

From 7819cf768789bc1987939c65a0dc8970bc60aedf Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Wed, 22 Apr 2015 18:40:12 +0000
Subject: [PATCH 05/88] Add libffi-dev, libssl-dev, and libyaml-dev to base git
 install packages

---
 hooks/neutron_api_utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index 0739e33e..b0966c8d 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -66,8 +66,11 @@ KILO_PACKAGES = [
 ]
 
 BASE_GIT_PACKAGES = [
+    'libffi-dev',
+    'libssl-dev',
     'libxml2-dev',
     'libxslt1-dev',
+    'libyaml-dev',
     'python-dev',
     'python-pip',
     'python-setuptools',

From 3472c5c8abeb8766992d32ac87508c4565e2a37a Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Thu, 23 Apr 2015 11:55:32 +0000
Subject: [PATCH 06/88] Sync charm-helpers

---
 hooks/charmhelpers/contrib/openstack/utils.py | 4 ++--
 hooks/charmhelpers/contrib/python/packages.py | 5 +----
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index f4824f99..0460e076 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -604,9 +604,9 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy,
 
     juju_log('Installing git repo from dir: {}'.format(repo_dir))
     if http_proxy:
-        pip_install(repo_dir, ignore=True, proxy=http_proxy)
+        pip_install(repo_dir, proxy=http_proxy)
     else:
-        pip_install(repo_dir, ignore=True)
+        pip_install(repo_dir)
 
     return repo_dir
 
diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py
index ceaa5971..3f1fb09d 100644
--- a/hooks/charmhelpers/contrib/python/packages.py
+++ b/hooks/charmhelpers/contrib/python/packages.py
@@ -51,7 +51,7 @@ def pip_install_requirements(requirements, **options):
     pip_execute(command)
 
 
-def pip_install(package, fatal=False, upgrade=False, ignore=False, **options):
+def pip_install(package, fatal=False, upgrade=False, **options):
     """Install a python package"""
     command = ["install"]
 
@@ -62,9 +62,6 @@ def pip_install(package, fatal=False, upgrade=False, ignore=False, **options):
     if upgrade:
         command.append('--upgrade')
 
-    if ignore:
-        command.append('--ignore-installed')
-
     if isinstance(package, list):
         command.extend(package)
     else:

From 05902e855be88f403c301e802c13b06cc77c1a4f Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Thu, 23 Apr 2015 13:01:33 +0000
Subject: [PATCH 07/88] Sync charm-helpers

---
 hooks/charmhelpers/contrib/openstack/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index 0460e076..86bf91f5 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -604,9 +604,9 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy,
 
     juju_log('Installing git repo from dir: {}'.format(repo_dir))
     if http_proxy:
-        pip_install(repo_dir, proxy=http_proxy)
+        pip_install(repo_dir, upgrade=True, proxy=http_proxy)
     else:
-        pip_install(repo_dir)
+        pip_install(repo_dir, upgrade=True)
 
     return repo_dir
 

From 11ec41748030c2ea0ad41fb4769ebb9fc4acde06 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Thu, 23 Apr 2015 14:43:27 +0000
Subject: [PATCH 08/88] Sync charm-helpers

---
 hooks/charmhelpers/contrib/openstack/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index 86bf91f5..0460e076 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -604,9 +604,9 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy,
 
     juju_log('Installing git repo from dir: {}'.format(repo_dir))
     if http_proxy:
-        pip_install(repo_dir, upgrade=True, proxy=http_proxy)
+        pip_install(repo_dir, proxy=http_proxy)
     else:
-        pip_install(repo_dir, upgrade=True)
+        pip_install(repo_dir)
 
     return repo_dir
 

From b47dfc157be4c4ec83cfb89d14408eac6145fb83 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Thu, 23 Apr 2015 16:05:54 +0000
Subject: [PATCH 09/88] Drop added packages

---
 hooks/neutron_api_utils.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index b0966c8d..0739e33e 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -66,11 +66,8 @@ KILO_PACKAGES = [
 ]
 
 BASE_GIT_PACKAGES = [
-    'libffi-dev',
-    'libssl-dev',
     'libxml2-dev',
     'libxslt1-dev',
-    'libyaml-dev',
     'python-dev',
     'python-pip',
     'python-setuptools',

From e9c70fb71e315d6500d77ac52f7a66036e05cd9c Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Tue, 28 Apr 2015 19:05:05 +0000
Subject: [PATCH 10/88] Sync charm-helpers

---
 hooks/charmhelpers/fetch/__init__.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py
index 792e629a..57a179b8 100644
--- a/hooks/charmhelpers/fetch/__init__.py
+++ b/hooks/charmhelpers/fetch/__init__.py
@@ -367,13 +367,15 @@ def install_remote(source, *args, **kwargs):
     # explaining why it can't handle a given source.
     handlers = [h for h in plugins() if h.can_handle(source) is True]
     installed_to = None
-    for handler in handlers:
-        try:
-            installed_to = handler.install(source, *args, **kwargs)
-        except UnhandledSource:
-            pass
-    if not installed_to:
-        raise UnhandledSource("No handler found for source {}".format(source))
+    while not installed_to:
+        for handler in handlers:
+            try:
+                installed_to = handler.install(source, *args, **kwargs)
+            except UnhandledSource:
+                pass
+            log("CB: install loop - installed_to = |%s|".format(installed_to))
+        #if not installed_to:
+        #    raise UnhandledSource("No handler found for source {}".format(source))
     return installed_to
 
 

From abadc1caafebeb688ee657332239b92eb60d3173 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Fri, 1 May 2015 14:56:34 +0000
Subject: [PATCH 11/88] Sync charm-helpers

---
 hooks/charmhelpers/contrib/openstack/utils.py | 15 ++++++++-------
 hooks/charmhelpers/fetch/__init__.py          | 16 +++++++---------
 hooks/charmhelpers/fetch/giturl.py            | 11 +++++++----
 3 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index 0460e076..b47ed937 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -497,7 +497,7 @@ def git_install_requested():
 requirements_dir = None
 
 
-def git_clone_and_install(projects_yaml, core_project):
+def git_clone_and_install(projects_yaml, core_project, depth=1):
     """
     Clone/install all specified OpenStack repositories.
 
@@ -540,13 +540,13 @@ def git_clone_and_install(projects_yaml, core_project):
         repo = p['repository']
         branch = p['branch']
         if p['name'] == 'requirements':
-            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
-                                                     http_proxy,
+            repo_dir = _git_clone_and_install_single(repo, branch, depth,
+                                                     parent_dir, http_proxy,
                                                      update_requirements=False)
             requirements_dir = repo_dir
         else:
-            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
-                                                     http_proxy,
+            repo_dir = _git_clone_and_install_single(repo, branch, depth,
+                                                     parent_dir, http_proxy,
                                                      update_requirements=True)
 
     os.environ = old_environ
@@ -578,7 +578,7 @@ def _git_ensure_key_exists(key, keys):
         error_out('openstack-origin-git key \'{}\' is missing'.format(key))
 
 
-def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy,
+def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
                                   update_requirements):
     """
     Clone and install a single git repository.
@@ -592,7 +592,8 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy,
 
     if not os.path.exists(dest_dir):
         juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
-        repo_dir = install_remote(repo, dest=parent_dir, branch=branch)
+        repo_dir = install_remote(repo, dest=parent_dir, branch=branch,
+                                  depth=depth)
     else:
         repo_dir = dest_dir
 
diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py
index 57a179b8..792e629a 100644
--- a/hooks/charmhelpers/fetch/__init__.py
+++ b/hooks/charmhelpers/fetch/__init__.py
@@ -367,15 +367,13 @@ def install_remote(source, *args, **kwargs):
     # explaining why it can't handle a given source.
     handlers = [h for h in plugins() if h.can_handle(source) is True]
     installed_to = None
-    while not installed_to:
-        for handler in handlers:
-            try:
-                installed_to = handler.install(source, *args, **kwargs)
-            except UnhandledSource:
-                pass
-            log("CB: install loop - installed_to = |%s|".format(installed_to))
-        #if not installed_to:
-        #    raise UnhandledSource("No handler found for source {}".format(source))
+    for handler in handlers:
+        try:
+            installed_to = handler.install(source, *args, **kwargs)
+        except UnhandledSource:
+            pass
+    if not installed_to:
+        raise UnhandledSource("No handler found for source {}".format(source))
     return installed_to
 
 
diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py
index 93aae87b..06b0e767 100644
--- a/hooks/charmhelpers/fetch/giturl.py
+++ b/hooks/charmhelpers/fetch/giturl.py
@@ -45,14 +45,17 @@ class GitUrlFetchHandler(BaseFetchHandler):
         else:
             return True
 
-    def clone(self, source, dest, branch):
+    def clone(self, source, dest, branch, depth):
         if not self.can_handle(source):
             raise UnhandledSource("Cannot handle {}".format(source))
 
-        repo = Repo.clone_from(source, dest)
+        if depth:
+            repo = Repo.clone_from(source, dest, depth)
+        else:
+            repo = Repo.clone_from(source, dest)
         repo.git.checkout(branch)
 
-    def install(self, source, branch="master", dest=None):
+    def install(self, source, branch="master", dest=None, depth=None):
         url_parts = self.parse_url(source)
         branch_name = url_parts.path.strip("/").split("/")[-1]
         if dest:
@@ -63,7 +66,7 @@ class GitUrlFetchHandler(BaseFetchHandler):
         if not os.path.exists(dest_dir):
             mkdir(dest_dir, perms=0o755)
         try:
-            self.clone(source, dest_dir, branch)
+            self.clone(source, dest_dir, branch, depth)
         except GitCommandError as e:
             raise UnhandledSource(e.message)
         except OSError as e:

From ebb8fe70b62e9a3bac58dc2d7f06f60623627c83 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Sat, 2 May 2015 18:33:53 +0000
Subject: [PATCH 12/88] Sync charm-helpers

---
 hooks/charmhelpers/fetch/giturl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py
index 06b0e767..742cf319 100644
--- a/hooks/charmhelpers/fetch/giturl.py
+++ b/hooks/charmhelpers/fetch/giturl.py
@@ -50,7 +50,7 @@ class GitUrlFetchHandler(BaseFetchHandler):
             raise UnhandledSource("Cannot handle {}".format(source))
 
         if depth:
-            repo = Repo.clone_from(source, dest, depth)
+            repo = Repo.clone_from(source, dest, depth=depth)
         else:
             repo = Repo.clone_from(source, dest)
         repo.git.checkout(branch)

From 7df6f3346b2e454c08c15b065affd4dec0e7d51e Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Sat, 2 May 2015 20:10:25 +0000
Subject: [PATCH 13/88] Sync charm-helpers

---
 hooks/charmhelpers/fetch/giturl.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py
index 742cf319..4e05c08a 100644
--- a/hooks/charmhelpers/fetch/giturl.py
+++ b/hooks/charmhelpers/fetch/giturl.py
@@ -50,10 +50,9 @@ class GitUrlFetchHandler(BaseFetchHandler):
             raise UnhandledSource("Cannot handle {}".format(source))
 
         if depth:
-            repo = Repo.clone_from(source, dest, depth=depth)
+            repo = Repo.clone_from(source, dest, branch=branch, depth=depth)
         else:
-            repo = Repo.clone_from(source, dest)
-        repo.git.checkout(branch)
+            repo = Repo.clone_from(source, dest, branch=branch)
 
     def install(self, source, branch="master", dest=None, depth=None):
         url_parts = self.parse_url(source)

From 82152971095709f863dc0c08fff59023ed969348 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Tue, 5 May 2015 19:17:59 +0000
Subject: [PATCH 14/88] Sync charm-helpers

---
 hooks/charmhelpers/contrib/openstack/utils.py |  8 +++++++-
 hooks/charmhelpers/contrib/python/packages.py | 19 ++++++++++++++++++-
 hooks/charmhelpers/fetch/giturl.py            |  4 ++--
 3 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index b47ed937..ce8c8213 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -53,9 +53,13 @@ from charmhelpers.contrib.network.ip import (
     get_ipv6_addr
 )
 
+from charmhelpers.contrib.python.packages import (
+    pip_create_virtualenv,
+    pip_install,
+)
+
 from charmhelpers.core.host import lsb_release, mounts, umount
 from charmhelpers.fetch import apt_install, apt_cache, install_remote
-from charmhelpers.contrib.python.packages import pip_install
 from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
 from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
 
@@ -536,6 +540,8 @@ def git_clone_and_install(projects_yaml, core_project, depth=1):
     if 'directory' in projects.keys():
         parent_dir = projects['directory']
 
+    pip_create_virtualenv(proxy=http_proxy)
+
     for p in projects['repositories']:
         repo = p['repository']
         branch = p['branch']
diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py
index 3f1fb09d..e62240b4 100644
--- a/hooks/charmhelpers/contrib/python/packages.py
+++ b/hooks/charmhelpers/contrib/python/packages.py
@@ -17,8 +17,11 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
+import subprocess
+
 from charmhelpers.fetch import apt_install, apt_update
-from charmhelpers.core.hookenv import log
+from charmhelpers.core.hookenv import charm_dir, log
 
 try:
     from pip import main as pip_execute
@@ -94,3 +97,17 @@ def pip_list():
     """Returns the list of current python installed packages
     """
     return pip_execute(["list"])
+
+
+def pip_create_virtualenv(parent_dir=charm_dir(), proxy=None):
+    """Create and activate an isolated Python environment (virtualenv)."""
+    if proxy:
+        pip_install('virtualenv', proxy=proxy)
+    else:
+        pip_install('virtualenv')
+
+    venv_dir = os.path.join(parent_dir, 'venv')
+    subprocess.check_call(['virtualenv', '--no-site-packages', venv_dir])
+
+    venv_activate = os.path.join(venv_dir, 'bin/activate')
+    subprocess.check_call(['.', venv_activate], shell=True)
diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py
index 4e05c08a..7d842d49 100644
--- a/hooks/charmhelpers/fetch/giturl.py
+++ b/hooks/charmhelpers/fetch/giturl.py
@@ -50,9 +50,9 @@ class GitUrlFetchHandler(BaseFetchHandler):
             raise UnhandledSource("Cannot handle {}".format(source))
 
         if depth:
-            repo = Repo.clone_from(source, dest, branch=branch, depth=depth)
+            Repo.clone_from(source, dest, branch=branch, depth=depth)
         else:
-            repo = Repo.clone_from(source, dest, branch=branch)
+            Repo.clone_from(source, dest, branch=branch)
 
     def install(self, source, branch="master", dest=None, depth=None):
         url_parts = self.parse_url(source)

From e2615d975e06d7bbb45196bcbbcdfd8d8c643091 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Tue, 5 May 2015 19:48:05 +0000
Subject: [PATCH 15/88] Point upstart scripts at venv binaries

---
 hooks/neutron_api_utils.py                   | 3 +++
 templates/git/upstart/neutron-server.upstart | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index 0739e33e..15a1f61b 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -20,6 +20,7 @@ from charmhelpers.contrib.openstack.utils import (
 
 from charmhelpers.core.hookenv import (
     config,
+    charm_dir,
     log,
 )
 
@@ -383,10 +384,12 @@ def git_post_install(projects_yaml):
     render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {},
            perms=0o440)
 
+    bin_dir = os.path.join(charm_dir(), 'venv/bin')
     neutron_api_context = {
         'service_description': 'Neutron API server',
         'charm_name': 'neutron-api',
         'process_name': 'neutron-server',
+        'executable_name': os.path.join(bin_dir, 'neutron-server'),
     }
 
     # NOTE(coreycb): Needs systemd support
diff --git a/templates/git/upstart/neutron-server.upstart b/templates/git/upstart/neutron-server.upstart
index 7211e129..4bd8e268 100644
--- a/templates/git/upstart/neutron-server.upstart
+++ b/templates/git/upstart/neutron-server.upstart
@@ -16,7 +16,7 @@ end script
 script
   [ -r /etc/default/{{ process_name }} ] && . /etc/default/{{ process_name }}
   [ -r "$NEUTRON_PLUGIN_CONFIG" ] && CONF_ARG="--config-file $NEUTRON_PLUGIN_CONFIG"
-  exec start-stop-daemon --start --chuid neutron --exec /usr/local/bin/neutron-server -- \
+  exec start-stop-daemon --start --chuid neutron --exec {{ executable_name }} -- \
     --config-file /etc/neutron/neutron.conf \
     --log-file /var/log/neutron/server.log $CONF_ARG
 end script

From 3f9996a970d255bb247ed04ec8eb3bfe5dc4bedf Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Tue, 5 May 2015 19:55:25 +0000
Subject: [PATCH 16/88] Update unit tests

---
 unit_tests/test_neutron_api_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/unit_tests/test_neutron_api_utils.py b/unit_tests/test_neutron_api_utils.py
index b5760c0c..33879232 100644
--- a/unit_tests/test_neutron_api_utils.py
+++ b/unit_tests/test_neutron_api_utils.py
@@ -263,6 +263,7 @@ class TestNeutronAPIUtils(CharmTestCase):
             'service_description': 'Neutron API server',
             'charm_name': 'neutron-api',
             'process_name': 'neutron-server',
+            'executable_name': 'joined-string',
         }
         expected = [
             call('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {},

From 6e5a2b3a02cefa125244dc1740d3c2fd29b081b1 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Wed, 6 May 2015 16:00:11 +0000
Subject: [PATCH 17/88] Sync charm-helpers

---
 hooks/charmhelpers/contrib/openstack/utils.py |  6 ++--
 hooks/charmhelpers/contrib/python/packages.py | 29 +++++++++----------
 2 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index ce8c8213..5909b0f5 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -540,7 +540,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1):
     if 'directory' in projects.keys():
         parent_dir = projects['directory']
 
-    pip_create_virtualenv(proxy=http_proxy)
+    pip_create_virtualenv()
 
     for p in projects['repositories']:
         repo = p['repository']
@@ -611,9 +611,9 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
 
     juju_log('Installing git repo from dir: {}'.format(repo_dir))
     if http_proxy:
-        pip_install(repo_dir, proxy=http_proxy)
+        pip_install(repo_dir, proxy=http_proxy, venv=True)
     else:
-        pip_install(repo_dir)
+        pip_install(repo_dir, venv=True)
 
     return repo_dir
 
diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py
index e62240b4..740eaa51 100644
--- a/hooks/charmhelpers/contrib/python/packages.py
+++ b/hooks/charmhelpers/contrib/python/packages.py
@@ -54,9 +54,13 @@ def pip_install_requirements(requirements, **options):
     pip_execute(command)
 
 
-def pip_install(package, fatal=False, upgrade=False, **options):
+def pip_install(package, fatal=False, upgrade=False, venv=False, **options):
     """Install a python package"""
-    command = ["install"]
+    if venv:
+        venv_python = os.path.join(charm_dir(), 'venv/bin/pip')
+        command = [venv_python, "install"]
+    else:
+        command = ["install"]
 
     available_options = ('proxy', 'src', 'log', 'index-url', )
     for option in parse_options(options, available_options):
@@ -72,7 +76,10 @@ def pip_install(package, fatal=False, upgrade=False, **options):
 
     log("Installing {} package with options: {}".format(package,
                                                         command))
-    pip_execute(command)
+    if venv:
+        subprocess.check_call(command)
+    else:
+        pip_execute(command)
 
 
 def pip_uninstall(package, **options):
@@ -99,15 +106,7 @@ def pip_list():
     return pip_execute(["list"])
 
 
-def pip_create_virtualenv(parent_dir=charm_dir(), proxy=None):
-    """Create and activate an isolated Python environment (virtualenv)."""
-    if proxy:
-        pip_install('virtualenv', proxy=proxy)
-    else:
-        pip_install('virtualenv')
-
-    venv_dir = os.path.join(parent_dir, 'venv')
-    subprocess.check_call(['virtualenv', '--no-site-packages', venv_dir])
-
-    venv_activate = os.path.join(venv_dir, 'bin/activate')
-    subprocess.check_call(['.', venv_activate], shell=True)
+def pip_create_virtualenv():
+    """Create an isolated Python environment."""
+    apt_install('python-virtualenv')
+    subprocess.check_call(['virtualenv', os.path.join(charm_dir(), 'venv')])

From 470d8dd388d55cba308ed7b2087bb0463a7cd850 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Thu, 7 May 2015 12:44:23 +0000
Subject: [PATCH 18/88] Sync charm-helpers

---
 hooks/charmhelpers/contrib/openstack/utils.py | 17 +++++++++++++++-
 hooks/charmhelpers/contrib/python/packages.py | 20 ++++++++++++++++---
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index 5909b0f5..29ac7b78 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -540,7 +540,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1):
     if 'directory' in projects.keys():
         parent_dir = projects['directory']
 
-    pip_create_virtualenv()
+    pip_create_virtualenv(os.path.join(parent_dir, 'venv'))
 
     for p in projects['repositories']:
         repo = p['repository']
@@ -655,3 +655,18 @@ def git_src_dir(projects_yaml, project):
             return os.path.join(parent_dir, os.path.basename(p['repository']))
 
     return None
+
+
+def git_yaml_value(projects_yaml, key):
+    """
+    Return the value in projects_yaml for the specified key.
+    """
+    if not projects_yaml:
+        return None
+
+    projects = yaml.load(projects_yaml)
+
+    if key in projects.keys():
+        return projects[key]
+
+    return None
diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py
index 740eaa51..12838d24 100644
--- a/hooks/charmhelpers/contrib/python/packages.py
+++ b/hooks/charmhelpers/contrib/python/packages.py
@@ -30,6 +30,8 @@ except ImportError:
     apt_install('python-pip')
     from pip import main as pip_execute
 
+pip_venv_path = None
+
 __author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
 
 
@@ -57,7 +59,7 @@ def pip_install_requirements(requirements, **options):
 def pip_install(package, fatal=False, upgrade=False, venv=False, **options):
     """Install a python package"""
     if venv:
-        venv_python = os.path.join(charm_dir(), 'venv/bin/pip')
+        venv_python = os.path.join(pip_venv_path, 'bin/pip')
         command = [venv_python, "install"]
     else:
         command = ["install"]
@@ -106,7 +108,19 @@ def pip_list():
     return pip_execute(["list"])
 
 
-def pip_create_virtualenv():
+def pip_create_virtualenv(path=None):
     """Create an isolated Python environment."""
+    global pip_venv_path
+
     apt_install('python-virtualenv')
-    subprocess.check_call(['virtualenv', os.path.join(charm_dir(), 'venv')])
+
+    if path:
+        pip_venv_path = path
+    else:
+        pip_venv_path = os.path.join(charm_dir(), 'venv')
+
+    subprocess.check_call(['virtualenv', pip_venv_path])
+
+
+def pip_get_virtualenv_path():
+    return pip_venv_path

From c77ae94f69f86237d0fe6d3f0d912a460163744a Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Thu, 7 May 2015 12:49:53 +0000
Subject: [PATCH 19/88] Use function to get pip venv path

---
 hooks/neutron_api_utils.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index c8084861..80b8318a 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -10,6 +10,10 @@ from charmhelpers.contrib.openstack.neutron import (
     neutron_plugin_attribute,
 )
 
+from charmhelpers.contrib.python.packages import (
+    pip_get_virtualenv_path,
+)
+
 from charmhelpers.contrib.openstack.utils import (
     os_release,
     get_os_codename_install_source,
@@ -439,7 +443,7 @@ def git_post_install(projects_yaml):
     render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {},
            perms=0o440)
 
-    bin_dir = os.path.join(charm_dir(), 'venv/bin')
+    bin_dir = os.path.join(pip_get_virtualenv_path(), 'bin')
     neutron_api_context = {
         'service_description': 'Neutron API server',
         'charm_name': 'neutron-api',

From 10a4f86a992541f939f483af54ef52d454eaa76c Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Thu, 7 May 2015 14:40:12 +0000
Subject: [PATCH 20/88] Sync charm-helpers

---
 hooks/charmhelpers/contrib/openstack/utils.py | 23 +++++++++++++++++--
 hooks/charmhelpers/contrib/python/packages.py | 18 ++++-----------
 2 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index 29ac7b78..c5e08ec2 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -611,9 +611,11 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
 
     juju_log('Installing git repo from dir: {}'.format(repo_dir))
     if http_proxy:
-        pip_install(repo_dir, proxy=http_proxy, venv=True)
+        pip_install(repo_dir, proxy=http_proxy,
+                    venv=os.path.join(parent_dir, 'venv'))
     else:
-        pip_install(repo_dir, venv=True)
+        pip_install(repo_dir,
+                    venv=os.path.join(parent_dir, 'venv'))
 
     return repo_dir
 
@@ -636,6 +638,23 @@ def _git_update_requirements(package_dir, reqs_dir):
     os.chdir(orig_dir)
 
 
+def git_pip_venv_dir(projects_yaml):
+    """
+    Return the pip virtualenv path.
+    """
+    parent_dir = '/mnt/openstack-git'
+
+    if not projects_yaml:
+        return
+
+    projects = yaml.load(projects_yaml)
+
+    if 'directory' in projects.keys():
+        parent_dir = projects['directory']
+
+    return os.path.join(parent_dir, 'venv')
+
+
 def git_src_dir(projects_yaml, project):
     """
     Return the directory where the specified project's source is located.
diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py
index 12838d24..88c8887d 100644
--- a/hooks/charmhelpers/contrib/python/packages.py
+++ b/hooks/charmhelpers/contrib/python/packages.py
@@ -30,8 +30,6 @@ except ImportError:
     apt_install('python-pip')
     from pip import main as pip_execute
 
-pip_venv_path = None
-
 __author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
 
 
@@ -56,10 +54,10 @@ def pip_install_requirements(requirements, **options):
     pip_execute(command)
 
 
-def pip_install(package, fatal=False, upgrade=False, venv=False, **options):
+def pip_install(package, fatal=False, upgrade=False, venv=None, **options):
     """Install a python package"""
     if venv:
-        venv_python = os.path.join(pip_venv_path, 'bin/pip')
+        venv_python = os.path.join(venv, 'bin/pip')
         command = [venv_python, "install"]
     else:
         command = ["install"]
@@ -110,17 +108,11 @@ def pip_list():
 
 def pip_create_virtualenv(path=None):
     """Create an isolated Python environment."""
-    global pip_venv_path
-
     apt_install('python-virtualenv')
 
     if path:
-        pip_venv_path = path
+        venv_path = path
     else:
-        pip_venv_path = os.path.join(charm_dir(), 'venv')
+        venv_path = os.path.join(charm_dir(), 'venv')
 
-    subprocess.check_call(['virtualenv', pip_venv_path])
-
-
-def pip_get_virtualenv_path():
-    return pip_venv_path
+    subprocess.check_call(['virtualenv', venv_path])

From 3eef78431f5f906cebd0b38cc05283f3ccd3bf32 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Thu, 7 May 2015 14:53:48 +0000
Subject: [PATCH 21/88] Use git_pip_venv_dir to get venv path

---
 hooks/neutron_api_utils.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index 80b8318a..90d21f96 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -10,16 +10,13 @@ from charmhelpers.contrib.openstack.neutron import (
     neutron_plugin_attribute,
 )
 
-from charmhelpers.contrib.python.packages import (
-    pip_get_virtualenv_path,
-)
-
 from charmhelpers.contrib.openstack.utils import (
     os_release,
     get_os_codename_install_source,
     git_install_requested,
     git_clone_and_install,
     git_src_dir,
+    git_pip_venv_dir,
     configure_installation_source,
 )
 
@@ -443,7 +440,7 @@ def git_post_install(projects_yaml):
     render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {},
            perms=0o440)
 
-    bin_dir = os.path.join(pip_get_virtualenv_path(), 'bin')
+    bin_dir = os.path.join(git_pip_venv_dir(), 'bin')
     neutron_api_context = {
         'service_description': 'Neutron API server',
         'charm_name': 'neutron-api',

From 461a7fa736513fefbb8e98da467910bbf90a18a0 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Thu, 7 May 2015 15:11:07 +0000
Subject: [PATCH 22/88] Add missing params

---
 hooks/neutron_api_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index 90d21f96..787fdf8a 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -440,7 +440,7 @@ def git_post_install(projects_yaml):
     render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {},
            perms=0o440)
 
-    bin_dir = os.path.join(git_pip_venv_dir(), 'bin')
+    bin_dir = os.path.join(git_pip_venv_dir(projects_yaml), 'bin')
     neutron_api_context = {
         'service_description': 'Neutron API server',
         'charm_name': 'neutron-api',

From 49e437f88008ef4d4b51e924fbd04f8bf5abb7ac Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Thu, 7 May 2015 16:42:06 +0000
Subject: [PATCH 23/88] Add rootwrap symlink

---
 hooks/neutron_api_utils.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index 787fdf8a..14cad323 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -437,6 +437,17 @@ def git_post_install(projects_yaml):
             shutil.rmtree(c['dest'])
         shutil.copytree(c['src'], c['dest'])
 
+    symlinks = [
+        {'src': os.path.join(git_pip_venv_dir(projects_yaml),
+                             'bin/neutron-rootwrap'),
+         'link': '/usr/local/bin/neutron-rootwrap'},
+    ]
+
+    for s in symlinks:
+        if os.path.lexists(s['link']):
+            os.remove(s['link'])
+        os.symlink(s['src'], s['link'])
+
     render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {},
            perms=0o440)
 

From 76eaa4c3972b582dfe384decdddd47c979abeecb Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Thu, 7 May 2015 17:11:14 +0000
Subject: [PATCH 24/88] Unit test updates

---
 hooks/neutron_api_utils.py           | 1 -
 unit_tests/test_neutron_api_utils.py | 9 +++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index 14cad323..b0389166 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -22,7 +22,6 @@ from charmhelpers.contrib.openstack.utils import (
 
 from charmhelpers.core.hookenv import (
     config,
-    charm_dir,
     log,
 )
 
diff --git a/unit_tests/test_neutron_api_utils.py b/unit_tests/test_neutron_api_utils.py
index 3880ca30..9993153f 100644
--- a/unit_tests/test_neutron_api_utils.py
+++ b/unit_tests/test_neutron_api_utils.py
@@ -417,10 +417,11 @@ class TestNeutronAPIUtils(CharmTestCase):
     @patch.object(nutils, 'render')
     @patch('os.path.join')
     @patch('os.path.exists')
+    @patch('os.symlink')
     @patch('shutil.copytree')
     @patch('shutil.rmtree')
-    def test_git_post_install(self, rmtree, copytree, exists, join, render,
-                              service_restart, git_src_dir):
+    def test_git_post_install(self, rmtree, copytree, symlink, exists, join,
+                              render, service_restart, git_src_dir):
         projects_yaml = openstack_origin_git
         join.return_value = 'joined-string'
         nutils.git_post_install(projects_yaml)
@@ -430,6 +431,10 @@ class TestNeutronAPIUtils(CharmTestCase):
             call('joined-string', '/etc/neutron/rootwrap.d'),
         ]
         copytree.assert_has_calls(expected)
+        expected = [
+            call('joined-string', '/usr/local/bin/neutron-rootwrap'),
+        ]
+        symlink.assert_has_calls(expected, any_order=True)
         neutron_api_context = {
             'service_description': 'Neutron API server',
             'charm_name': 'neutron-api',

From 6b0acbe459d029ff17d2dbcef33b0fb3b37398d4 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Thu, 7 May 2015 18:42:45 +0000
Subject: [PATCH 25/88] Add base mysql support

---
 hooks/neutron_api_utils.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index b0389166..448802dc 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -17,9 +17,14 @@ from charmhelpers.contrib.openstack.utils import (
     git_clone_and_install,
     git_src_dir,
     git_pip_venv_dir,
+    git_yaml_value,
     configure_installation_source,
 )
 
+from charmhelpers.contrib.python.packages import (
+    pip_install,
+)
+
 from charmhelpers.core.hookenv import (
     config,
     log,
@@ -68,6 +73,7 @@ KILO_PACKAGES = [
 ]
 
 BASE_GIT_PACKAGES = [
+    'libmysqlclient-dev',
     'libxml2-dev',
     'libxslt1-dev',
     'python-dev',
@@ -421,6 +427,14 @@ def git_pre_install():
 
 def git_post_install(projects_yaml):
     """Perform post-install setup."""
+    http_proxy = git_yaml_value(projects_yaml, 'http_proxy')
+    if http_proxy:
+        pip_install('mysql-python', proxy=http_proxy,
+                    venv=git_pip_venv_dir(projects_yaml))
+    else:
+        pip_install('mysql-python',
+                    venv=git_pip_venv_dir(projects_yaml))
+
     src_etc = os.path.join(git_src_dir(projects_yaml, 'neutron'), 'etc')
     configs = [
         {'src': src_etc,

From 722ccb5a3a93a5be09caf57fa550a951b1530d33 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Thu, 7 May 2015 18:47:03 +0000
Subject: [PATCH 26/88] Unit test updates

---
 unit_tests/test_neutron_api_utils.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/unit_tests/test_neutron_api_utils.py b/unit_tests/test_neutron_api_utils.py
index 9993153f..0eff01c8 100644
--- a/unit_tests/test_neutron_api_utils.py
+++ b/unit_tests/test_neutron_api_utils.py
@@ -415,15 +415,19 @@ class TestNeutronAPIUtils(CharmTestCase):
     @patch.object(nutils, 'git_src_dir')
     @patch.object(nutils, 'service_restart')
     @patch.object(nutils, 'render')
+    @patch.object(nutils, 'git_pip_venv_dir')
     @patch('os.path.join')
     @patch('os.path.exists')
     @patch('os.symlink')
     @patch('shutil.copytree')
     @patch('shutil.rmtree')
-    def test_git_post_install(self, rmtree, copytree, symlink, exists, join,
-                              render, service_restart, git_src_dir):
+    @patch('subprocess.check_call')
+    def test_git_post_install(self, check_call, rmtree, copytree, symlink,
+                              exists, join, venv, render, service_restart,
+                              git_src_dir):
         projects_yaml = openstack_origin_git
         join.return_value = 'joined-string'
+        venv.return_value = '/mnt/openstack-git/venv'
         nutils.git_post_install(projects_yaml)
         expected = [
             call('joined-string', '/etc/neutron'),

From 073dc6fd15d7ebf7bf17e8a69e776b4d50fbbe1f Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Thu, 7 May 2015 18:54:24 +0000
Subject: [PATCH 27/88] Sync charm-helpers

---
 hooks/charmhelpers/contrib/python/packages.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py
index 88c8887d..07b0c1d7 100644
--- a/hooks/charmhelpers/contrib/python/packages.py
+++ b/hooks/charmhelpers/contrib/python/packages.py
@@ -115,4 +115,5 @@ def pip_create_virtualenv(path=None):
     else:
         venv_path = os.path.join(charm_dir(), 'venv')
 
-    subprocess.check_call(['virtualenv', venv_path])
+    if not os.path.exists(venv_path):
+        subprocess.check_call(['virtualenv', venv_path])

From 59fcf046c45c12f25633813aa2af6d7bb4ad4919 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Fri, 8 May 2015 12:38:19 +0000
Subject: [PATCH 28/88] Patch out pip_install

---
 unit_tests/test_neutron_api_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/unit_tests/test_neutron_api_utils.py b/unit_tests/test_neutron_api_utils.py
index 0eff01c8..a1e518ef 100644
--- a/unit_tests/test_neutron_api_utils.py
+++ b/unit_tests/test_neutron_api_utils.py
@@ -31,6 +31,7 @@ TO_PATCH = [
     'log',
     'neutron_plugin_attribute',
     'os_release',
+    'pip_install',
     'subprocess',
 ]
 

From 156f969c314b9a4f88c65714c7c92d2ebdd1a64f Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Fri, 8 May 2015 15:31:08 +0000
Subject: [PATCH 29/88] Add libffi-dev to base git pkgs

---
 hooks/neutron_api_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index 448802dc..a9216a44 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -73,6 +73,7 @@ KILO_PACKAGES = [
 ]
 
 BASE_GIT_PACKAGES = [
+    'libffi-dev',
     'libmysqlclient-dev',
     'libxml2-dev',
     'libxslt1-dev',

From 73d89aca8210a5a28cbb0e180e3827fe3856cdb1 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Fri, 8 May 2015 15:43:50 +0000
Subject: [PATCH 30/88] Add libssl-dev to base git pkgs

---
 hooks/neutron_api_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index a9216a44..93c9f315 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -75,6 +75,7 @@ KILO_PACKAGES = [
 BASE_GIT_PACKAGES = [
     'libffi-dev',
     'libmysqlclient-dev',
+    'libssl-dev',
     'libxml2-dev',
     'libxslt1-dev',
     'python-dev',

From 8fe5cb7ed02f5c054553361369f1bd289465503e Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Fri, 8 May 2015 16:46:38 +0000
Subject: [PATCH 31/88] Add symlink for neutron-db-manage

---
 hooks/neutron_api_utils.py           | 3 +++
 unit_tests/test_neutron_api_utils.py | 1 +
 2 files changed, 4 insertions(+)

diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index 93c9f315..43b80b75 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -456,6 +456,9 @@ def git_post_install(projects_yaml):
         {'src': os.path.join(git_pip_venv_dir(projects_yaml),
                              'bin/neutron-rootwrap'),
          'link': '/usr/local/bin/neutron-rootwrap'},
+        {'src': os.path.join(git_pip_venv_dir(projects_yaml),
+                             'bin/neutron-db-manage'),
+         'link': '/usr/local/bin/neutron-db-manage'},
     ]
 
     for s in symlinks:
diff --git a/unit_tests/test_neutron_api_utils.py b/unit_tests/test_neutron_api_utils.py
index a1e518ef..7de08226 100644
--- a/unit_tests/test_neutron_api_utils.py
+++ b/unit_tests/test_neutron_api_utils.py
@@ -438,6 +438,7 @@ class TestNeutronAPIUtils(CharmTestCase):
         copytree.assert_has_calls(expected)
         expected = [
             call('joined-string', '/usr/local/bin/neutron-rootwrap'),
+            call('joined-string', '/usr/local/bin/neutron-db-manage'),
         ]
         symlink.assert_has_calls(expected, any_order=True)
         neutron_api_context = {

From 4115ab1f68f5b130643ec380b4b43664eb34ec01 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Mon, 11 May 2015 12:32:50 +0000
Subject: [PATCH 32/88] Sync charm-helpers

---
 hooks/charmhelpers/contrib/openstack/utils.py | 30 +++++++++----------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index c5e08ec2..774eaba1 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -501,6 +501,16 @@ def git_install_requested():
 requirements_dir = None
 
 
+def _git_yaml_load(projects_yaml):
+    """
+    Load the specified yaml into a dictionary.
+    """
+    if not projects_yaml:
+        return None
+
+    projects = yaml.load(projects_yaml)
+
+
 def git_clone_and_install(projects_yaml, core_project, depth=1):
     """
     Clone/install all specified OpenStack repositories.
@@ -523,10 +533,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1):
     parent_dir = '/mnt/openstack-git'
     http_proxy = None
 
-    if not projects_yaml:
-        return
-
-    projects = yaml.load(projects_yaml)
+    projects = _git_yaml_load(projects_yaml)
     _git_validate_projects_yaml(projects, core_project)
 
     old_environ = dict(os.environ)
@@ -644,10 +651,7 @@ def git_pip_venv_dir(projects_yaml):
     """
     parent_dir = '/mnt/openstack-git'
 
-    if not projects_yaml:
-        return
-
-    projects = yaml.load(projects_yaml)
+    projects = _git_yaml_load(projects_yaml)
 
     if 'directory' in projects.keys():
         parent_dir = projects['directory']
@@ -661,10 +665,7 @@ def git_src_dir(projects_yaml, project):
     """
     parent_dir = '/mnt/openstack-git'
 
-    if not projects_yaml:
-        return
-
-    projects = yaml.load(projects_yaml)
+    projects = _git_yaml_load(projects_yaml)
 
     if 'directory' in projects.keys():
         parent_dir = projects['directory']
@@ -680,10 +681,7 @@ def git_yaml_value(projects_yaml, key):
     """
     Return the value in projects_yaml for the specified key.
     """
-    if not projects_yaml:
-        return None
-
-    projects = yaml.load(projects_yaml)
+    projects = _git_yaml_load(projects_yaml)
 
     if key in projects.keys():
         return projects[key]

From 3c674e7fbd5150f5ed28604ff001ccaeba6a4341 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Mon, 11 May 2015 12:37:15 +0000
Subject: [PATCH 33/88] Add comment to fix bin symlinks

---
 hooks/neutron_api_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index 43b80b75..71450bff 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -452,6 +452,7 @@ def git_post_install(projects_yaml):
             shutil.rmtree(c['dest'])
         shutil.copytree(c['src'], c['dest'])
 
+    # NOTE(coreycb): Need to find better solution than bin symlinks.
     symlinks = [
         {'src': os.path.join(git_pip_venv_dir(projects_yaml),
                              'bin/neutron-rootwrap'),

From 2891bfdb2b9eb01adc8f8aeedbef9418d2b3d39a Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Mon, 11 May 2015 12:46:05 +0000
Subject: [PATCH 34/88] Sync charm-helpers

---
 hooks/charmhelpers/contrib/openstack/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index 774eaba1..8bd2356e 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -508,7 +508,7 @@ def _git_yaml_load(projects_yaml):
     if not projects_yaml:
         return None
 
-    projects = yaml.load(projects_yaml)
+    return yaml.load(projects_yaml)
 
 
 def git_clone_and_install(projects_yaml, core_project, depth=1):

From 574cf2f106b1185b67b177fc49fef62465a4109c Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Tue, 12 May 2015 14:24:44 +0000
Subject: [PATCH 35/88] Sync charm-helpers

---
 hooks/charmhelpers/contrib/openstack/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index 8bd2356e..d795a358 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -524,8 +524,8 @@ def git_clone_and_install(projects_yaml, core_project, depth=1):
              repository: 'git://git.openstack.org/openstack/requirements.git',
              branch: 'stable/icehouse'}
         directory: /mnt/openstack-git
-        http_proxy: http://squid.internal:3128
-        https_proxy: https://squid.internal:3128
+        http_proxy: squid-proxy-url
+        https_proxy: squid-proxy-url
 
         The directory, http_proxy, and https_proxy keys are optional.
     """

From 11145b85b708c8de2910bb42ff507058012e4028 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Tue, 12 May 2015 14:49:28 +0000
Subject: [PATCH 36/88] Clone from github in deploy from source amulet tests

---
 tests/basic_deployment.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index f4617162..2428557d 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -76,10 +76,10 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
             openstack_origin_git = {
                 'repositories': [
                     {'name': 'requirements',
-                     'repository': 'git://git.openstack.org/openstack/requirements',
+                     'repository': 'git://github.com/openstack/requirements',
                      'branch': branch},
                     {'name': 'neutron',
-                     'repository': 'git://git.openstack.org/openstack/neutron',
+                     'repository': 'git://github.com/openstack/neutron',
                      'branch': branch},
                 ],
                 'directory': '/mnt/openstack-git',

From 84204aba62319d58fa0ad153ecd1fada8c07b445 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Tue, 12 May 2015 19:50:14 +0000
Subject: [PATCH 37/88] Add libyaml-dev as base git package

---
 hooks/neutron_api_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index 71450bff..9f3724aa 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -78,6 +78,7 @@ BASE_GIT_PACKAGES = [
     'libssl-dev',
     'libxml2-dev',
     'libxslt1-dev',
+    'libyaml-dev',
     'python-dev',
     'python-pip',
     'python-setuptools',

From 728f829e4043682c02b31fea229dc0b647811250 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Wed, 27 May 2015 13:02:18 +0000
Subject: [PATCH 38/88] Sync charm-helpers

---
 hooks/charmhelpers/fetch/giturl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py
index 7d842d49..ddc25b7e 100644
--- a/hooks/charmhelpers/fetch/giturl.py
+++ b/hooks/charmhelpers/fetch/giturl.py
@@ -45,7 +45,7 @@ class GitUrlFetchHandler(BaseFetchHandler):
         else:
             return True
 
-    def clone(self, source, dest, branch, depth):
+    def clone(self, source, dest, branch, depth=None):
         if not self.can_handle(source):
             raise UnhandledSource("Cannot handle {}".format(source))
 

From 3f601c58243c4dce9dcade58ad99c7e570c3e7a0 Mon Sep 17 00:00:00 2001
From: Billy Olsen <billy.olsen@canonical.com>
Date: Tue, 2 Jun 2015 14:49:10 -0700
Subject: [PATCH 39/88] [wolsen,r=] Add support for overriding public endpoint
 addresses.

Adds in the config option for overriding public endpoint addresses
and introduces a unit tests to ensure that the override for the
public address is functioning correctly.

Closes-Bug: #1398182
---
 config.yaml                                | 12 ++++
 hooks/charmhelpers/contrib/openstack/ip.py | 75 +++++++++-------------
 unit_tests/test_neutron_api_hooks.py       | 42 ++++++++++--
 3 files changed, 79 insertions(+), 50 deletions(-)

diff --git a/config.yaml b/config.yaml
index 8157f7c9..10e853dd 100644
--- a/config.yaml
+++ b/config.yaml
@@ -235,6 +235,18 @@ options:
       192.168.0.0/24)
       .
       This network will be used for public endpoints.
+  endpoint-public-name:
+    type: string
+    default:
+    description: |
+      The hostname or address of the public endpoints created for neutron-api
+      in the keystone identity provider.
+      .
+      This value will be used for public endpoints. For example, an
+      endpoint-public-name set to 'neutron-api.example.com' with ssl enabled
+      will create the following endpoint for neutron-api:
+      .
+      https://neutron-api.example.com:9696/
   ssl_cert:
     type: string
     default:
diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py
index 29bbddcb..45531b57 100644
--- a/hooks/charmhelpers/contrib/openstack/ip.py
+++ b/hooks/charmhelpers/contrib/openstack/ip.py
@@ -26,8 +26,6 @@ from charmhelpers.contrib.network.ip import (
 )
 from charmhelpers.contrib.hahelpers.cluster import is_clustered
 
-from functools import partial
-
 PUBLIC = 'public'
 INTERNAL = 'int'
 ADMIN = 'admin'
@@ -35,15 +33,18 @@ ADMIN = 'admin'
 ADDRESS_MAP = {
     PUBLIC: {
         'config': 'os-public-network',
-        'fallback': 'public-address'
+        'fallback': 'public-address',
+        'override': 'endpoint-public-name',
     },
     INTERNAL: {
         'config': 'os-internal-network',
-        'fallback': 'private-address'
+        'fallback': 'private-address',
+        'override': 'endpoint-internal-name',
     },
     ADMIN: {
         'config': 'os-admin-network',
-        'fallback': 'private-address'
+        'fallback': 'private-address',
+        'override': 'endpoint-admin-name',
     }
 }
 
@@ -57,15 +58,30 @@ def canonical_url(configs, endpoint_type=PUBLIC):
     :param endpoint_type: str endpoint type to resolve.
     :param returns: str base URL for services on the current service unit.
     """
-    scheme = 'http'
-    if 'https' in configs.complete_contexts():
-        scheme = 'https'
+    scheme = _get_scheme(configs)
+
     address = resolve_address(endpoint_type)
     if is_ipv6(address):
         address = "[{}]".format(address)
+
     return '%s://%s' % (scheme, address)
 
 
+def _get_scheme(configs):
+    """Returns the scheme to use for the url (either http or https)
+    depending upon whether https is in the configs value.
+
+    :param configs: OSTemplateRenderer config templating object to inspect
+                    for a complete https context.
+    :returns: either 'http' or 'https' depending on whether https is
+              configured within the configs context.
+    """
+    scheme = 'http'
+    if configs and 'https' in configs.complete_contexts():
+        scheme = 'https'
+    return scheme
+
+
 def resolve_address(endpoint_type=PUBLIC):
     """Return unit address depending on net config.
 
@@ -78,6 +94,14 @@ def resolve_address(endpoint_type=PUBLIC):
     :param endpoint_type: Network endpoing type
     """
     resolved_address = None
+
+    # Allow the user to override the address which is used. This is
+    # useful for proxy services or exposing a public endpoint url, etc.
+    override_key = ADDRESS_MAP[endpoint_type]['override']
+    addr_override = config(override_key)
+    if addr_override:
+        return addr_override
+
     vips = config('vip')
     if vips:
         vips = vips.split()
@@ -109,38 +133,3 @@ def resolve_address(endpoint_type=PUBLIC):
                          "clustered=%s)" % (net_type, clustered))
 
     return resolved_address
-
-
-def endpoint_url(configs, url_template, port, endpoint_type=PUBLIC,
-                 override=None):
-    """Returns the correct endpoint URL to advertise to Keystone.
-
-    This method provides the correct endpoint URL which should be advertised to
-    the keystone charm for endpoint creation. This method allows for the url to
-    be overridden to force a keystone endpoint to have specific URL for any of
-    the defined scopes (admin, internal, public).
-
-    :param configs: OSTemplateRenderer config templating object to inspect
-                    for a complete https context.
-    :param url_template: str format string for creating the url template. Only
-                         two values will be passed - the scheme+hostname
-                        returned by the canonical_url and the port.
-    :param endpoint_type: str endpoint type to resolve.
-    :param override: str the name of the config option which overrides the
-                     endpoint URL defined by the charm itself. None will
-                     disable any overrides (default).
-    """
-    if override:
-        # Return any user-defined overrides for the keystone endpoint URL.
-        user_value = config(override)
-        if user_value:
-            return user_value.strip()
-
-    return url_template % (canonical_url(configs, endpoint_type), port)
-
-
-public_endpoint = partial(endpoint_url, endpoint_type=PUBLIC)
-
-internal_endpoint = partial(endpoint_url, endpoint_type=INTERNAL)
-
-admin_endpoint = partial(endpoint_url, endpoint_type=ADMIN)
diff --git a/unit_tests/test_neutron_api_hooks.py b/unit_tests/test_neutron_api_hooks.py
index 70ee610c..92913fe9 100644
--- a/unit_tests/test_neutron_api_hooks.py
+++ b/unit_tests/test_neutron_api_hooks.py
@@ -25,7 +25,6 @@ TO_PATCH = [
     'api_port',
     'apt_update',
     'apt_install',
-    'canonical_url',
     'config',
     'CONFIGS',
     'check_call',
@@ -319,8 +318,9 @@ class NeutronAPIHooksTests(CharmTestCase):
         self._call_hook('amqp-relation-broken')
         self.assertTrue(self.CONFIGS.write_all.called)
 
-    def test_identity_joined(self):
-        self.canonical_url.return_value = 'http://127.0.0.1'
+    @patch.object(hooks, 'canonical_url')
+    def test_identity_joined(self, _canonical_url):
+        _canonical_url.return_value = 'http://127.0.0.1'
         self.api_port.return_value = '9696'
         self.test_config.set('region', 'region1')
         _neutron_url = 'http://127.0.0.1:9696'
@@ -337,6 +337,32 @@ class NeutronAPIHooksTests(CharmTestCase):
             relation_settings=_endpoints
         )
 
+    @patch('charmhelpers.contrib.openstack.ip.unit_get')
+    @patch('charmhelpers.contrib.openstack.ip.is_clustered')
+    @patch('charmhelpers.contrib.openstack.ip.config')
+    def test_identity_changed_public_name(self, _config, _is_clustered,
+                                          _unit_get):
+        _unit_get.return_value = '127.0.0.1'
+        _is_clustered.return_value = False
+        _config.side_effect = self.test_config.get
+        self.api_port.return_value = '9696'
+        self.test_config.set('region', 'region1')
+        self.test_config.set('endpoint-public-name',
+                             'neutron-api.example.com')
+        self._call_hook('identity-service-relation-joined')
+        _neutron_url = 'http://127.0.0.1:9696'
+        _endpoints = {
+            'quantum_service': 'quantum',
+            'quantum_region': 'region1',
+            'quantum_public_url': 'http://neutron-api.example.com:9696',
+            'quantum_admin_url': _neutron_url,
+            'quantum_internal_url': _neutron_url,
+        }
+        self.relation_set.assert_called_with(
+            relation_id=None,
+            relation_settings=_endpoints
+        )
+
     def test_identity_changed_partial_ctxt(self):
         self.CONFIGS.complete_contexts.return_value = []
         _api_rel_joined = self.patch('neutron_api_relation_joined')
@@ -353,12 +379,13 @@ class NeutronAPIHooksTests(CharmTestCase):
         self.assertTrue(self.CONFIGS.write.called_with(NEUTRON_CONF))
         self.assertTrue(_api_rel_joined.called)
 
-    def test_neutron_api_relation_no_id_joined(self):
+    @patch.object(hooks, 'canonical_url')
+    def test_neutron_api_relation_no_id_joined(self, _canonical_url):
         host = 'http://127.0.0.1'
         port = 1234
         _id_rel_joined = self.patch('identity_joined')
         self.relation_ids.side_effect = self._fake_relids
-        self.canonical_url.return_value = host
+        _canonical_url.return_value = host
         self.api_port.return_value = port
         self.is_relation_made = False
         neutron_url = '%s:%s' % (host, port)
@@ -381,10 +408,11 @@ class NeutronAPIHooksTests(CharmTestCase):
             **_relation_data
         )
 
-    def test_neutron_api_relation_joined(self):
+    @patch.object(hooks, 'canonical_url')
+    def test_neutron_api_relation_joined(self, _canonical_url):
         host = 'http://127.0.0.1'
         port = 1234
-        self.canonical_url.return_value = host
+        _canonical_url.return_value = host
         self.api_port.return_value = port
         self.is_relation_made = True
         neutron_url = '%s:%s' % (host, port)

From acdb35633e7a5c497f392a3d9bfcc08cdd85121d Mon Sep 17 00:00:00 2001
From: Billy Olsen <billy.olsen@canonical.com>
Date: Wed, 3 Jun 2015 11:25:21 -0700
Subject: [PATCH 40/88] Change config option to be os-public-hostname

---
 config.yaml                                | 4 ++--
 hooks/charmhelpers/contrib/openstack/ip.py | 6 +++---
 unit_tests/test_neutron_api_hooks.py       | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/config.yaml b/config.yaml
index 10e853dd..05d366cd 100644
--- a/config.yaml
+++ b/config.yaml
@@ -235,7 +235,7 @@ options:
       192.168.0.0/24)
       .
       This network will be used for public endpoints.
-  endpoint-public-name:
+  os-public-hostname:
     type: string
     default:
     description: |
@@ -243,7 +243,7 @@ options:
       in the keystone identity provider.
       .
       This value will be used for public endpoints. For example, an
-      endpoint-public-name set to 'neutron-api.example.com' with ssl enabled
+      os-public-hostname set to 'neutron-api.example.com' with ssl enabled
       will create the following endpoint for neutron-api:
       .
       https://neutron-api.example.com:9696/
diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py
index 45531b57..6e18c98a 100644
--- a/hooks/charmhelpers/contrib/openstack/ip.py
+++ b/hooks/charmhelpers/contrib/openstack/ip.py
@@ -34,17 +34,17 @@ ADDRESS_MAP = {
     PUBLIC: {
         'config': 'os-public-network',
         'fallback': 'public-address',
-        'override': 'endpoint-public-name',
+        'override': 'os-public-hostname',
     },
     INTERNAL: {
         'config': 'os-internal-network',
         'fallback': 'private-address',
-        'override': 'endpoint-internal-name',
+        'override': 'os-internal-hostname',
     },
     ADMIN: {
         'config': 'os-admin-network',
         'fallback': 'private-address',
-        'override': 'endpoint-admin-name',
+        'override': 'os-admin-hostname',
     }
 }
 
diff --git a/unit_tests/test_neutron_api_hooks.py b/unit_tests/test_neutron_api_hooks.py
index 92913fe9..ff909fbb 100644
--- a/unit_tests/test_neutron_api_hooks.py
+++ b/unit_tests/test_neutron_api_hooks.py
@@ -347,7 +347,7 @@ class NeutronAPIHooksTests(CharmTestCase):
         _config.side_effect = self.test_config.get
         self.api_port.return_value = '9696'
         self.test_config.set('region', 'region1')
-        self.test_config.set('endpoint-public-name',
+        self.test_config.set('os-public-hostname',
                              'neutron-api.example.com')
         self._call_hook('identity-service-relation-joined')
         _neutron_url = 'http://127.0.0.1:9696'

From 7a5ec35e47c6b01ccef2f143002b57cd9754887b Mon Sep 17 00:00:00 2001
From: Billy Olsen <billy.olsen@canonical.com>
Date: Thu, 4 Jun 2015 16:26:36 -0700
Subject: [PATCH 41/88] c-h sync

---
 hooks/charmhelpers/contrib/openstack/ip.py | 32 ++++++++++++++++------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py
index 6e18c98a..3dca6dc1 100644
--- a/hooks/charmhelpers/contrib/openstack/ip.py
+++ b/hooks/charmhelpers/contrib/openstack/ip.py
@@ -17,6 +17,7 @@
 from charmhelpers.core.hookenv import (
     config,
     unit_get,
+    service_name,
 )
 from charmhelpers.contrib.network.ip import (
     get_address_in_network,
@@ -82,6 +83,26 @@ def _get_scheme(configs):
     return scheme
 
 
+def _get_address_override(endpoint_type=PUBLIC):
+    """Returns any address overrides that the user has defined based on the
+    endpoint type.
+
+    Note: this function allows for the service name to be inserted into the
+    address if the user specifies {service_name}.somehost.org.
+
+    :param endpoint_type: the type of endpoint to retrieve the override
+                          value for.
+    :returns: any endpoint address or hostname that the user has overridden
+              or None if an override is not present.
+    """
+    override_key = ADDRESS_MAP[endpoint_type]['override']
+    addr_override = config(override_key)
+    if not addr_override:
+        return None
+    else:
+        return addr_override.format(service_name=service_name())
+
+
 def resolve_address(endpoint_type=PUBLIC):
     """Return unit address depending on net config.
 
@@ -93,14 +114,9 @@ def resolve_address(endpoint_type=PUBLIC):
 
     :param endpoint_type: Network endpoing type
     """
-    resolved_address = None
-
-    # Allow the user to override the address which is used. This is
-    # useful for proxy services or exposing a public endpoint url, etc.
-    override_key = ADDRESS_MAP[endpoint_type]['override']
-    addr_override = config(override_key)
-    if addr_override:
-        return addr_override
+    resolved_address = _get_address_override(endpoint_type)
+    if resolved_address:
+        return resolved_address
 
     vips = config('vip')
     if vips:

From 3b96c016b4a2a03dd4e07ad719d264802ffc7950 Mon Sep 17 00:00:00 2001
From: Billy Olsen <billy.olsen@canonical.com>
Date: Thu, 4 Jun 2015 16:28:14 -0700
Subject: [PATCH 42/88] Fix unit test error from c-h sync

---
 unit_tests/test_neutron_api_hooks.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/unit_tests/test_neutron_api_hooks.py b/unit_tests/test_neutron_api_hooks.py
index ff909fbb..377f1483 100644
--- a/unit_tests/test_neutron_api_hooks.py
+++ b/unit_tests/test_neutron_api_hooks.py
@@ -337,6 +337,8 @@ class NeutronAPIHooksTests(CharmTestCase):
             relation_settings=_endpoints
         )
 
+    @patch('charmhelpers.contrib.openstack.ip.service_name',
+           lambda *args: 'neutron-api')
     @patch('charmhelpers.contrib.openstack.ip.unit_get')
     @patch('charmhelpers.contrib.openstack.ip.is_clustered')
     @patch('charmhelpers.contrib.openstack.ip.config')

From f54243db7ef3d929be74551ac4f4fd91b6bf5d96 Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Thu, 11 Jun 2015 14:17:54 +0000
Subject: [PATCH 43/88] Support a neutron api sdn plugin

---
 config.yaml                     |  6 ++++
 hooks/neutron_api_context.py    | 59 +++++++++++++++++++++++++++++++++
 hooks/neutron_api_utils.py      | 51 +++++++++++++++++-----------
 metadata.yaml                   |  6 ++++
 templates/icehouse/neutron.conf |  4 +++
 templates/kilo/neutron.conf     |  5 +++
 6 files changed, 111 insertions(+), 20 deletions(-)

diff --git a/config.yaml b/config.yaml
index 05d366cd..3affd361 100644
--- a/config.yaml
+++ b/config.yaml
@@ -355,3 +355,9 @@ options:
     description: |
       A comma-separated list of nagios servicegroups.
       If left empty, the nagios_context will be used as the servicegroup
+  manage-neutron-plugin-legacy-mode:
+    type: boolean
+    default: True
+    description: |
+      If True neutron-server will install neutron packages for the plugin
+      stipulated.
diff --git a/hooks/neutron_api_context.py b/hooks/neutron_api_context.py
index c2cd990f..4a957f4f 100644
--- a/hooks/neutron_api_context.py
+++ b/hooks/neutron_api_context.py
@@ -234,3 +234,62 @@ class HAProxyContext(context.HAProxyContext):
         # for haproxy.conf
         ctxt['service_ports'] = port_mapping
         return ctxt
+
+
+class NeutronApiSDNContext(context.OSContextGenerator):
+    interfaces = ['neutron-test']
+
+    def __call__(self):
+        ctxt = {}
+        defaults = {
+            'core-plugin': {
+                'templ_key': 'core_plugin',
+                'value': 'neutron.plugins.ml2.plugin.Ml2Plugin',
+            },
+            'neutron-plugin-config': {
+                'templ_key': 'neutron_plugin_config',
+                'value': '/etc/neutron/plugins/ml2/ml2_conf.ini',
+            },
+            'service-plugins': {
+                'templ_key': 'service_plugins',
+                'value': 'router,firewall,lbaas,vpnaas,metering',
+            },
+        }
+        for rid in relation_ids('neutron-test'):
+            for unit in related_units(rid):
+                rdata = relation_get(rid=rid, unit=unit)
+                ctxt = {
+                    'neutron_plugin': rdata.get('neutron-plugin'),
+                }
+                if not context.context_complete(ctxt):
+                    continue
+                for key in defaults.keys():
+                    remote_value = rdata.get(key)
+                    ctxt_key = defaults[key]['templ_key']
+                    if remote_value:
+                        ctxt[ctxt_key] = remote_value
+                    else:
+                        ctxt[ctxt_key] = defaults[key]['value']
+                print ctxt
+                return ctxt
+        return ctxt
+
+class NeutronApiSDNConfigFileContext(context.OSContextGenerator):
+    interfaces = ['neutron-test']
+
+    def __call__(self):
+        ctxt = {}
+        defaults = {
+            'neutron-plugin-config': {
+                'templ_key': 'neutron_plugin_config',
+                'value': '/etc/neutron/plugins/ml2/ml2_conf.ini',
+            },
+        }
+        for rid in relation_ids('neutron-test'):
+            for unit in related_units(rid):
+                rdata = relation_get(rid=rid, unit=unit)
+                neutron_server_plugin_config = rdata.get('neutron-plugin-config')
+                print neutron_server_plugin_config
+                if neutron_server_plugin_config:
+                    return { 'config': neutron_server_plugin_config }
+        return { 'config': '/etc/neutron/plugins/ml2/ml2_conf.ini' }
diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index 9f3724aa..a1fc6389 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -158,16 +158,20 @@ def api_port(service):
     return API_PORTS[service]
 
 
+def manage_plugin():
+    return config('manage-neutron-plugin-legacy-mode')
+
 def determine_packages(source=None):
     # currently all packages match service names
     packages = [] + BASE_PACKAGES
 
     for v in resource_map().values():
         packages.extend(v['services'])
-        pkgs = neutron_plugin_attribute(config('neutron-plugin'),
-                                        'server_packages',
-                                        'neutron')
-        packages.extend(pkgs)
+        if manage_plugin():
+            pkgs = neutron_plugin_attribute(config('neutron-plugin'),
+                                            'server_packages',
+                                            'neutron')
+            packages.extend(pkgs)
 
     if get_os_codename_install_source(source) >= 'kilo':
         packages.extend(KILO_PACKAGES)
@@ -209,24 +213,31 @@ def resource_map():
     else:
         resource_map.pop(APACHE_24_CONF)
 
-    # add neutron plugin requirements. nova-c-c only needs the neutron-server
-    # associated with configs, not the plugin agent.
-    plugin = config('neutron-plugin')
-    conf = neutron_plugin_attribute(plugin, 'config', 'neutron')
-    ctxts = (neutron_plugin_attribute(plugin, 'contexts', 'neutron')
-             or [])
-    services = neutron_plugin_attribute(plugin, 'server_services',
-                                        'neutron')
-    resource_map[conf] = {}
-    resource_map[conf]['services'] = services
-    resource_map[conf]['contexts'] = ctxts
-    resource_map[conf]['contexts'].append(
-        neutron_api_context.NeutronCCContext())
+    if manage_plugin():
+        # add neutron plugin requirements. nova-c-c only needs the neutron-server
+        # associated with configs, not the plugin agent.
+        plugin = config('neutron-plugin')
+        conf = neutron_plugin_attribute(plugin, 'config', 'neutron')
+        ctxts = (neutron_plugin_attribute(plugin, 'contexts', 'neutron')
+                 or [])
+        services = neutron_plugin_attribute(plugin, 'server_services',
+                                            'neutron')
+        resource_map[conf] = {}
+        resource_map[conf]['services'] = services
+        resource_map[conf]['contexts'] = ctxts
+        resource_map[conf]['contexts'].append(
+            neutron_api_context.NeutronCCContext())
 
-    # update for postgres
-    resource_map[conf]['contexts'].append(
-        context.PostgresqlDBContext(database=config('database')))
+        # update for postgres
+        resource_map[conf]['contexts'].append(
+            context.PostgresqlDBContext(database=config('database')))
 
+    else:
+        resource_map[NEUTRON_CONF]['contexts'].append(
+            neutron_api_context.NeutronApiSDNContext()
+        )
+        resource_map[NEUTRON_DEFAULT]['contexts'] = \
+            neutron_api_context.NeutronApiSDNConfigFileContext()
     return resource_map
 
 
diff --git a/metadata.yaml b/metadata.yaml
index 0c8256be..26bda6f3 100644
--- a/metadata.yaml
+++ b/metadata.yaml
@@ -37,6 +37,12 @@ requires:
   zeromq-configuration:
     interface: zeromq-configuration
     scope: container
+  neutron-plugin:
+    interface: neutron-plugin
+    scope: container
+  neutron-test:
+    interface: neutron-test
+    scope: container
 peers:
   cluster:
     interface: neutron-api-ha
diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf
index c78169d5..08d14fc7 100644
--- a/templates/icehouse/neutron.conf
+++ b/templates/icehouse/neutron.conf
@@ -27,10 +27,14 @@ bind_port = 9696
 
 {% if core_plugin -%}
 core_plugin = {{ core_plugin }}
+{% if service_plugins -%}
+service_plugins = {{ service_plugins }}
+{% else -%}
 {% if neutron_plugin in ['ovs', 'ml2'] -%}
 service_plugins = neutron.services.l3_router.l3_router_plugin.L3RouterPlugin,neutron.services.firewall.fwaas_plugin.FirewallPlugin,neutron.services.loadbalancer.plugin.LoadBalancerPlugin,neutron.services.vpn.plugin.VPNDriverPlugin,neutron.services.metering.metering_plugin.MeteringPlugin
 {% endif -%}
 {% endif -%}
+{% endif -%}
 
 {% if neutron_security_groups -%}
 allow_overlapping_ips = True
diff --git a/templates/kilo/neutron.conf b/templates/kilo/neutron.conf
index a6a3c664..5afd6e5c 100644
--- a/templates/kilo/neutron.conf
+++ b/templates/kilo/neutron.conf
@@ -29,12 +29,17 @@ bind_port = {{ neutron_bind_port }}
 bind_port = 9696
 {% endif -%}
 
+# {{ service_plugins }}
 {% if core_plugin -%}
 core_plugin = {{ core_plugin }}
+{% if service_plugins -%}
+service_plugins = {{ service_plugins }}
+{% else -%}
 {% if neutron_plugin in ['ovs', 'ml2'] -%}
 service_plugins = router,firewall,lbaas,vpnaas,metering
 {% endif -%}
 {% endif -%}
+{% endif -%}
 
 {% if neutron_security_groups -%}
 allow_overlapping_ips = True

From b619f1cc68c3ec38b5afd5a757d9ccd9682fa9b4 Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Thu, 11 Jun 2015 16:48:46 +0000
Subject: [PATCH 44/88] Add restart trigger support

---
 hooks/neutron_api_context.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/hooks/neutron_api_context.py b/hooks/neutron_api_context.py
index 4a957f4f..240fa1b3 100644
--- a/hooks/neutron_api_context.py
+++ b/hooks/neutron_api_context.py
@@ -254,6 +254,10 @@ class NeutronApiSDNContext(context.OSContextGenerator):
                 'templ_key': 'service_plugins',
                 'value': 'router,firewall,lbaas,vpnaas,metering',
             },
+            'restart-trigger': {
+                'templ_key': 'restart_trigger',
+                'value': '',
+            },
         }
         for rid in relation_ids('neutron-test'):
             for unit in related_units(rid):
@@ -270,7 +274,6 @@ class NeutronApiSDNContext(context.OSContextGenerator):
                         ctxt[ctxt_key] = remote_value
                     else:
                         ctxt[ctxt_key] = defaults[key]['value']
-                print ctxt
                 return ctxt
         return ctxt
 
@@ -289,7 +292,6 @@ class NeutronApiSDNConfigFileContext(context.OSContextGenerator):
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 neutron_server_plugin_config = rdata.get('neutron-plugin-config')
-                print neutron_server_plugin_config
                 if neutron_server_plugin_config:
                     return { 'config': neutron_server_plugin_config }
         return { 'config': '/etc/neutron/plugins/ml2/ml2_conf.ini' }

From 7a2fa1b185e04cf1b8c42ec4eb14f22e9fa4db93 Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Fri, 12 Jun 2015 09:26:59 +0000
Subject: [PATCH 45/88] Add support for config for neutron-server coming from
 subordinate

---
 hooks/neutron_api_context.py | 15 +++++++++------
 hooks/neutron_api_utils.py   |  1 +
 templates/kilo/neutron.conf  | 26 ++++++++++++++++++++++++++
 3 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/hooks/neutron_api_context.py b/hooks/neutron_api_context.py
index 240fa1b3..25c6ac34 100644
--- a/hooks/neutron_api_context.py
+++ b/hooks/neutron_api_context.py
@@ -236,11 +236,16 @@ class HAProxyContext(context.HAProxyContext):
         return ctxt
 
 
-class NeutronApiSDNContext(context.OSContextGenerator):
-    interfaces = ['neutron-test']
+class NeutronApiSDNContext(context.SubordinateConfigContext):
+    interfaces = 'neutron-test'
+
+    def __init__(self):
+        super(NeutronApiSDNContext, self).__init__(interface='neutron-test',
+                                                   service='neutron-api',
+                                                   config_file='/etc/neutron/neutron.conf')
 
     def __call__(self):
-        ctxt = {}
+        ctxt = super(NeutronApiSDNContext, self).__call__()
         defaults = {
             'core-plugin': {
                 'templ_key': 'core_plugin',
@@ -262,9 +267,7 @@ class NeutronApiSDNContext(context.OSContextGenerator):
         for rid in relation_ids('neutron-test'):
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
-                ctxt = {
-                    'neutron_plugin': rdata.get('neutron-plugin'),
-                }
+                ctxt['neutron_plugin'] = rdata.get('neutron-plugin')
                 if not context.context_complete(ctxt):
                     continue
                 for key in defaults.keys():
diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index a1fc6389..ca105a0e 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -233,6 +233,7 @@ def resource_map():
             context.PostgresqlDBContext(database=config('database')))
 
     else:
+        print "Adding NeutronApiSDNContext"
         resource_map[NEUTRON_CONF]['contexts'].append(
             neutron_api_context.NeutronApiSDNContext()
         )
diff --git a/templates/kilo/neutron.conf b/templates/kilo/neutron.conf
index 5afd6e5c..bf62a467 100644
--- a/templates/kilo/neutron.conf
+++ b/templates/kilo/neutron.conf
@@ -59,6 +59,12 @@ nova_admin_auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v2.0
 
 {% include "section-zeromq" %}
 
+{% if sections and 'DEFAULT' in sections -%}
+{% for key, value in sections['DEFAULT'] -%}
+{{ key }} = {{ value }}
+{% endfor -%}
+{% endif %}
+
 [quotas]
 quota_driver = neutron.db.quota_db.DbQuotaDriver
 {% if neutron_security_groups -%}
@@ -77,6 +83,11 @@ quota_member = {{ quota_member }}
 quota_health_monitors = {{ quota_health_monitors }}
 quota_router = {{ quota_router }}
 quota_floatingip = {{ quota_floatingip }}
+{% if sections and 'quotas' in sections -%}
+{% for key, value in sections['quotas'] -%}
+{{ key }} = {{ value }}
+{% endfor -%}
+{% endif %}
 
 [agent]
 root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
@@ -86,11 +97,26 @@ root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
 {% include "parts/section-database" %}
 
 {% include "section-rabbitmq-oslo" %}
+{% if sections and 'agent' in sections -%}
+{% for key, value in sections['agent'] -%}
+{{ key }} = {{ value }}
+{% endfor -%}
+{% endif %}
 
 [service_providers]
 service_provider=LOADBALANCER:Haproxy:neutron_lbaas.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
 service_provider=VPN:openswan:neutron_vpnaas.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default
 service_provider=FIREWALL:Iptables:neutron_fwaas.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver:default
+{% if sections and 'service_providers' in sections -%}
+{% for key, value in sections['service_providers'] -%}
+{{ key }} = {{ value }}
+{% endfor -%}
+{% endif %}
 
 [oslo_concurrency]
 lock_path = $state_path/lock
+{% if sections and 'oslo_concurrency' in sections -%}
+{% for key, value in sections['oslo_concurrency'] -%}
+{{ key }} = {{ value }}
+{% endfor -%}
+{% endif %}

From d835d0c9e5247dfb1369e1aec6f466627839afc1 Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Fri, 12 Jun 2015 12:23:29 +0000
Subject: [PATCH 46/88] Update relation name

---
 hooks/neutron_api_context.py | 10 +++++-----
 metadata.yaml                |  4 ++--
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/hooks/neutron_api_context.py b/hooks/neutron_api_context.py
index 25c6ac34..80662841 100644
--- a/hooks/neutron_api_context.py
+++ b/hooks/neutron_api_context.py
@@ -237,10 +237,10 @@ class HAProxyContext(context.HAProxyContext):
 
 
 class NeutronApiSDNContext(context.SubordinateConfigContext):
-    interfaces = 'neutron-test'
+    interfaces = 'neutron-plugin-api-subordinate'
 
     def __init__(self):
-        super(NeutronApiSDNContext, self).__init__(interface='neutron-test',
+        super(NeutronApiSDNContext, self).__init__(interface='neutron-plugin-api-subordinate',
                                                    service='neutron-api',
                                                    config_file='/etc/neutron/neutron.conf')
 
@@ -264,7 +264,7 @@ class NeutronApiSDNContext(context.SubordinateConfigContext):
                 'value': '',
             },
         }
-        for rid in relation_ids('neutron-test'):
+        for rid in relation_ids('neutron-plugin-api-subordinate'):
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 ctxt['neutron_plugin'] = rdata.get('neutron-plugin')
@@ -281,7 +281,7 @@ class NeutronApiSDNContext(context.SubordinateConfigContext):
         return ctxt
 
 class NeutronApiSDNConfigFileContext(context.OSContextGenerator):
-    interfaces = ['neutron-test']
+    interfaces = ['neutron-plugin-api-subordinate']
 
     def __call__(self):
         ctxt = {}
@@ -291,7 +291,7 @@ class NeutronApiSDNConfigFileContext(context.OSContextGenerator):
                 'value': '/etc/neutron/plugins/ml2/ml2_conf.ini',
             },
         }
-        for rid in relation_ids('neutron-test'):
+        for rid in relation_ids('neutron-plugin-api-subordinate'):
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 neutron_server_plugin_config = rdata.get('neutron-plugin-config')
diff --git a/metadata.yaml b/metadata.yaml
index 26bda6f3..be2bd85a 100644
--- a/metadata.yaml
+++ b/metadata.yaml
@@ -40,8 +40,8 @@ requires:
   neutron-plugin:
     interface: neutron-plugin
     scope: container
-  neutron-test:
-    interface: neutron-test
+  neutron-plugin-api-subordinate:
+    interface: neutron-plugin-api-subordinate
     scope: container
 peers:
   cluster:

From 2ba62d1903c3074563ca6ceb2cb224204ec87bec Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Sat, 13 Jun 2015 09:05:22 +0000
Subject: [PATCH 47/88] Remove unused interface

---
 metadata.yaml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/metadata.yaml b/metadata.yaml
index be2bd85a..3ef46991 100644
--- a/metadata.yaml
+++ b/metadata.yaml
@@ -37,9 +37,6 @@ requires:
   zeromq-configuration:
     interface: zeromq-configuration
     scope: container
-  neutron-plugin:
-    interface: neutron-plugin
-    scope: container
   neutron-plugin-api-subordinate:
     interface: neutron-plugin-api-subordinate
     scope: container

From 8040e05ddf8021bdbc93066c3b31301d4fae9275 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Fri, 19 Jun 2015 15:09:05 +0000
Subject: [PATCH 48/88] Sync charm-helpers

---
 .../charmhelpers/contrib/hahelpers/cluster.py |  15 ++-
 .../contrib/openstack/amulet/deployment.py    |   8 +-
 .../contrib/openstack/amulet/utils.py         | 125 +++++++++++++++++-
 .../charmhelpers/contrib/openstack/context.py |   2 +-
 .../charmhelpers/contrib/openstack/neutron.py |  10 +-
 hooks/charmhelpers/contrib/openstack/utils.py |  29 ++--
 hooks/charmhelpers/core/host.py               |  30 ++++-
 tests/charmhelpers/contrib/amulet/utils.py    |  97 +++++++++++++-
 .../contrib/openstack/amulet/deployment.py    |   8 +-
 .../contrib/openstack/amulet/utils.py         | 125 +++++++++++++++++-
 10 files changed, 411 insertions(+), 38 deletions(-)

diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py
index 5790b46f..aa0b515d 100644
--- a/hooks/charmhelpers/contrib/hahelpers/cluster.py
+++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py
@@ -64,6 +64,10 @@ class CRMResourceNotFound(Exception):
     pass
 
 
+class CRMDCNotFound(Exception):
+    pass
+
+
 def is_elected_leader(resource):
     """
     Returns True if the charm executing this is the elected cluster leader.
@@ -116,8 +120,9 @@ def is_crm_dc():
         status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
         if not isinstance(status, six.text_type):
             status = six.text_type(status, "utf-8")
-    except subprocess.CalledProcessError:
-        return False
+    except subprocess.CalledProcessError as ex:
+        raise CRMDCNotFound(str(ex))
+
     current_dc = ''
     for line in status.split('\n'):
         if line.startswith('Current DC'):
@@ -125,10 +130,14 @@ def is_crm_dc():
             current_dc = line.split(':')[1].split()[0]
     if current_dc == get_unit_hostname():
         return True
+    elif current_dc == 'NONE':
+        raise CRMDCNotFound('Current DC: NONE')
+
     return False
 
 
-@retry_on_exception(5, base_delay=2, exc_type=CRMResourceNotFound)
+@retry_on_exception(5, base_delay=2,
+                    exc_type=(CRMResourceNotFound, CRMDCNotFound))
 def is_crm_leader(resource, retry=False):
     """
     Returns True if the charm calling this is the elected corosync leader,
diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
index 461a702f..c664c9d0 100644
--- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -110,7 +110,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
          self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
-         self.trusty_kilo, self.vivid_kilo) = range(10)
+         self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
+         self.wily_liberty) = range(12)
 
         releases = {
             ('precise', None): self.precise_essex,
@@ -121,8 +122,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('trusty', None): self.trusty_icehouse,
             ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
             ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
+            ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
             ('utopic', None): self.utopic_juno,
-            ('vivid', None): self.vivid_kilo}
+            ('vivid', None): self.vivid_kilo,
+            ('wily', None): self.wily_liberty}
         return releases[(self.series, self.openstack)]
 
     def _get_openstack_release_string(self):
@@ -138,6 +141,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('trusty', 'icehouse'),
             ('utopic', 'juno'),
             ('vivid', 'kilo'),
+            ('wily', 'liberty'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]
diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py
index 9c3d918a..576bf0b5 100644
--- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py
@@ -16,15 +16,15 @@
 
 import logging
 import os
+import six
 import time
 import urllib
 
 import glanceclient.v1.client as glance_client
+import heatclient.v1.client as heat_client
 import keystoneclient.v2_0 as keystone_client
 import novaclient.v1_1.client as nova_client
 
-import six
-
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
 )
@@ -37,7 +37,7 @@ class OpenStackAmuletUtils(AmuletUtils):
     """OpenStack amulet utilities.
 
        This class inherits from AmuletUtils and has additional support
-       that is specifically for use by OpenStack charms.
+       that is specifically for use by OpenStack charm tests.
        """
 
     def __init__(self, log_level=ERROR):
@@ -51,6 +51,8 @@ class OpenStackAmuletUtils(AmuletUtils):
            Validate actual endpoint data vs expected endpoint data. The ports
            are used to find the matching endpoint.
            """
+        self.log.debug('Validating endpoint data...')
+        self.log.debug('actual: {}'.format(repr(endpoints)))
         found = False
         for ep in endpoints:
             self.log.debug('endpoint: {}'.format(repr(ep)))
@@ -77,6 +79,7 @@ class OpenStackAmuletUtils(AmuletUtils):
            Validate a list of actual service catalog endpoints vs a list of
            expected service catalog endpoints.
            """
+        self.log.debug('Validating service catalog endpoint data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for k, v in six.iteritems(expected):
             if k in actual:
@@ -93,6 +96,7 @@ class OpenStackAmuletUtils(AmuletUtils):
            Validate a list of actual tenant data vs list of expected tenant
            data.
            """
+        self.log.debug('Validating tenant data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -114,6 +118,7 @@ class OpenStackAmuletUtils(AmuletUtils):
            Validate a list of actual role data vs a list of expected role
            data.
            """
+        self.log.debug('Validating role data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -134,6 +139,7 @@ class OpenStackAmuletUtils(AmuletUtils):
            Validate a list of actual user data vs a list of expected user
            data.
            """
+        self.log.debug('Validating user data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -155,17 +161,20 @@ class OpenStackAmuletUtils(AmuletUtils):
 
            Validate a list of actual flavors vs a list of expected flavors.
            """
+        self.log.debug('Validating flavor data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         act = [a.name for a in actual]
         return self._validate_list_data(expected, act)
 
     def tenant_exists(self, keystone, tenant):
         """Return True if tenant exists."""
+        self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
                                     tenant):
         """Authenticates admin user with the keystone admin endpoint."""
+        self.log.debug('Authenticating keystone admin...')
         unit = keystone_sentry
         service_ip = unit.relation('shared-db',
                                    'mysql:shared-db')['private-address']
@@ -175,6 +184,7 @@ class OpenStackAmuletUtils(AmuletUtils):
 
     def authenticate_keystone_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with the keystone public endpoint."""
+        self.log.debug('Authenticating keystone user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
         return keystone_client.Client(username=user, password=password,
@@ -182,12 +192,21 @@ class OpenStackAmuletUtils(AmuletUtils):
 
     def authenticate_glance_admin(self, keystone):
         """Authenticates admin user with glance."""
+        self.log.debug('Authenticating glance admin...')
         ep = keystone.service_catalog.url_for(service_type='image',
                                               endpoint_type='adminURL')
         return glance_client.Client(ep, token=keystone.auth_token)
 
+    def authenticate_heat_admin(self, keystone):
+        """Authenticates the admin user with heat."""
+        self.log.debug('Authenticating heat admin...')
+        ep = keystone.service_catalog.url_for(service_type='orchestration',
+                                              endpoint_type='publicURL')
+        return heat_client.Client(endpoint=ep, token=keystone.auth_token)
+
     def authenticate_nova_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with nova-api."""
+        self.log.debug('Authenticating nova user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
         return nova_client.Client(username=user, api_key=password,
@@ -195,6 +214,7 @@ class OpenStackAmuletUtils(AmuletUtils):
 
     def create_cirros_image(self, glance, image_name):
         """Download the latest cirros image and upload it to glance."""
+        self.log.debug('Creating glance image ({})...'.format(image_name))
         http_proxy = os.getenv('AMULET_HTTP_PROXY')
         self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
         if http_proxy:
@@ -235,6 +255,11 @@ class OpenStackAmuletUtils(AmuletUtils):
 
     def delete_image(self, glance, image):
         """Delete the specified image."""
+
+        # /!\ DEPRECATION WARNING
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'delete_resource instead of delete_image.')
+        self.log.debug('Deleting glance image ({})...'.format(image))
         num_before = len(list(glance.images.list()))
         glance.images.delete(image)
 
@@ -254,6 +279,8 @@ class OpenStackAmuletUtils(AmuletUtils):
 
     def create_instance(self, nova, image_name, instance_name, flavor):
         """Create the specified instance."""
+        self.log.debug('Creating instance '
+                       '({}|{}|{})'.format(instance_name, image_name, flavor))
         image = nova.images.find(name=image_name)
         flavor = nova.flavors.find(name=flavor)
         instance = nova.servers.create(name=instance_name, image=image,
@@ -276,6 +303,11 @@ class OpenStackAmuletUtils(AmuletUtils):
 
     def delete_instance(self, nova, instance):
         """Delete the specified instance."""
+
+        # /!\ DEPRECATION WARNING
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'delete_resource instead of delete_instance.')
+        self.log.debug('Deleting instance ({})...'.format(instance))
         num_before = len(list(nova.servers.list()))
         nova.servers.delete(instance)
 
@@ -292,3 +324,90 @@ class OpenStackAmuletUtils(AmuletUtils):
             return False
 
         return True
+
+    def create_or_get_keypair(self, nova, keypair_name="testkey"):
+        """Create a new keypair, or return pointer if it already exists."""
+        try:
+            _keypair = nova.keypairs.get(keypair_name)
+            self.log.debug('Keypair ({}) already exists, '
+                           'using it.'.format(keypair_name))
+            return _keypair
+        except:
+            self.log.debug('Keypair ({}) does not exist, '
+                           'creating it.'.format(keypair_name))
+
+        _keypair = nova.keypairs.create(name=keypair_name)
+        return _keypair
+
+    def delete_resource(self, resource, resource_id,
+                        msg="resource", max_wait=120):
+        """Delete one openstack resource, such as one instance, keypair,
+        image, volume, stack, etc., and confirm deletion within max wait time.
+
+        :param resource: pointer to os resource type, ex:glance_client.images
+        :param resource_id: unique name or id for the openstack resource
+        :param msg: text to identify purpose in logging
+        :param max_wait: maximum wait time in seconds
+        :returns: True if successful, otherwise False
+        """
+        num_before = len(list(resource.list()))
+        resource.delete(resource_id)
+
+        tries = 0
+        num_after = len(list(resource.list()))
+        while num_after != (num_before - 1) and tries < (max_wait / 4):
+            self.log.debug('{} delete check: '
+                           '{} [{}:{}] {}'.format(msg, tries,
+                                                  num_before,
+                                                  num_after,
+                                                  resource_id))
+            time.sleep(4)
+            num_after = len(list(resource.list()))
+            tries += 1
+
+        self.log.debug('{}:  expected, actual count = {}, '
+                       '{}'.format(msg, num_before - 1, num_after))
+
+        if num_after == (num_before - 1):
+            return True
+        else:
+            self.log.error('{} delete timed out'.format(msg))
+            return False
+
+    def resource_reaches_status(self, resource, resource_id,
+                                expected_stat='available',
+                                msg='resource', max_wait=120):
+        """Wait for an openstack resources status to reach an
+           expected status within a specified time.  Useful to confirm that
+           nova instances, cinder vols, snapshots, glance images, heat stacks
+           and other resources eventually reach the expected status.
+
+        :param resource: pointer to os resource type, ex: heat_client.stacks
+        :param resource_id: unique id for the openstack resource
+        :param expected_stat: status to expect resource to reach
+        :param msg: text to identify purpose in logging
+        :param max_wait: maximum wait time in seconds
+        :returns: True if successful, False if status is not reached
+        """
+
+        tries = 0
+        resource_stat = resource.get(resource_id).status
+        while resource_stat != expected_stat and tries < (max_wait / 4):
+            self.log.debug('{} status check: '
+                           '{} [{}:{}] {}'.format(msg, tries,
+                                                  resource_stat,
+                                                  expected_stat,
+                                                  resource_id))
+            time.sleep(4)
+            resource_stat = resource.get(resource_id).status
+            tries += 1
+
+        self.log.debug('{}:  expected, actual status = {}, '
+                       '{}'.format(msg, resource_stat, expected_stat))
+
+        if resource_stat == expected_stat:
+            return True
+        else:
+            self.log.debug('{} never reached expected status: '
+                           '{}'.format(resource_id, expected_stat))
+            return False
diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py
index 400eaf8e..ab400060 100644
--- a/hooks/charmhelpers/contrib/openstack/context.py
+++ b/hooks/charmhelpers/contrib/openstack/context.py
@@ -240,7 +240,7 @@ class SharedDBContext(OSContextGenerator):
         if self.relation_prefix:
             password_setting = self.relation_prefix + '_password'
 
-        for rid in relation_ids('shared-db'):
+        for rid in relation_ids(self.interfaces[0]):
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 host = rdata.get('db_host')
diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py
index b3aa3d4c..f7b72352 100644
--- a/hooks/charmhelpers/contrib/openstack/neutron.py
+++ b/hooks/charmhelpers/contrib/openstack/neutron.py
@@ -172,14 +172,16 @@ def neutron_plugins():
             'services': ['calico-felix',
                          'bird',
                          'neutron-dhcp-agent',
-                         'nova-api-metadata'],
+                         'nova-api-metadata',
+                         'etcd'],
             'packages': [[headers_package()] + determine_dkms_package(),
                          ['calico-compute',
                           'bird',
                           'neutron-dhcp-agent',
-                          'nova-api-metadata']],
-            'server_packages': ['neutron-server', 'calico-control'],
-            'server_services': ['neutron-server']
+                          'nova-api-metadata',
+                          'etcd']],
+            'server_packages': ['neutron-server', 'calico-control', 'etcd'],
+            'server_services': ['neutron-server', 'etcd']
         },
         'vsp': {
             'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini',
diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index d795a358..28532c98 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -79,6 +79,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('trusty', 'icehouse'),
     ('utopic', 'juno'),
     ('vivid', 'kilo'),
+    ('wily', 'liberty'),
 ])
 
 
@@ -91,6 +92,7 @@ OPENSTACK_CODENAMES = OrderedDict([
     ('2014.1', 'icehouse'),
     ('2014.2', 'juno'),
     ('2015.1', 'kilo'),
+    ('2015.2', 'liberty'),
 ])
 
 # The ugly duckling
@@ -113,6 +115,7 @@ SWIFT_CODENAMES = OrderedDict([
     ('2.2.0', 'juno'),
     ('2.2.1', 'kilo'),
     ('2.2.2', 'kilo'),
+    ('2.3.0', 'liberty'),
 ])
 
 DEFAULT_LOOPBACK_SIZE = '5G'
@@ -321,6 +324,9 @@ def configure_installation_source(rel):
             'kilo': 'trusty-updates/kilo',
             'kilo/updates': 'trusty-updates/kilo',
             'kilo/proposed': 'trusty-proposed/kilo',
+            'liberty': 'trusty-updates/liberty',
+            'liberty/updates': 'trusty-updates/liberty',
+            'liberty/proposed': 'trusty-proposed/liberty',
         }
 
         try:
@@ -549,6 +555,11 @@ def git_clone_and_install(projects_yaml, core_project, depth=1):
 
     pip_create_virtualenv(os.path.join(parent_dir, 'venv'))
 
+    # Upgrade setuptools from default virtualenv version. The default version
+    # in trusty breaks update.py in global requirements master branch.
+    pip_install('setuptools', upgrade=True, proxy=http_proxy,
+                venv=os.path.join(parent_dir, 'venv'))
+
     for p in projects['repositories']:
         repo = p['repository']
         branch = p['branch']
@@ -610,24 +621,24 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
     else:
         repo_dir = dest_dir
 
+    venv = os.path.join(parent_dir, 'venv')
+
     if update_requirements:
         if not requirements_dir:
             error_out('requirements repo must be cloned before '
                       'updating from global requirements.')
-        _git_update_requirements(repo_dir, requirements_dir)
+        _git_update_requirements(venv, repo_dir, requirements_dir)
 
     juju_log('Installing git repo from dir: {}'.format(repo_dir))
     if http_proxy:
-        pip_install(repo_dir, proxy=http_proxy,
-                    venv=os.path.join(parent_dir, 'venv'))
+        pip_install(repo_dir, proxy=http_proxy, venv=venv)
     else:
-        pip_install(repo_dir,
-                    venv=os.path.join(parent_dir, 'venv'))
+        pip_install(repo_dir, venv=venv)
 
     return repo_dir
 
 
-def _git_update_requirements(package_dir, reqs_dir):
+def _git_update_requirements(venv, package_dir, reqs_dir):
     """
     Update from global requirements.
 
@@ -636,12 +647,14 @@ def _git_update_requirements(package_dir, reqs_dir):
     """
     orig_dir = os.getcwd()
     os.chdir(reqs_dir)
-    cmd = ['python', 'update.py', package_dir]
+    python = os.path.join(venv, 'bin/python')
+    cmd = [python, 'update.py', package_dir]
     try:
         subprocess.check_call(cmd)
     except subprocess.CalledProcessError:
         package = os.path.basename(package_dir)
-        error_out("Error updating {} from global-requirements.txt".format(package))
+        error_out("Error updating {} from "
+                  "global-requirements.txt".format(package))
     os.chdir(orig_dir)
 
 
diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py
index 0d2ab4b4..901a4cfe 100644
--- a/hooks/charmhelpers/core/host.py
+++ b/hooks/charmhelpers/core/host.py
@@ -24,6 +24,7 @@
 import os
 import re
 import pwd
+import glob
 import grp
 import random
 import string
@@ -269,6 +270,21 @@ def file_hash(path, hash_type='md5'):
         return None
 
 
+def path_hash(path):
+    """
+    Generate a hash checksum of all files matching 'path'. Standard wildcards
+    like '*' and '?' are supported, see documentation for the 'glob' module for
+    more information.
+
+    :return: dict: A { filename: hash } dictionary for all matched files.
+                   Empty if none found.
+    """
+    return {
+        filename: file_hash(filename)
+        for filename in glob.iglob(path)
+    }
+
+
 def check_hash(path, checksum, hash_type='md5'):
     """
     Validate a file using a cryptographic checksum.
@@ -296,23 +312,25 @@ def restart_on_change(restart_map, stopstart=False):
 
         @restart_on_change({
             '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ]
+            '/etc/apache/sites-enabled/*': [ 'apache2' ]
             })
-        def ceph_client_changed():
+        def config_changed():
             pass  # your code here
 
     In this example, the cinder-api and cinder-volume services
     would be restarted if /etc/ceph/ceph.conf is changed by the
-    ceph_client_changed function.
+    ceph_client_changed function. The apache2 service would be
+    restarted if any file matching the pattern got changed, created
+    or removed. Standard wildcards are supported, see documentation
+    for the 'glob' module for more information.
     """
     def wrap(f):
         def wrapped_f(*args, **kwargs):
-            checksums = {}
-            for path in restart_map:
-                checksums[path] = file_hash(path)
+            checksums = {path: path_hash(path) for path in restart_map}
             f(*args, **kwargs)
             restarts = []
             for path in restart_map:
-                if checksums[path] != file_hash(path):
+                if path_hash(path) != checksums[path]:
                     restarts += restart_map[path]
             services_list = list(OrderedDict.fromkeys(restarts))
             if not stopstart:
diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py
index f61c2e8b..e8c4a274 100644
--- a/tests/charmhelpers/contrib/amulet/utils.py
+++ b/tests/charmhelpers/contrib/amulet/utils.py
@@ -15,13 +15,15 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import ConfigParser
+import distro_info
 import io
 import logging
+import os
 import re
+import six
 import sys
 import time
-
-import six
+import urlparse
 
 
 class AmuletUtils(object):
@@ -33,6 +35,7 @@ class AmuletUtils(object):
 
     def __init__(self, log_level=logging.ERROR):
         self.log = self.get_logger(level=log_level)
+        self.ubuntu_releases = self.get_ubuntu_releases()
 
     def get_logger(self, name="amulet-logger", level=logging.DEBUG):
         """Get a logger object that will log to stdout."""
@@ -70,12 +73,44 @@ class AmuletUtils(object):
         else:
             return False
 
-    def validate_services(self, commands):
-        """Validate services.
+    def get_ubuntu_release_from_sentry(self, sentry_unit):
+        """Get Ubuntu release codename from sentry unit.
 
-           Verify the specified services are running on the corresponding
+        :param sentry_unit: amulet sentry/service unit pointer
+        :returns: list of strings - release codename, failure message
+        """
+        msg = None
+        cmd = 'lsb_release -cs'
+        release, code = sentry_unit.run(cmd)
+        if code == 0:
+            self.log.debug('{} lsb_release: {}'.format(
+                sentry_unit.info['unit_name'], release))
+        else:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, release, code))
+        if release not in self.ubuntu_releases:
+            msg = ("Release ({}) not found in Ubuntu releases "
+                   "({})".format(release, self.ubuntu_releases))
+        return release, msg
+
+    def validate_services(self, commands):
+        """Validate that lists of commands succeed on service units.  Can be
+           used to verify system services are running on the corresponding
            service units.
-           """
+
+        :param commands: dict with sentry keys and arbitrary command list vals
+        :returns: None if successful, Failure string message otherwise
+        """
+        self.log.debug('Checking status of system services...')
+
+        # /!\ DEPRECATION WARNING (beisner):
+        # New and existing tests should be rewritten to use
+        # validate_services_by_name() as it is aware of init systems.
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'validate_services_by_name instead of validate_services '
+                      'due to init system differences.')
+
         for k, v in six.iteritems(commands):
             for cmd in v:
                 output, code = k.run(cmd)
@@ -86,6 +121,41 @@ class AmuletUtils(object):
                     return "command `{}` returned {}".format(cmd, str(code))
         return None
 
+    def validate_services_by_name(self, sentry_services):
+        """Validate system service status by service name, automatically
+           detecting init system based on Ubuntu release codename.
+
+        :param sentry_services: dict with sentry keys and svc list values
+        :returns: None if successful, Failure string message otherwise
+        """
+        self.log.debug('Checking status of system services...')
+
+        # Point at which systemd became a thing
+        systemd_switch = self.ubuntu_releases.index('vivid')
+
+        for sentry_unit, services_list in six.iteritems(sentry_services):
+            # Get lsb_release codename from unit
+            release, ret = self.get_ubuntu_release_from_sentry(sentry_unit)
+            if ret:
+                return ret
+
+            for service_name in services_list:
+                if (self.ubuntu_releases.index(release) >= systemd_switch or
+                        service_name == "rabbitmq-server"):
+                    # init is systemd
+                    cmd = 'sudo service {} status'.format(service_name)
+                elif self.ubuntu_releases.index(release) < systemd_switch:
+                    # init is upstart
+                    cmd = 'sudo status {}'.format(service_name)
+
+                output, code = sentry_unit.run(cmd)
+                self.log.debug('{} `{}` returned '
+                               '{}'.format(sentry_unit.info['unit_name'],
+                                           cmd, code))
+                if code != 0:
+                    return "command `{}` returned {}".format(cmd, str(code))
+        return None
+
     def _get_config(self, unit, filename):
         """Get a ConfigParser object for parsing a unit's config file."""
         file_contents = unit.file_contents(filename)
@@ -104,6 +174,9 @@ class AmuletUtils(object):
            Verify that the specified section of the config file contains
            the expected option key:value pairs.
            """
+        self.log.debug('Validating config file data ({} in {} on {})'
+                       '...'.format(section, config_file,
+                                    sentry_unit.info['unit_name']))
         config = self._get_config(sentry_unit, config_file)
 
         if section != 'DEFAULT' and not config.has_section(section):
@@ -321,3 +394,15 @@ class AmuletUtils(object):
 
     def endpoint_error(self, name, data):
         return 'unexpected endpoint data in {} - {}'.format(name, data)
+
+    def get_ubuntu_releases(self):
+        """Return a list of all Ubuntu releases in order of release."""
+        _d = distro_info.UbuntuDistroInfo()
+        _release_list = _d.all
+        self.log.debug('Ubuntu release list: {}'.format(_release_list))
+        return _release_list
+
+    def file_to_url(self, file_rel_path):
+        """Convert a relative file path to a file URL."""
+        _abs_path = os.path.abspath(file_rel_path)
+        return urlparse.urlparse(_abs_path, scheme='file').geturl()
diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
index 461a702f..c664c9d0 100644
--- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -110,7 +110,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
          self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
-         self.trusty_kilo, self.vivid_kilo) = range(10)
+         self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
+         self.wily_liberty) = range(12)
 
         releases = {
             ('precise', None): self.precise_essex,
@@ -121,8 +122,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('trusty', None): self.trusty_icehouse,
             ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
             ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
+            ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
             ('utopic', None): self.utopic_juno,
-            ('vivid', None): self.vivid_kilo}
+            ('vivid', None): self.vivid_kilo,
+            ('wily', None): self.wily_liberty}
         return releases[(self.series, self.openstack)]
 
     def _get_openstack_release_string(self):
@@ -138,6 +141,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('trusty', 'icehouse'),
             ('utopic', 'juno'),
             ('vivid', 'kilo'),
+            ('wily', 'liberty'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]
diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py
index 9c3d918a..576bf0b5 100644
--- a/tests/charmhelpers/contrib/openstack/amulet/utils.py
+++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py
@@ -16,15 +16,15 @@
 
 import logging
 import os
+import six
 import time
 import urllib
 
 import glanceclient.v1.client as glance_client
+import heatclient.v1.client as heat_client
 import keystoneclient.v2_0 as keystone_client
 import novaclient.v1_1.client as nova_client
 
-import six
-
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
 )
@@ -37,7 +37,7 @@ class OpenStackAmuletUtils(AmuletUtils):
     """OpenStack amulet utilities.
 
        This class inherits from AmuletUtils and has additional support
-       that is specifically for use by OpenStack charms.
+       that is specifically for use by OpenStack charm tests.
        """
 
     def __init__(self, log_level=ERROR):
@@ -51,6 +51,8 @@ class OpenStackAmuletUtils(AmuletUtils):
            Validate actual endpoint data vs expected endpoint data. The ports
            are used to find the matching endpoint.
            """
+        self.log.debug('Validating endpoint data...')
+        self.log.debug('actual: {}'.format(repr(endpoints)))
         found = False
         for ep in endpoints:
             self.log.debug('endpoint: {}'.format(repr(ep)))
@@ -77,6 +79,7 @@ class OpenStackAmuletUtils(AmuletUtils):
            Validate a list of actual service catalog endpoints vs a list of
            expected service catalog endpoints.
            """
+        self.log.debug('Validating service catalog endpoint data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for k, v in six.iteritems(expected):
             if k in actual:
@@ -93,6 +96,7 @@ class OpenStackAmuletUtils(AmuletUtils):
            Validate a list of actual tenant data vs list of expected tenant
            data.
            """
+        self.log.debug('Validating tenant data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -114,6 +118,7 @@ class OpenStackAmuletUtils(AmuletUtils):
            Validate a list of actual role data vs a list of expected role
            data.
            """
+        self.log.debug('Validating role data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -134,6 +139,7 @@ class OpenStackAmuletUtils(AmuletUtils):
            Validate a list of actual user data vs a list of expected user
            data.
            """
+        self.log.debug('Validating user data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -155,17 +161,20 @@ class OpenStackAmuletUtils(AmuletUtils):
 
            Validate a list of actual flavors vs a list of expected flavors.
            """
+        self.log.debug('Validating flavor data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         act = [a.name for a in actual]
         return self._validate_list_data(expected, act)
 
     def tenant_exists(self, keystone, tenant):
         """Return True if tenant exists."""
+        self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
                                     tenant):
         """Authenticates admin user with the keystone admin endpoint."""
+        self.log.debug('Authenticating keystone admin...')
         unit = keystone_sentry
         service_ip = unit.relation('shared-db',
                                    'mysql:shared-db')['private-address']
@@ -175,6 +184,7 @@ class OpenStackAmuletUtils(AmuletUtils):
 
     def authenticate_keystone_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with the keystone public endpoint."""
+        self.log.debug('Authenticating keystone user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
         return keystone_client.Client(username=user, password=password,
@@ -182,12 +192,21 @@ class OpenStackAmuletUtils(AmuletUtils):
 
     def authenticate_glance_admin(self, keystone):
         """Authenticates admin user with glance."""
+        self.log.debug('Authenticating glance admin...')
         ep = keystone.service_catalog.url_for(service_type='image',
                                               endpoint_type='adminURL')
         return glance_client.Client(ep, token=keystone.auth_token)
 
+    def authenticate_heat_admin(self, keystone):
+        """Authenticates the admin user with heat."""
+        self.log.debug('Authenticating heat admin...')
+        ep = keystone.service_catalog.url_for(service_type='orchestration',
+                                              endpoint_type='publicURL')
+        return heat_client.Client(endpoint=ep, token=keystone.auth_token)
+
     def authenticate_nova_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with nova-api."""
+        self.log.debug('Authenticating nova user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
         return nova_client.Client(username=user, api_key=password,
@@ -195,6 +214,7 @@ class OpenStackAmuletUtils(AmuletUtils):
 
     def create_cirros_image(self, glance, image_name):
         """Download the latest cirros image and upload it to glance."""
+        self.log.debug('Creating glance image ({})...'.format(image_name))
         http_proxy = os.getenv('AMULET_HTTP_PROXY')
         self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
         if http_proxy:
@@ -235,6 +255,11 @@ class OpenStackAmuletUtils(AmuletUtils):
 
     def delete_image(self, glance, image):
         """Delete the specified image."""
+
+        # /!\ DEPRECATION WARNING
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'delete_resource instead of delete_image.')
+        self.log.debug('Deleting glance image ({})...'.format(image))
         num_before = len(list(glance.images.list()))
         glance.images.delete(image)
 
@@ -254,6 +279,8 @@ class OpenStackAmuletUtils(AmuletUtils):
 
     def create_instance(self, nova, image_name, instance_name, flavor):
         """Create the specified instance."""
+        self.log.debug('Creating instance '
+                       '({}|{}|{})'.format(instance_name, image_name, flavor))
         image = nova.images.find(name=image_name)
         flavor = nova.flavors.find(name=flavor)
         instance = nova.servers.create(name=instance_name, image=image,
@@ -276,6 +303,11 @@ class OpenStackAmuletUtils(AmuletUtils):
 
     def delete_instance(self, nova, instance):
         """Delete the specified instance."""
+
+        # /!\ DEPRECATION WARNING
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'delete_resource instead of delete_instance.')
+        self.log.debug('Deleting instance ({})...'.format(instance))
         num_before = len(list(nova.servers.list()))
         nova.servers.delete(instance)
 
@@ -292,3 +324,90 @@ class OpenStackAmuletUtils(AmuletUtils):
             return False
 
         return True
+
+    def create_or_get_keypair(self, nova, keypair_name="testkey"):
+        """Create a new keypair, or return pointer if it already exists."""
+        try:
+            _keypair = nova.keypairs.get(keypair_name)
+            self.log.debug('Keypair ({}) already exists, '
+                           'using it.'.format(keypair_name))
+            return _keypair
+        except:
+            self.log.debug('Keypair ({}) does not exist, '
+                           'creating it.'.format(keypair_name))
+
+        _keypair = nova.keypairs.create(name=keypair_name)
+        return _keypair
+
+    def delete_resource(self, resource, resource_id,
+                        msg="resource", max_wait=120):
+        """Delete one openstack resource, such as one instance, keypair,
+        image, volume, stack, etc., and confirm deletion within max wait time.
+
+        :param resource: pointer to os resource type, ex:glance_client.images
+        :param resource_id: unique name or id for the openstack resource
+        :param msg: text to identify purpose in logging
+        :param max_wait: maximum wait time in seconds
+        :returns: True if successful, otherwise False
+        """
+        num_before = len(list(resource.list()))
+        resource.delete(resource_id)
+
+        tries = 0
+        num_after = len(list(resource.list()))
+        while num_after != (num_before - 1) and tries < (max_wait / 4):
+            self.log.debug('{} delete check: '
+                           '{} [{}:{}] {}'.format(msg, tries,
+                                                  num_before,
+                                                  num_after,
+                                                  resource_id))
+            time.sleep(4)
+            num_after = len(list(resource.list()))
+            tries += 1
+
+        self.log.debug('{}:  expected, actual count = {}, '
+                       '{}'.format(msg, num_before - 1, num_after))
+
+        if num_after == (num_before - 1):
+            return True
+        else:
+            self.log.error('{} delete timed out'.format(msg))
+            return False
+
+    def resource_reaches_status(self, resource, resource_id,
+                                expected_stat='available',
+                                msg='resource', max_wait=120):
+        """Wait for an openstack resources status to reach an
+           expected status within a specified time.  Useful to confirm that
+           nova instances, cinder vols, snapshots, glance images, heat stacks
+           and other resources eventually reach the expected status.
+
+        :param resource: pointer to os resource type, ex: heat_client.stacks
+        :param resource_id: unique id for the openstack resource
+        :param expected_stat: status to expect resource to reach
+        :param msg: text to identify purpose in logging
+        :param max_wait: maximum wait time in seconds
+        :returns: True if successful, False if status is not reached
+        """
+
+        tries = 0
+        resource_stat = resource.get(resource_id).status
+        while resource_stat != expected_stat and tries < (max_wait / 4):
+            self.log.debug('{} status check: '
+                           '{} [{}:{}] {}'.format(msg, tries,
+                                                  resource_stat,
+                                                  expected_stat,
+                                                  resource_id))
+            time.sleep(4)
+            resource_stat = resource.get(resource_id).status
+            tries += 1
+
+        self.log.debug('{}:  expected, actual status = {}, '
+                       '{}'.format(msg, resource_stat, expected_stat))
+
+        if resource_stat == expected_stat:
+            return True
+        else:
+            self.log.debug('{} never reached expected status: '
+                           '{}'.format(resource_id, expected_stat))
+            return False

From bc7438a700624362089664b38d993e00e47064db Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Wed, 24 Jun 2015 18:12:33 +0000
Subject: [PATCH 49/88] Add temporary ppa with patched neutron code

---
 hooks/neutron_api_hooks.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hooks/neutron_api_hooks.py b/hooks/neutron_api_hooks.py
index fde9e188..843d4242 100755
--- a/hooks/neutron_api_hooks.py
+++ b/hooks/neutron_api_hooks.py
@@ -143,6 +143,8 @@ def configure_https():
 def install():
     execd_preinstall()
     configure_installation_source(config('openstack-origin'))
+    # XXX Remove me when patched nova and neutron are in the main ppa
+    configure_installation_source('ppa:gnuoy/sdn-test')
 
     apt_update()
     apt_install(determine_packages(config('openstack-origin')),

From d7935cdd71d6acb6a7fbed8f587b5524c929f656 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Wed, 24 Jun 2015 19:05:48 +0000
Subject: [PATCH 50/88] Sync charm-helpers

---
 hooks/charmhelpers/contrib/python/packages.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py
index 07b0c1d7..10b32e33 100644
--- a/hooks/charmhelpers/contrib/python/packages.py
+++ b/hooks/charmhelpers/contrib/python/packages.py
@@ -36,6 +36,8 @@ __author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
 def parse_options(given, available):
     """Given a set of options, check if available"""
     for key, value in sorted(given.items()):
+        if not value:
+            continue
         if key in available:
             yield "--{0}={1}".format(key, value)
 

From 2d2e950e8f94a4bd9355f746913ecd37c3526b45 Mon Sep 17 00:00:00 2001
From: James Page <james.page@ubuntu.com>
Date: Mon, 29 Jun 2015 12:33:36 +0100
Subject: [PATCH 51/88] Fixup upgrades to ensure that only the leader attempts
 to migrate the database

---
 hooks/neutron_api_utils.py           |  3 +-
 unit_tests/test_neutron_api_utils.py | 43 ++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index 9f3724aa..8feef030 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -48,6 +48,7 @@ from charmhelpers.core.host import (
 )
 
 from charmhelpers.core.templating import render
+from charmhelpers.contrib.hahelpers.cluster import is_elected_leader
 
 import neutron_api_context
 
@@ -292,7 +293,7 @@ def do_openstack_upgrade(configs):
     # set CONFIGS to load templates from new release
     configs.set_release(openstack_release=new_os_rel)
     # Before kilo it's nova-cloud-controllers job
-    if new_os_rel >= 'kilo':
+    if is_elected_leader(CLUSTER_RES) and new_os_rel >= 'kilo':
         stamp_neutron_database(cur_os_rel)
         migrate_neutron_database()
 
diff --git a/unit_tests/test_neutron_api_utils.py b/unit_tests/test_neutron_api_utils.py
index 7de08226..f92bf27c 100644
--- a/unit_tests/test_neutron_api_utils.py
+++ b/unit_tests/test_neutron_api_utils.py
@@ -33,6 +33,7 @@ TO_PATCH = [
     'os_release',
     'pip_install',
     'subprocess',
+    'is_elected_leader',
 ]
 
 openstack_origin_git = \
@@ -190,6 +191,7 @@ class TestNeutronAPIUtils(CharmTestCase):
     def test_do_openstack_upgrade_juno(self, git_requested,
                                        stamp_neutron_db, migrate_neutron_db):
         git_requested.return_value = False
+        self.is_elected_leader.return_value = True
         self.config.side_effect = self.test_config.get
         self.test_config.set('openstack-origin', 'cloud:trusty-juno')
         self.os_release.return_value = 'icehouse'
@@ -227,6 +229,7 @@ class TestNeutronAPIUtils(CharmTestCase):
                                        stamp_neutron_db, migrate_neutron_db,
                                        gsrc):
         git_requested.return_value = False
+        self.is_elected_leader.return_value = True
         self.os_release.return_value = 'juno'
         self.config.side_effect = self.test_config.get
         self.test_config.set('openstack-origin', 'cloud:trusty-kilo')
@@ -256,6 +259,46 @@ class TestNeutronAPIUtils(CharmTestCase):
         stamp_neutron_db.assert_called_with('juno')
         migrate_neutron_db.assert_called_with()
 
+    @patch.object(charmhelpers.contrib.openstack.utils,
+                  'get_os_codename_install_source')
+    @patch.object(nutils, 'migrate_neutron_database')
+    @patch.object(nutils, 'stamp_neutron_database')
+    @patch.object(nutils, 'git_install_requested')
+    def test_do_openstack_upgrade_kilo_notleader(self, git_requested,
+                                                 stamp_neutron_db,
+                                                 migrate_neutron_db,
+                                                 gsrc):
+        git_requested.return_value = False
+        self.is_elected_leader.return_value = False
+        self.os_release.return_value = 'juno'
+        self.config.side_effect = self.test_config.get
+        self.test_config.set('openstack-origin', 'cloud:trusty-kilo')
+        gsrc.return_value = 'kilo'
+        self.get_os_codename_install_source.return_value = 'kilo'
+        configs = MagicMock()
+        nutils.do_openstack_upgrade(configs)
+        self.os_release.assert_called_with('neutron-server')
+        self.log.assert_called()
+        self.configure_installation_source.assert_called_with(
+            'cloud:trusty-kilo'
+        )
+        self.apt_update.assert_called_with(fatal=True)
+        dpkg_opts = [
+            '--option', 'Dpkg::Options::=--force-confnew',
+            '--option', 'Dpkg::Options::=--force-confdef',
+        ]
+        self.apt_upgrade.assert_called_with(options=dpkg_opts,
+                                            fatal=True,
+                                            dist=True)
+        pkgs = nutils.determine_packages()
+        pkgs.sort()
+        self.apt_install.assert_called_with(packages=pkgs,
+                                            options=dpkg_opts,
+                                            fatal=True)
+        configs.set_release.assert_called_with(openstack_release='kilo')
+        self.assertFalse(stamp_neutron_db.called)
+        self.assertFalse(migrate_neutron_db.called)
+
     @patch.object(ncontext, 'IdentityServiceContext')
     @patch('neutronclient.v2_0.client.Client')
     def test_get_neutron_client(self, nclient, IdentityServiceContext):

From d189c5ebe19f9055003d828f6f048015121d47ff Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Mon, 29 Jun 2015 13:21:05 +0100
Subject: [PATCH 52/88] Lint and tipdyup

---
 hooks/neutron_api_context.py | 23 +++++++++--------------
 hooks/neutron_api_hooks.py   |  2 --
 hooks/neutron_api_utils.py   |  5 +++--
 3 files changed, 12 insertions(+), 18 deletions(-)

diff --git a/hooks/neutron_api_context.py b/hooks/neutron_api_context.py
index 80662841..86d3172c 100644
--- a/hooks/neutron_api_context.py
+++ b/hooks/neutron_api_context.py
@@ -240,9 +240,10 @@ class NeutronApiSDNContext(context.SubordinateConfigContext):
     interfaces = 'neutron-plugin-api-subordinate'
 
     def __init__(self):
-        super(NeutronApiSDNContext, self).__init__(interface='neutron-plugin-api-subordinate',
-                                                   service='neutron-api',
-                                                   config_file='/etc/neutron/neutron.conf')
+        super(NeutronApiSDNContext, self).__init__(
+            interface='neutron-plugin-api-subordinate',
+            service='neutron-api',
+            config_file='/etc/neutron/neutron.conf')
 
     def __call__(self):
         ctxt = super(NeutronApiSDNContext, self).__call__()
@@ -280,21 +281,15 @@ class NeutronApiSDNContext(context.SubordinateConfigContext):
                 return ctxt
         return ctxt
 
+
 class NeutronApiSDNConfigFileContext(context.OSContextGenerator):
     interfaces = ['neutron-plugin-api-subordinate']
 
     def __call__(self):
-        ctxt = {}
-        defaults = {
-            'neutron-plugin-config': {
-                'templ_key': 'neutron_plugin_config',
-                'value': '/etc/neutron/plugins/ml2/ml2_conf.ini',
-            },
-        }
         for rid in relation_ids('neutron-plugin-api-subordinate'):
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
-                neutron_server_plugin_config = rdata.get('neutron-plugin-config')
-                if neutron_server_plugin_config:
-                    return { 'config': neutron_server_plugin_config }
-        return { 'config': '/etc/neutron/plugins/ml2/ml2_conf.ini' }
+                neutron_server_plugin_conf = rdata.get('neutron-plugin-config')
+                if neutron_server_plugin_conf:
+                    return {'config': neutron_server_plugin_conf}
+        return {'config': '/etc/neutron/plugins/ml2/ml2_conf.ini'}
diff --git a/hooks/neutron_api_hooks.py b/hooks/neutron_api_hooks.py
index 843d4242..fde9e188 100755
--- a/hooks/neutron_api_hooks.py
+++ b/hooks/neutron_api_hooks.py
@@ -143,8 +143,6 @@ def configure_https():
 def install():
     execd_preinstall()
     configure_installation_source(config('openstack-origin'))
-    # XXX Remove me when patched nova and neutron are in the main ppa
-    configure_installation_source('ppa:gnuoy/sdn-test')
 
     apt_update()
     apt_install(determine_packages(config('openstack-origin')),
diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index ca105a0e..b6f11493 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -161,6 +161,7 @@ def api_port(service):
 def manage_plugin():
     return config('manage-neutron-plugin-legacy-mode')
 
+
 def determine_packages(source=None):
     # currently all packages match service names
     packages = [] + BASE_PACKAGES
@@ -214,8 +215,8 @@ def resource_map():
         resource_map.pop(APACHE_24_CONF)
 
     if manage_plugin():
-        # add neutron plugin requirements. nova-c-c only needs the neutron-server
-        # associated with configs, not the plugin agent.
+        # add neutron plugin requirements. nova-c-c only needs the
+        # neutron-server associated with configs, not the plugin agent.
         plugin = config('neutron-plugin')
         conf = neutron_plugin_attribute(plugin, 'config', 'neutron')
         ctxts = (neutron_plugin_attribute(plugin, 'contexts', 'neutron')

From 36fc3a0fcbc5b6e5d2d0bfb5874504458afb4112 Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Tue, 30 Jun 2015 09:26:51 +0100
Subject: [PATCH 53/88] Add unit tests for new contexts

---
 hooks/neutron_api_context.py           |   5 +-
 unit_tests/test_neutron_api_context.py | 133 +++++++++++++++++++++++++
 2 files changed, 136 insertions(+), 2 deletions(-)

diff --git a/hooks/neutron_api_context.py b/hooks/neutron_api_context.py
index 86d3172c..a7e60af3 100644
--- a/hooks/neutron_api_context.py
+++ b/hooks/neutron_api_context.py
@@ -268,9 +268,10 @@ class NeutronApiSDNContext(context.SubordinateConfigContext):
         for rid in relation_ids('neutron-plugin-api-subordinate'):
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
-                ctxt['neutron_plugin'] = rdata.get('neutron-plugin')
-                if not context.context_complete(ctxt):
+                plugin = rdata.get('neutron-plugin')
+                if not plugin:
                     continue
+                ctxt['neutron_plugin'] = plugin
                 for key in defaults.keys():
                     remote_value = rdata.get(key)
                     ctxt_key = defaults[key]['templ_key']
diff --git a/unit_tests/test_neutron_api_context.py b/unit_tests/test_neutron_api_context.py
index 11edcffa..a0ef6af4 100644
--- a/unit_tests/test_neutron_api_context.py
+++ b/unit_tests/test_neutron_api_context.py
@@ -1,3 +1,4 @@
+import json
 from test_utils import CharmTestCase
 from mock import patch
 import neutron_api_context as context
@@ -432,3 +433,135 @@ class NeutronCCContextTest(CharmTestCase):
         }
         for key in expect.iterkeys():
             self.assertEquals(napi_ctxt[key], expect[key])
+
+
+class NeutronApiSDNContextTest(CharmTestCase):
+
+    def setUp(self):
+        super(NeutronApiSDNContextTest, self).setUp(context, TO_PATCH)
+        self.relation_get.side_effect = self.test_relation.get
+
+    def tearDown(self):
+        super(NeutronApiSDNContextTest, self).tearDown()
+
+    def test_init(self):
+        napisdn_ctxt = context.NeutronApiSDNContext()
+        self.assertEquals(
+            napisdn_ctxt.interface,
+            'neutron-plugin-api-subordinate'
+        )
+        self.assertEquals(napisdn_ctxt.service, 'neutron-api')
+        self.assertEquals(
+            napisdn_ctxt.config_file,
+            '/etc/neutron/neutron.conf'
+        )
+
+    @patch.object(charmhelpers.contrib.openstack.context, 'log')
+    @patch.object(charmhelpers.contrib.openstack.context, 'relation_get')
+    @patch.object(charmhelpers.contrib.openstack.context, 'related_units')
+    @patch.object(charmhelpers.contrib.openstack.context, 'relation_ids')
+    def ctxt_check(self, rel_settings, expect, _rids, _runits, _rget, _log):
+        self.test_relation.set(rel_settings)
+        _runits.return_value = ['unit1']
+        _rids.return_value = ['rid2']
+        _rget.side_effect = self.test_relation.get
+        self.relation_ids.return_value = ['rid2']
+        self.related_units.return_value = ['unit1']
+        napisdn_ctxt = context.NeutronApiSDNContext()()
+        self.assertEquals(napisdn_ctxt, expect)
+
+    def test_defaults(self):
+        self.ctxt_check(
+            {'neutron-plugin': 'ovs'},
+            {
+                'core_plugin': 'neutron.plugins.ml2.plugin.Ml2Plugin',
+                'neutron_plugin_config': ('/etc/neutron/plugins/ml2/'
+                                          'ml2_conf.ini'),
+                'service_plugins': 'router,firewall,lbaas,vpnaas,metering',
+                'restart_trigger': '',
+                'neutron_plugin': 'ovs',
+                'sections': {},
+            }
+        )
+
+    def test_overrides(self):
+        self.ctxt_check(
+            {
+                'neutron-plugin': 'ovs',
+                'core-plugin': 'neutron.plugins.ml2.plugin.MidoPlumODL',
+                'neutron-plugin-config': '/etc/neutron/plugins/fl/flump.ini',
+                'service-plugins': 'router,unicorn,rainbows',
+                'restart-trigger': 'restartnow',
+            },
+            {
+                'core_plugin': 'neutron.plugins.ml2.plugin.MidoPlumODL',
+                'neutron_plugin_config': '/etc/neutron/plugins/fl/flump.ini',
+                'service_plugins': 'router,unicorn,rainbows',
+                'restart_trigger': 'restartnow',
+                'neutron_plugin': 'ovs',
+                'sections': {},
+            }
+        )
+
+    def test_subordinateconfig(self):
+        principle_config = {
+            "neutron-api": {
+                "/etc/neutron/neutron.conf": {
+                    "sections": {
+                        'DEFAULT': [
+                            ('neutronboost', True)
+                        ],
+                    }
+                }
+            }
+        }
+        self.ctxt_check(
+            {
+                'neutron-plugin': 'ovs',
+                'subordinate_configuration': json.dumps(principle_config),
+            },
+            {
+                'core_plugin': 'neutron.plugins.ml2.plugin.Ml2Plugin',
+                'neutron_plugin_config': ('/etc/neutron/plugins/ml2/'
+                                          'ml2_conf.ini'),
+                'service_plugins': 'router,firewall,lbaas,vpnaas,metering',
+                'restart_trigger': '',
+                'neutron_plugin': 'ovs',
+                'sections': {u'DEFAULT': [[u'neutronboost', True]]},
+            }
+        )
+
+    def test_empty(self):
+        self.ctxt_check(
+            {},
+            {'sections': {}},
+        )
+
+
+class NeutronApiSDNConfigFileContextTest(CharmTestCase):
+
+    def setUp(self):
+        super(NeutronApiSDNConfigFileContextTest, self).setUp(
+            context, TO_PATCH)
+        self.relation_get.side_effect = self.test_relation.get
+
+    def tearDown(self):
+        super(NeutronApiSDNConfigFileContextTest, self).tearDown()
+
+    def test_configset(self):
+        self.test_relation.set({
+            'neutron-plugin-config': '/etc/neutron/superplugin.ini'
+        })
+        self.relation_ids.return_value = ['rid2']
+        self.related_units.return_value = ['unit1']
+        napisdn_ctxt = context.NeutronApiSDNConfigFileContext()()
+        self.assertEquals(napisdn_ctxt, {
+            'config': '/etc/neutron/superplugin.ini'
+        })
+
+    def test_default(self):
+        self.relation_ids.return_value = []
+        napisdn_ctxt = context.NeutronApiSDNConfigFileContext()()
+        self.assertEquals(napisdn_ctxt, {
+            'config': '/etc/neutron/plugins/ml2/ml2_conf.ini'
+        })

From a6c3663eb806bdff86a82fe25a97027cec2b4544 Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Tue, 30 Jun 2015 10:03:16 +0100
Subject: [PATCH 54/88] Add unit tests for new utils and add missing relation
 hook links

---
 ...on-plugin-api-subordinate-relation-changed |  1 +
 ...n-plugin-api-subordinate-relation-departed |  1 +
 ...ron-plugin-api-subordinate-relation-joined |  1 +
 hooks/neutron_api_utils.py                    |  3 +-
 unit_tests/test_neutron_api_utils.py          | 43 ++++++++++++++++++-
 5 files changed, 45 insertions(+), 4 deletions(-)
 create mode 120000 hooks/neutron-plugin-api-subordinate-relation-changed
 create mode 120000 hooks/neutron-plugin-api-subordinate-relation-departed
 create mode 120000 hooks/neutron-plugin-api-subordinate-relation-joined

diff --git a/hooks/neutron-plugin-api-subordinate-relation-changed b/hooks/neutron-plugin-api-subordinate-relation-changed
new file mode 120000
index 00000000..1fb10fd5
--- /dev/null
+++ b/hooks/neutron-plugin-api-subordinate-relation-changed
@@ -0,0 +1 @@
+neutron_api_hooks.py
\ No newline at end of file
diff --git a/hooks/neutron-plugin-api-subordinate-relation-departed b/hooks/neutron-plugin-api-subordinate-relation-departed
new file mode 120000
index 00000000..1fb10fd5
--- /dev/null
+++ b/hooks/neutron-plugin-api-subordinate-relation-departed
@@ -0,0 +1 @@
+neutron_api_hooks.py
\ No newline at end of file
diff --git a/hooks/neutron-plugin-api-subordinate-relation-joined b/hooks/neutron-plugin-api-subordinate-relation-joined
new file mode 120000
index 00000000..1fb10fd5
--- /dev/null
+++ b/hooks/neutron-plugin-api-subordinate-relation-joined
@@ -0,0 +1 @@
+neutron_api_hooks.py
\ No newline at end of file
diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index b6f11493..e5589178 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -234,12 +234,11 @@ def resource_map():
             context.PostgresqlDBContext(database=config('database')))
 
     else:
-        print "Adding NeutronApiSDNContext"
         resource_map[NEUTRON_CONF]['contexts'].append(
             neutron_api_context.NeutronApiSDNContext()
         )
         resource_map[NEUTRON_DEFAULT]['contexts'] = \
-            neutron_api_context.NeutronApiSDNConfigFileContext()
+            [neutron_api_context.NeutronApiSDNConfigFileContext()]
     return resource_map
 
 
diff --git a/unit_tests/test_neutron_api_utils.py b/unit_tests/test_neutron_api_utils.py
index 7de08226..207dcda1 100644
--- a/unit_tests/test_neutron_api_utils.py
+++ b/unit_tests/test_neutron_api_utils.py
@@ -103,28 +103,57 @@ class TestNeutronAPIUtils(CharmTestCase):
         expect.extend(nutils.KILO_PACKAGES)
         self.assertItemsEqual(pkg_list, expect)
 
+    @patch.object(nutils, 'git_install_requested')
+    def test_determine_packages_noplugin(self, git_requested):
+        git_requested.return_value = False
+        self.test_config.set('manage-neutron-plugin-legacy-mode', False)
+        pkg_list = nutils.determine_packages()
+        expect = deepcopy(nutils.BASE_PACKAGES)
+        expect.extend(['neutron-server'])
+        self.assertItemsEqual(pkg_list, expect)
+
     def test_determine_ports(self):
         port_list = nutils.determine_ports()
         self.assertItemsEqual(port_list, [9696])
 
+    @patch.object(nutils, 'manage_plugin')
     @patch('os.path.exists')
-    def test_resource_map(self, _path_exists):
+    def test_resource_map(self, _path_exists, _manage_plugin):
         _path_exists.return_value = False
+        _manage_plugin.return_value = True
         _map = nutils.resource_map()
         confs = [nutils.NEUTRON_CONF, nutils.NEUTRON_DEFAULT,
                  nutils.APACHE_CONF]
         [self.assertIn(q_conf, _map.keys()) for q_conf in confs]
         self.assertTrue(nutils.APACHE_24_CONF not in _map.keys())
 
+    @patch.object(nutils, 'manage_plugin')
     @patch('os.path.exists')
-    def test_resource_map_apache24(self, _path_exists):
+    def test_resource_map_apache24(self, _path_exists, _manage_plugin):
         _path_exists.return_value = True
+        _manage_plugin.return_value = True
         _map = nutils.resource_map()
         confs = [nutils.NEUTRON_CONF, nutils.NEUTRON_DEFAULT,
                  nutils.APACHE_24_CONF]
         [self.assertIn(q_conf, _map.keys()) for q_conf in confs]
         self.assertTrue(nutils.APACHE_CONF not in _map.keys())
 
+    @patch.object(nutils, 'manage_plugin')
+    @patch('os.path.exists')
+    def test_resource_map_noplugin(self, _path_exists, _manage_plugin):
+        _path_exists.return_value = True
+        _manage_plugin.return_value = False
+        _map = nutils.resource_map()
+        found_sdn_ctxt = False
+        found_sdnconfig_ctxt = False
+        for ctxt in _map[nutils.NEUTRON_CONF]['contexts']:
+            if isinstance(ctxt, ncontext.NeutronApiSDNContext):
+                found_sdn_ctxt = True
+        for ctxt in _map[nutils.NEUTRON_DEFAULT]['contexts']:
+            if isinstance(ctxt, ncontext.NeutronApiSDNConfigFileContext):
+                found_sdnconfig_ctxt = True
+        self.assertTrue(found_sdn_ctxt and found_sdnconfig_ctxt)
+
     @patch('os.path.exists')
     def test_restart_map(self, mock_path_exists):
         mock_path_exists.return_value = False
@@ -477,3 +506,13 @@ class TestNeutronAPIUtils(CharmTestCase):
                'upgrade',
                'head']
         self.subprocess.check_output.assert_called_with(cmd)
+
+    def test_manage_plugin_true(self):
+        self.test_config.set('manage-neutron-plugin-legacy-mode', True)
+        manage = nutils.manage_plugin()
+        self.assertTrue(manage)
+
+    def test_manage_plugin_false(self):
+        self.test_config.set('manage-neutron-plugin-legacy-mode', False)
+        manage = nutils.manage_plugin()
+        self.assertFalse(manage)

From 141fef1dbeda551492a6f54e9183b499f8c89b5c Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Wed, 8 Jul 2015 08:19:34 +0000
Subject: [PATCH 55/88] Point at ppa and pin packages

---
 files/patched-icehouse     | 4 ++++
 hooks/neutron_api_hooks.py | 7 +++++--
 2 files changed, 9 insertions(+), 2 deletions(-)
 create mode 100644 files/patched-icehouse

diff --git a/files/patched-icehouse b/files/patched-icehouse
new file mode 100644
index 00000000..c7bea34c
--- /dev/null
+++ b/files/patched-icehouse
@@ -0,0 +1,4 @@
+[ /etc/apt/preferences.d/patched-icehouse ]
+Package: *
+Pin: release o=LP-PPA-sdn-charmers-cisco-vpp-testing
+Pin-Priority: 990
diff --git a/hooks/neutron_api_hooks.py b/hooks/neutron_api_hooks.py
index 843d4242..919da5d1 100755
--- a/hooks/neutron_api_hooks.py
+++ b/hooks/neutron_api_hooks.py
@@ -9,6 +9,7 @@ from subprocess import (
 from charmhelpers.core.hookenv import (
     Hooks,
     UnregisteredHookError,
+    charm_dir,
     config,
     is_relation_made,
     local_unit,
@@ -144,8 +145,10 @@ def install():
     execd_preinstall()
     configure_installation_source(config('openstack-origin'))
     # XXX Remove me when patched nova and neutron are in the main ppa
-    configure_installation_source('ppa:gnuoy/sdn-test')
-
+    configure_installation_source('ppa:sdn-charmers/cisco-vpp-testing')
+    apt_pin_file = charm_dir() + '/files/patched-icehouse'
+    import shutil
+    shutil.copyfile(apt_pin_file, '/etc/apt/preferences.d/patched-icehouse')
     apt_update()
     apt_install(determine_packages(config('openstack-origin')),
                 fatal=True)

From c77c21ba039cde7052f3f454b094af8d1a87deba Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Wed, 8 Jul 2015 13:31:40 -0400
Subject: [PATCH 56/88] [corey.bryant,trivial] Add basic-trusty-kilo-git amulet
 tests.

---
 tests/052-basic-trusty-kilo-git | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100755 tests/052-basic-trusty-kilo-git

diff --git a/tests/052-basic-trusty-kilo-git b/tests/052-basic-trusty-kilo-git
new file mode 100755
index 00000000..f5542acd
--- /dev/null
+++ b/tests/052-basic-trusty-kilo-git
@@ -0,0 +1,12 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic neutron-api git deployment on trusty-kilo."""
+
+from basic_deployment import NeutronAPIBasicDeployment
+
+if __name__ == '__main__':
+    deployment = NeutronAPIBasicDeployment(series='trusty',
+                                           openstack='cloud:trusty-kilo',
+                                           source='cloud:trusty-updates/kilo',
+                                           git=True)
+    deployment.run_tests()

From 4c2a6830be0545c4b7c1507494504440642b3249 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Thu, 9 Jul 2015 11:40:37 -0400
Subject: [PATCH 57/88] [corey.bryant,trivial] Drop icehouse git amulet tests
 since upstream is EOL.

---
 tests/050-basic-trusty-icehouse-git | 9 ---------
 1 file changed, 9 deletions(-)
 delete mode 100755 tests/050-basic-trusty-icehouse-git

diff --git a/tests/050-basic-trusty-icehouse-git b/tests/050-basic-trusty-icehouse-git
deleted file mode 100755
index 51517017..00000000
--- a/tests/050-basic-trusty-icehouse-git
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/usr/bin/python
-
-"""Amulet tests on a basic neutron-api git deployment on trusty-icehouse."""
-
-from basic_deployment import NeutronAPIBasicDeployment
-
-if __name__ == '__main__':
-    deployment = NeutronAPIBasicDeployment(series='trusty', git=True)
-    deployment.run_tests()

From bc3ea309ae23d1b31ca9262d3b2767a9a35d5836 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Thu, 9 Jul 2015 18:16:48 +0000
Subject: [PATCH 58/88] [corey.bryant,trivial] Add icehouse git amulet tests
 back with new branches.

---
 tests/050-basic-trusty-icehouse-git |  9 +++++++++
 tests/basic_deployment.py           | 11 ++++++++---
 2 files changed, 17 insertions(+), 3 deletions(-)
 create mode 100755 tests/050-basic-trusty-icehouse-git

diff --git a/tests/050-basic-trusty-icehouse-git b/tests/050-basic-trusty-icehouse-git
new file mode 100755
index 00000000..51517017
--- /dev/null
+++ b/tests/050-basic-trusty-icehouse-git
@@ -0,0 +1,9 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic neutron-api git deployment on trusty-icehouse."""
+
+from basic_deployment import NeutronAPIBasicDeployment
+
+if __name__ == '__main__':
+    deployment = NeutronAPIBasicDeployment(series='trusty', git=True)
+    deployment.run_tests()
diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index ecd9f1b1..8c7d5009 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -107,16 +107,21 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
         """Configure all of the services."""
         neutron_api_config = {}
         if self.git:
-            branch = 'stable/' + self._get_openstack_release_string()
+            release = self._get_openstack_release_string()
+            reqs_branch = 'stable/' + release
+            if self._get_openstack_release() == self.trusty_icehouse:
+                neutron_branch = release + '-eol'
+            else:
+                neutron_branch = 'stable/' + release
             amulet_http_proxy = os.environ.get('AMULET_HTTP_PROXY')
             openstack_origin_git = {
                 'repositories': [
                     {'name': 'requirements',
                      'repository': 'git://github.com/openstack/requirements',
-                     'branch': branch},
+                     'branch': reqs_branch},
                     {'name': 'neutron',
                      'repository': 'git://github.com/openstack/neutron',
-                     'branch': branch},
+                     'branch': neutron_branch},
                 ],
                 'directory': '/mnt/openstack-git',
                 'http_proxy': amulet_http_proxy,

From 6951febb972944758a4994c727bada4fc35f7e3c Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Fri, 10 Jul 2015 13:28:48 +0000
Subject: [PATCH 59/88] Install python-neutronclient for deploy from source
 because utils function imports from neutronclient and charm itself doesn't
 run in virtualenv.

---
 hooks/neutron_api_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index 8feef030..9728a52a 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -81,6 +81,7 @@ BASE_GIT_PACKAGES = [
     'libxslt1-dev',
     'libyaml-dev',
     'python-dev',
+    'python-neutronclient', # required for get_neutron_client() import
     'python-pip',
     'python-setuptools',
     'zlib1g-dev',

From dbc90a2fb893785f6a01ddce73ef5387502bb84e Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Fri, 10 Jul 2015 13:34:03 +0000
Subject: [PATCH 60/88] Add amulet pre-req python-distro-info.

---
 tests/00-setup | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/00-setup b/tests/00-setup
index 06cfdb07..27476744 100755
--- a/tests/00-setup
+++ b/tests/00-setup
@@ -5,6 +5,7 @@ set -ex
 sudo add-apt-repository --yes ppa:juju/stable
 sudo apt-get update --yes
 sudo apt-get install --yes python-amulet \
+                           python-distro-info \
                            python-neutronclient \
                            python-keystoneclient \
                            python-novaclient \

From 80e6ca1b9f99034e6853c8b79f32334cb266e19a Mon Sep 17 00:00:00 2001
From: Edward Hope-Morley <edward.hope-morley@canonical.com>
Date: Fri, 10 Jul 2015 15:14:33 +0100
Subject: [PATCH 61/88] [trivial] Cleanup config.yaml

Partially-Closes-Bug: 1473426
---
 config.yaml | 77 +++++++++++++++++++++++++++++++----------------------
 1 file changed, 45 insertions(+), 32 deletions(-)

diff --git a/config.yaml b/config.yaml
index 05d366cd..13c9511f 100644
--- a/config.yaml
+++ b/config.yaml
@@ -1,4 +1,17 @@
 options:
+  debug:
+    default: False
+    type: boolean
+    description: Enable debug logging.
+  verbose:
+    default: False
+    type: boolean
+    description: Enable verbose logging.
+  use-syslog:
+    type: boolean
+    default: False
+    description: |
+      Setting this to True will allow supporting services to log to syslog.
   openstack-origin:
     default: distro
     type: string
@@ -7,17 +20,27 @@ options:
       distro (default), ppa:somecustom/ppa, a deb url sources entry,
       or a supported Cloud Archive release pocket.
 
-      Supported Cloud Archive sources include: cloud:precise-folsom,
-      cloud:precise-folsom/updates, cloud:precise-folsom/staging,
-      cloud:precise-folsom/proposed.
+      Supported Cloud Archive sources include:
 
-      Note that updating this setting to a source that is known to
-      provide a later version of OpenStack will trigger a software
-      upgrade.
+      cloud:<series>-<openstack-release>
+      cloud:<series>-<openstack-release>/updates
+      cloud:<series>-<openstack-release>/staging
+      cloud:<series>-<openstack-release>/proposed
 
-      Note that when openstack-origin-git is specified, openstack
-      specific packages will be installed from source rather than
-      from the openstack-origin repository.
+      For series=Precise we support cloud archives for openstack-release:
+         * icehouse
+
+      For series=Trusty we support cloud archives for openstack-release:
+         * juno
+         * kilo
+         * ...
+
+      NOTE: updating this setting to a source that is known to provide
+      a later version of OpenStack will trigger a software upgrade.
+
+      NOTE: when openstack-origin-git is specified, openstack specific
+      packages will be installed from source rather than from the
+      openstack-origin repository.
   openstack-origin-git:
     default:
     type: string
@@ -46,11 +69,6 @@ options:
     default: neutron
     type: string
     description: Database name for Neutron (if enabled)
-  use-syslog:
-    type: boolean
-    default: False
-    description: |
-      If set to True, supporting services will log to syslog.
   region:
     default: RegionOne
     type: string
@@ -63,7 +81,9 @@ options:
   neutron-external-network:
     type: string
     default: ext_net
-    description: Name of the external network for floating IP addresses provided by Neutron.
+    description: |
+      Name of the external network for floating IP addresses provided by
+      Neutron.
   network-device-mtu:
     type: int
     default:
@@ -145,10 +165,10 @@ options:
     default: -1
     type: int
     description: |
-      Number of pool members allowed per tenant. A negative value means unlimited.
-      The default is unlimited because a member is not a real resource consumer
-      on Openstack. However, on back-end, a member is a resource consumer
-      and that is the reason why quota is possible.
+      Number of pool members allowed per tenant. A negative value means
+      unlimited. The default is unlimited because a member is not a real
+      resource consumer on Openstack. However, on back-end, a member is a
+      resource consumer and that is the reason why quota is possible.
   quota-health-monitors:
     default: -1
     type: int
@@ -156,8 +176,8 @@ options:
       Number of health monitors allowed per tenant. A negative value means
       unlimited.
       The default is unlimited because a health monitor is not a real resource
-      consumer on Openstack. However, on back-end, a member is a resource consumer
-      and that is the reason why quota is possible.
+      consumer on Openstack. However, on back-end, a member is a resource
+      consumer and that is the reason why quota is possible.
   quota-router:
     default: 10
     type: int
@@ -167,7 +187,8 @@ options:
     default: 50
     type: int
     description: |
-      Number of floating IPs allowed per tenant. A negative value means unlimited.
+      Number of floating IPs allowed per tenant. A negative value means
+      unlimited.
   # HA configuration settings
   vip:
     type: string
@@ -181,8 +202,8 @@ options:
     type: string
     default: eth0
     description: |
-      Default network interface to use for HA vip when it cannot be automatically
-      determined.
+      Default network interface to use for HA vip when it cannot be
+      automatically determined.
   vip_cidr:
     type: int
     default: 24
@@ -201,14 +222,6 @@ options:
     description: |
        Default multicast port number that will be used to communicate between
        HA Cluster nodes.
-  debug:
-    default: False
-    type: boolean
-    description: Enable debug logging
-  verbose:
-    default: False
-    type: boolean
-    description: Enable verbose logging
   # Network configuration options
   # by default all access is over 'private-address'
   os-admin-network:

From 3d22b0ccf7fae56df9efb64de3482d8025a6d515 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Sun, 12 Jul 2015 20:37:02 +0000
Subject: [PATCH 62/88] Have icehouse amulet tests use repo with pinned oslo
 requirements and add neutron-*aas repos for kilo amulet tests.

---
 tests/basic_deployment.py | 57 ++++++++++++++++++++++++++-------------
 1 file changed, 38 insertions(+), 19 deletions(-)

diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index 8c7d5009..f71e7d5f 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -107,26 +107,45 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
         """Configure all of the services."""
         neutron_api_config = {}
         if self.git:
-            release = self._get_openstack_release_string()
-            reqs_branch = 'stable/' + release
-            if self._get_openstack_release() == self.trusty_icehouse:
-                neutron_branch = release + '-eol'
-            else:
-                neutron_branch = 'stable/' + release
+            branch = 'stable/' + self._get_openstack_release_string()
             amulet_http_proxy = os.environ.get('AMULET_HTTP_PROXY')
-            openstack_origin_git = {
-                'repositories': [
-                    {'name': 'requirements',
-                     'repository': 'git://github.com/openstack/requirements',
-                     'branch': reqs_branch},
-                    {'name': 'neutron',
-                     'repository': 'git://github.com/openstack/neutron',
-                     'branch': neutron_branch},
-                ],
-                'directory': '/mnt/openstack-git',
-                'http_proxy': amulet_http_proxy,
-                'https_proxy': amulet_http_proxy,
-            }
+            if self._get_openstack_release() >= self.trusty_kilo:
+                openstack_origin_git = {
+                    'repositories': [
+                        {'name': 'requirements',
+                         'repository': 'git://github.com/openstack/requirements',
+                         'branch': branch},
+                        {'name': 'neutron-fwaas',
+                         'repository': 'git://github.com/coreycb/neutron-fwaas',
+                         'branch': branch},
+                        {'name': 'neutron-lbaas',
+                         'repository': 'git://github.com/coreycb/neutron-lbaas',
+                         'branch': branch},
+                        {'name': 'neutron-vpnaas',
+                         'repository': 'git://github.com/coreycb/neutron-vpnaas',
+                         'branch': branch},
+                        {'name': 'neutron',
+                         'repository': 'git://github.com/coreycb/neutron',
+                         'branch': branch},
+                    ],
+                    'directory': '/mnt/openstack-git',
+                    'http_proxy': amulet_http_proxy,
+                    'https_proxy': amulet_http_proxy,
+                }
+            else:
+                openstack_origin_git = {
+                    'repositories': [
+                        {'name': 'requirements',
+                         'repository': 'git://github.com/openstack/requirements',
+                         'branch': branch},
+                        {'name': 'neutron',
+                         'repository': 'git://github.com/coreycb/neutron',
+                         'branch': branch},
+                    ],
+                    'directory': '/mnt/openstack-git',
+                    'http_proxy': amulet_http_proxy,
+                    'https_proxy': amulet_http_proxy,
+                }
             neutron_api_config['openstack-origin-git'] = yaml.dump(openstack_origin_git)
         keystone_config = {'admin-password': 'openstack',
                            'admin-token': 'ubuntutesting'}

From c4cdce8ce5d6cca73657c927f45749f15260f91a Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Sun, 12 Jul 2015 21:03:26 +0000
Subject: [PATCH 63/88] Fixup amulet git repos

---
 tests/basic_deployment.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index f71e7d5f..50a94160 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -116,16 +116,16 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
                          'repository': 'git://github.com/openstack/requirements',
                          'branch': branch},
                         {'name': 'neutron-fwaas',
-                         'repository': 'git://github.com/coreycb/neutron-fwaas',
+                         'repository': 'git://github.com/openstack/neutron-fwaas',
                          'branch': branch},
                         {'name': 'neutron-lbaas',
-                         'repository': 'git://github.com/coreycb/neutron-lbaas',
+                         'repository': 'git://github.com/openstack/neutron-lbaas',
                          'branch': branch},
                         {'name': 'neutron-vpnaas',
-                         'repository': 'git://github.com/coreycb/neutron-vpnaas',
+                         'repository': 'git://github.com/openstack/neutron-vpnaas',
                          'branch': branch},
                         {'name': 'neutron',
-                         'repository': 'git://github.com/coreycb/neutron',
+                         'repository': 'git://github.com/openstack/neutron',
                          'branch': branch},
                     ],
                     'directory': '/mnt/openstack-git',
@@ -133,13 +133,19 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
                     'https_proxy': amulet_http_proxy,
                 }
             else:
+                if self._get_openstack_release() == self.trusty_icehouse:
+                    reqs_repo = 'git://github.com/coreycb/requirements'
+                    neutron_repo = 'git://github.com/coreycb/neutron'
+                else:
+                    reqs_repo = 'git://github.com/openstack/requirements'
+                    neutron_repo = 'git://github.com/openstack/neutron'
                 openstack_origin_git = {
                     'repositories': [
                         {'name': 'requirements',
-                         'repository': 'git://github.com/openstack/requirements',
+                         'repository': reqs_repo,
                          'branch': branch},
                         {'name': 'neutron',
-                         'repository': 'git://github.com/coreycb/neutron',
+                         'repository': neutron_repo,
                          'branch': branch},
                     ],
                     'directory': '/mnt/openstack-git',

From 6c6384366b74a269a07e9aed3e18b8a81179f41f Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Mon, 13 Jul 2015 11:26:11 +0000
Subject: [PATCH 64/88] Switch amulet icehouse tests back to using upstream
 core repositories.

---
 tests/basic_deployment.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index 50a94160..075a6dbf 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -135,10 +135,9 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
             else:
                 if self._get_openstack_release() == self.trusty_icehouse:
                     reqs_repo = 'git://github.com/coreycb/requirements'
-                    neutron_repo = 'git://github.com/coreycb/neutron'
                 else:
                     reqs_repo = 'git://github.com/openstack/requirements'
-                    neutron_repo = 'git://github.com/openstack/neutron'
+                neutron_repo = 'git://github.com/openstack/neutron'
                 openstack_origin_git = {
                     'repositories': [
                         {'name': 'requirements',

From b0fe2ac311efa0678d72e947fc03832cdacf32e5 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Mon, 13 Jul 2015 12:44:41 +0000
Subject: [PATCH 65/88] More amulet deploy from source branch fixups.

---
 tests/basic_deployment.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index 075a6dbf..0fa0457a 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -107,26 +107,30 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
         """Configure all of the services."""
         neutron_api_config = {}
         if self.git:
-            branch = 'stable/' + self._get_openstack_release_string()
             amulet_http_proxy = os.environ.get('AMULET_HTTP_PROXY')
+
+            release = self._get_openstack_release_string()
+            reqs_branch = 'stable/' + release
+
             if self._get_openstack_release() >= self.trusty_kilo:
+                neutron_branch = 'stable/' + release
                 openstack_origin_git = {
                     'repositories': [
                         {'name': 'requirements',
                          'repository': 'git://github.com/openstack/requirements',
-                         'branch': branch},
+                         'branch': reqs_branch},
                         {'name': 'neutron-fwaas',
                          'repository': 'git://github.com/openstack/neutron-fwaas',
-                         'branch': branch},
+                         'branch': neutron_branch},
                         {'name': 'neutron-lbaas',
                          'repository': 'git://github.com/openstack/neutron-lbaas',
-                         'branch': branch},
+                         'branch': neutron_branch},
                         {'name': 'neutron-vpnaas',
                          'repository': 'git://github.com/openstack/neutron-vpnaas',
-                         'branch': branch},
+                         'branch': neutron_branch},
                         {'name': 'neutron',
                          'repository': 'git://github.com/openstack/neutron',
-                         'branch': branch},
+                         'branch': neutron_branch},
                     ],
                     'directory': '/mnt/openstack-git',
                     'http_proxy': amulet_http_proxy,
@@ -134,18 +138,20 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
                 }
             else:
                 if self._get_openstack_release() == self.trusty_icehouse:
+                    neutron_branch = release + '-eol'
                     reqs_repo = 'git://github.com/coreycb/requirements'
                 else:
+                    neutron_branch = 'stable/' + release
                     reqs_repo = 'git://github.com/openstack/requirements'
                 neutron_repo = 'git://github.com/openstack/neutron'
                 openstack_origin_git = {
                     'repositories': [
                         {'name': 'requirements',
                          'repository': reqs_repo,
-                         'branch': branch},
+                         'branch': neutron_branch},
                         {'name': 'neutron',
                          'repository': neutron_repo,
-                         'branch': branch},
+                         'branch': neutron_branch},
                     ],
                     'directory': '/mnt/openstack-git',
                     'http_proxy': amulet_http_proxy,

From 1a6676aee0d815556661fb3537a894e0c486c3a9 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Mon, 13 Jul 2015 13:05:49 +0000
Subject: [PATCH 66/88] More amulet deploy from source branch fixups.

---
 tests/basic_deployment.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index 0fa0457a..c278c28a 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -137,13 +137,13 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
                     'https_proxy': amulet_http_proxy,
                 }
             else:
+                neutron_branch = 'stable/' + release
+                neutron_repo = 'git://github.com/openstack/neutron'
+                reqs_repo = 'git://github.com/openstack/requirements'
                 if self._get_openstack_release() == self.trusty_icehouse:
                     neutron_branch = release + '-eol'
                     reqs_repo = 'git://github.com/coreycb/requirements'
-                else:
-                    neutron_branch = 'stable/' + release
-                    reqs_repo = 'git://github.com/openstack/requirements'
-                neutron_repo = 'git://github.com/openstack/neutron'
+
                 openstack_origin_git = {
                     'repositories': [
                         {'name': 'requirements',

From a47abcd168520b7ccc24b0067edfe3febff20d5c Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Mon, 13 Jul 2015 13:32:07 +0000
Subject: [PATCH 67/88] More amulet deploy from source branch fixups.

---
 tests/basic_deployment.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index c278c28a..2643c790 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -137,12 +137,13 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
                     'https_proxy': amulet_http_proxy,
                 }
             else:
-                neutron_branch = 'stable/' + release
-                neutron_repo = 'git://github.com/openstack/neutron'
                 reqs_repo = 'git://github.com/openstack/requirements'
+                neutron_repo = 'git://github.com/openstack/neutron'
+                neutron_branch = 'stable/' + release
                 if self._get_openstack_release() == self.trusty_icehouse:
-                    neutron_branch = release + '-eol'
                     reqs_repo = 'git://github.com/coreycb/requirements'
+                    neutron_repo = 'git://github.com/coreycb/neutron'
+                    neutron_branch = release + '-eol'
 
                 openstack_origin_git = {
                     'repositories': [

From 6300c17b36f7ae29696192fbf2cb3d5e9c03f603 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Mon, 13 Jul 2015 13:40:29 +0000
Subject: [PATCH 68/88] More amulet deploy from source branch fixups.

---
 tests/basic_deployment.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index 2643c790..4341dd3e 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -143,7 +143,6 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
                 if self._get_openstack_release() == self.trusty_icehouse:
                     reqs_repo = 'git://github.com/coreycb/requirements'
                     neutron_repo = 'git://github.com/coreycb/neutron'
-                    neutron_branch = release + '-eol'
 
                 openstack_origin_git = {
                     'repositories': [

From f3e6527c56131ba278a06b6a021e51ac7244e466 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Mon, 13 Jul 2015 13:49:06 +0000
Subject: [PATCH 69/88] More amulet deploy from source branch fixups.

---
 tests/basic_deployment.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index 4341dd3e..432613db 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -111,9 +111,9 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
 
             release = self._get_openstack_release_string()
             reqs_branch = 'stable/' + release
+            neutron_branch = 'stable/' + release
 
             if self._get_openstack_release() >= self.trusty_kilo:
-                neutron_branch = 'stable/' + release
                 openstack_origin_git = {
                     'repositories': [
                         {'name': 'requirements',
@@ -139,7 +139,6 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
             else:
                 reqs_repo = 'git://github.com/openstack/requirements'
                 neutron_repo = 'git://github.com/openstack/neutron'
-                neutron_branch = 'stable/' + release
                 if self._get_openstack_release() == self.trusty_icehouse:
                     reqs_repo = 'git://github.com/coreycb/requirements'
                     neutron_repo = 'git://github.com/coreycb/neutron'

From 93b34d7a88770e8343537809ece58d900734b5f3 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Mon, 13 Jul 2015 13:54:22 +0000
Subject: [PATCH 70/88] More amulet deploy from source branch fixups.

---
 tests/basic_deployment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index 432613db..0ddde893 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -147,7 +147,7 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
                     'repositories': [
                         {'name': 'requirements',
                          'repository': reqs_repo,
-                         'branch': neutron_branch},
+                         'branch': reqs_branch},
                         {'name': 'neutron',
                          'repository': neutron_repo,
                          'branch': neutron_branch},

From d1c25b7b0131d407f77b2c744ed20fe809d7c393 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Mon, 13 Jul 2015 16:07:29 +0000
Subject: [PATCH 71/88] More amulet deploy from source branch fixups.

---
 tests/basic_deployment.py | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index 0ddde893..dced4d4c 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -109,28 +109,26 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
         if self.git:
             amulet_http_proxy = os.environ.get('AMULET_HTTP_PROXY')
 
-            release = self._get_openstack_release_string()
-            reqs_branch = 'stable/' + release
-            neutron_branch = 'stable/' + release
+            branch = 'stable/' + self._get_openstack_release_string()
 
             if self._get_openstack_release() >= self.trusty_kilo:
                 openstack_origin_git = {
                     'repositories': [
                         {'name': 'requirements',
                          'repository': 'git://github.com/openstack/requirements',
-                         'branch': reqs_branch},
+                         'branch': branch},
                         {'name': 'neutron-fwaas',
                          'repository': 'git://github.com/openstack/neutron-fwaas',
-                         'branch': neutron_branch},
+                         'branch': branch},
                         {'name': 'neutron-lbaas',
                          'repository': 'git://github.com/openstack/neutron-lbaas',
-                         'branch': neutron_branch},
+                         'branch': branch},
                         {'name': 'neutron-vpnaas',
                          'repository': 'git://github.com/openstack/neutron-vpnaas',
-                         'branch': neutron_branch},
+                         'branch': branch},
                         {'name': 'neutron',
                          'repository': 'git://github.com/openstack/neutron',
-                         'branch': neutron_branch},
+                         'branch': branch},
                     ],
                     'directory': '/mnt/openstack-git',
                     'http_proxy': amulet_http_proxy,
@@ -147,10 +145,10 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
                     'repositories': [
                         {'name': 'requirements',
                          'repository': reqs_repo,
-                         'branch': reqs_branch},
+                         'branch': branch},
                         {'name': 'neutron',
                          'repository': neutron_repo,
-                         'branch': neutron_branch},
+                         'branch': branch},
                     ],
                     'directory': '/mnt/openstack-git',
                     'http_proxy': amulet_http_proxy,

From 5e2e16dee6bd93626320b20dbe5df13bf2112fe3 Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Mon, 13 Jul 2015 19:07:37 +0000
Subject: [PATCH 72/88] Fix lint error

---
 hooks/neutron_api_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py
index 9728a52a..89869bdd 100644
--- a/hooks/neutron_api_utils.py
+++ b/hooks/neutron_api_utils.py
@@ -81,7 +81,7 @@ BASE_GIT_PACKAGES = [
     'libxslt1-dev',
     'libyaml-dev',
     'python-dev',
-    'python-neutronclient', # required for get_neutron_client() import
+    'python-neutronclient',  # required for get_neutron_client() import
     'python-pip',
     'python-setuptools',
     'zlib1g-dev',

From 1ee42477dd02c338753e632775127688f953f930 Mon Sep 17 00:00:00 2001
From: James Page <james.page@ubuntu.com>
Date: Wed, 15 Jul 2015 10:53:12 +0100
Subject: [PATCH 73/88] Drop temporary PPA and pin, ensure subordinate changes
 are detected

---
 files/patched-icehouse     |  4 ----
 hooks/neutron_api_hooks.py | 10 +++-------
 2 files changed, 3 insertions(+), 11 deletions(-)
 delete mode 100644 files/patched-icehouse

diff --git a/files/patched-icehouse b/files/patched-icehouse
deleted file mode 100644
index c7bea34c..00000000
--- a/files/patched-icehouse
+++ /dev/null
@@ -1,4 +0,0 @@
-[ /etc/apt/preferences.d/patched-icehouse ]
-Package: *
-Pin: release o=LP-PPA-sdn-charmers-cisco-vpp-testing
-Pin-Priority: 990
diff --git a/hooks/neutron_api_hooks.py b/hooks/neutron_api_hooks.py
index 919da5d1..b79571e1 100755
--- a/hooks/neutron_api_hooks.py
+++ b/hooks/neutron_api_hooks.py
@@ -9,7 +9,6 @@ from subprocess import (
 from charmhelpers.core.hookenv import (
     Hooks,
     UnregisteredHookError,
-    charm_dir,
     config,
     is_relation_made,
     local_unit,
@@ -144,11 +143,7 @@ def configure_https():
 def install():
     execd_preinstall()
     configure_installation_source(config('openstack-origin'))
-    # XXX Remove me when patched nova and neutron are in the main ppa
-    configure_installation_source('ppa:sdn-charmers/cisco-vpp-testing')
-    apt_pin_file = charm_dir() + '/files/patched-icehouse'
-    import shutil
-    shutil.copyfile(apt_pin_file, '/etc/apt/preferences.d/patched-icehouse')
+
     apt_update()
     apt_install(determine_packages(config('openstack-origin')),
                 fatal=True)
@@ -484,7 +479,8 @@ def zeromq_configuration_relation_joined(relid=None):
                  users="neutron")
 
 
-@hooks.hook('zeromq-configuration-relation-changed')
+@hooks.hook('zeromq-configuration-relation-changed',
+            'neutron-plugin-api-subordinate-relation-changed')
 @restart_on_change(restart_map(), stopstart=True)
 def zeromq_configuration_relation_changed():
     CONFIGS.write_all()

From c1d7e8119ff7d74481c977bd5aba076fad4a048d Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Thu, 16 Jul 2015 20:17:53 +0000
Subject: [PATCH 74/88] Sync charm-helpers.

---
 .../contrib/openstack/amulet/deployment.py    |  39 ++-
 .../contrib/openstack/amulet/utils.py         | 287 +++++++++++++++---
 .../charmhelpers/contrib/openstack/context.py |  15 +-
 .../contrib/openstack/templates/ceph.conf     |  12 +-
 hooks/charmhelpers/contrib/openstack/utils.py |  14 +-
 .../contrib/storage/linux/ceph.py             |  12 +-
 hooks/charmhelpers/core/hookenv.py            | 129 +++++---
 hooks/charmhelpers/core/host.py               |  36 ++-
 hooks/charmhelpers/core/services/base.py      |  21 +-
 hooks/charmhelpers/core/services/helpers.py   |   4 +-
 hooks/charmhelpers/fetch/__init__.py          |  23 +-
 hooks/charmhelpers/fetch/archiveurl.py        |   8 +-
 hooks/charmhelpers/fetch/giturl.py            |   2 +-
 tests/charmhelpers/contrib/amulet/utils.py    | 131 +++++++-
 .../contrib/openstack/amulet/deployment.py    |  39 ++-
 .../contrib/openstack/amulet/utils.py         | 287 +++++++++++++++---
 16 files changed, 869 insertions(+), 190 deletions(-)

diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
index c664c9d0..b01e6cb8 100644
--- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -79,9 +79,9 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         services.append(this_service)
         use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
                       'ceph-osd', 'ceph-radosgw']
-        # Openstack subordinate charms do not expose an origin option as that
-        # is controlled by the principle
-        ignore = ['neutron-openvswitch']
+        # Most OpenStack subordinate charms do not expose an origin option
+        # as that is controlled by the principle.
+        ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch']
 
         if self.openstack:
             for svc in services:
@@ -148,3 +148,36 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             return os_origin.split('%s-' % self.series)[1].split('/')[0]
         else:
             return releases[self.series]
+
+    def get_ceph_expected_pools(self, radosgw=False):
+        """Return a list of expected ceph pools in a ceph + cinder + glance
+        test scenario, based on OpenStack release and whether ceph radosgw
+        is flagged as present or not."""
+
+        if self._get_openstack_release() >= self.trusty_kilo:
+            # Kilo or later
+            pools = [
+                'rbd',
+                'cinder',
+                'glance'
+            ]
+        else:
+            # Juno or earlier
+            pools = [
+                'data',
+                'metadata',
+                'rbd',
+                'cinder',
+                'glance'
+            ]
+
+        if radosgw:
+            pools.extend([
+                '.rgw.root',
+                '.rgw.control',
+                '.rgw',
+                '.rgw.gc',
+                '.users.uid'
+            ])
+
+        return pools
diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py
index 576bf0b5..03f79277 100644
--- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py
@@ -14,16 +14,20 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import amulet
+import json
 import logging
 import os
 import six
 import time
 import urllib
 
+import cinderclient.v1.client as cinder_client
 import glanceclient.v1.client as glance_client
 import heatclient.v1.client as heat_client
 import keystoneclient.v2_0 as keystone_client
 import novaclient.v1_1.client as nova_client
+import swiftclient
 
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
@@ -171,6 +175,16 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
+    def authenticate_cinder_admin(self, keystone_sentry, username,
+                                  password, tenant):
+        """Authenticates admin user with cinder."""
+        # NOTE(beisner): cinder python client doesn't accept tokens.
+        service_ip = \
+            keystone_sentry.relation('shared-db',
+                                     'mysql:shared-db')['private-address']
+        ept = "http://{}:5000/v2.0".format(service_ip.strip().decode('utf-8'))
+        return cinder_client.Client(username, password, tenant, ept)
+
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
                                     tenant):
         """Authenticates admin user with the keystone admin endpoint."""
@@ -212,9 +226,29 @@ class OpenStackAmuletUtils(AmuletUtils):
         return nova_client.Client(username=user, api_key=password,
                                   project_id=tenant, auth_url=ep)
 
+    def authenticate_swift_user(self, keystone, user, password, tenant):
+        """Authenticates a regular user with swift api."""
+        self.log.debug('Authenticating swift user ({})...'.format(user))
+        ep = keystone.service_catalog.url_for(service_type='identity',
+                                              endpoint_type='publicURL')
+        return swiftclient.Connection(authurl=ep,
+                                      user=user,
+                                      key=password,
+                                      tenant_name=tenant,
+                                      auth_version='2.0')
+
     def create_cirros_image(self, glance, image_name):
-        """Download the latest cirros image and upload it to glance."""
-        self.log.debug('Creating glance image ({})...'.format(image_name))
+        """Download the latest cirros image and upload it to glance,
+        validate and return a resource pointer.
+
+        :param glance: pointer to authenticated glance connection
+        :param image_name: display name for new image
+        :returns: glance image pointer
+        """
+        self.log.debug('Creating glance cirros image '
+                       '({})...'.format(image_name))
+
+        # Download cirros image
         http_proxy = os.getenv('AMULET_HTTP_PROXY')
         self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
         if http_proxy:
@@ -223,33 +257,51 @@ class OpenStackAmuletUtils(AmuletUtils):
         else:
             opener = urllib.FancyURLopener()
 
-        f = opener.open("http://download.cirros-cloud.net/version/released")
+        f = opener.open('http://download.cirros-cloud.net/version/released')
         version = f.read().strip()
-        cirros_img = "cirros-{}-x86_64-disk.img".format(version)
+        cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
         local_path = os.path.join('tests', cirros_img)
 
         if not os.path.exists(local_path):
-            cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net",
+            cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
                                                   version, cirros_img)
             opener.retrieve(cirros_url, local_path)
         f.close()
 
+        # Create glance image
         with open(local_path) as f:
             image = glance.images.create(name=image_name, is_public=True,
                                          disk_format='qcow2',
                                          container_format='bare', data=f)
-        count = 1
-        status = image.status
-        while status != 'active' and count < 10:
-            time.sleep(3)
-            image = glance.images.get(image.id)
-            status = image.status
-            self.log.debug('image status: {}'.format(status))
-            count += 1
 
-        if status != 'active':
-            self.log.error('image creation timed out')
-            return None
+        # Wait for image to reach active status
+        img_id = image.id
+        ret = self.resource_reaches_status(glance.images, img_id,
+                                           expected_stat='active',
+                                           msg='Image status wait')
+        if not ret:
+            msg = 'Glance image failed to reach expected state.'
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Re-validate new image
+        self.log.debug('Validating image attributes...')
+        val_img_name = glance.images.get(img_id).name
+        val_img_stat = glance.images.get(img_id).status
+        val_img_pub = glance.images.get(img_id).is_public
+        val_img_cfmt = glance.images.get(img_id).container_format
+        val_img_dfmt = glance.images.get(img_id).disk_format
+        msg_attr = ('Image attributes - name:{} public:{} id:{} stat:{} '
+                    'container fmt:{} disk fmt:{}'.format(
+                        val_img_name, val_img_pub, img_id,
+                        val_img_stat, val_img_cfmt, val_img_dfmt))
+
+        if val_img_name == image_name and val_img_stat == 'active' \
+                and val_img_pub is True and val_img_cfmt == 'bare' \
+                and val_img_dfmt == 'qcow2':
+            self.log.debug(msg_attr)
+        else:
+            msg = ('Volume validation failed, {}'.format(msg_attr))
+            amulet.raise_status(amulet.FAIL, msg=msg)
 
         return image
 
@@ -260,22 +312,7 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.warn('/!\\ DEPRECATION WARNING:  use '
                       'delete_resource instead of delete_image.')
         self.log.debug('Deleting glance image ({})...'.format(image))
-        num_before = len(list(glance.images.list()))
-        glance.images.delete(image)
-
-        count = 1
-        num_after = len(list(glance.images.list()))
-        while num_after != (num_before - 1) and count < 10:
-            time.sleep(3)
-            num_after = len(list(glance.images.list()))
-            self.log.debug('number of images: {}'.format(num_after))
-            count += 1
-
-        if num_after != (num_before - 1):
-            self.log.error('image deletion timed out')
-            return False
-
-        return True
+        return self.delete_resource(glance.images, image, msg='glance image')
 
     def create_instance(self, nova, image_name, instance_name, flavor):
         """Create the specified instance."""
@@ -308,22 +345,8 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.warn('/!\\ DEPRECATION WARNING:  use '
                       'delete_resource instead of delete_instance.')
         self.log.debug('Deleting instance ({})...'.format(instance))
-        num_before = len(list(nova.servers.list()))
-        nova.servers.delete(instance)
-
-        count = 1
-        num_after = len(list(nova.servers.list()))
-        while num_after != (num_before - 1) and count < 10:
-            time.sleep(3)
-            num_after = len(list(nova.servers.list()))
-            self.log.debug('number of instances: {}'.format(num_after))
-            count += 1
-
-        if num_after != (num_before - 1):
-            self.log.error('instance deletion timed out')
-            return False
-
-        return True
+        return self.delete_resource(nova.servers, instance,
+                                    msg='nova instance')
 
     def create_or_get_keypair(self, nova, keypair_name="testkey"):
         """Create a new keypair, or return pointer if it already exists."""
@@ -339,6 +362,88 @@ class OpenStackAmuletUtils(AmuletUtils):
         _keypair = nova.keypairs.create(name=keypair_name)
         return _keypair
 
+    def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1,
+                             img_id=None, src_vol_id=None, snap_id=None):
+        """Create cinder volume, optionally from a glance image, OR
+        optionally as a clone of an existing volume, OR optionally
+        from a snapshot.  Wait for the new volume status to reach
+        the expected status, validate and return a resource pointer.
+
+        :param vol_name: cinder volume display name
+        :param vol_size: size in gigabytes
+        :param img_id: optional glance image id
+        :param src_vol_id: optional source volume id to clone
+        :param snap_id: optional snapshot id to use
+        :returns: cinder volume pointer
+        """
+        # Handle parameter input and avoid impossible combinations
+        if img_id and not src_vol_id and not snap_id:
+            # Create volume from image
+            self.log.debug('Creating cinder volume from glance image...')
+            bootable = 'true'
+        elif src_vol_id and not img_id and not snap_id:
+            # Clone an existing volume
+            self.log.debug('Cloning cinder volume...')
+            bootable = cinder.volumes.get(src_vol_id).bootable
+        elif snap_id and not src_vol_id and not img_id:
+            # Create volume from snapshot
+            self.log.debug('Creating cinder volume from snapshot...')
+            snap = cinder.volume_snapshots.find(id=snap_id)
+            vol_size = snap.size
+            snap_vol_id = cinder.volume_snapshots.get(snap_id).volume_id
+            bootable = cinder.volumes.get(snap_vol_id).bootable
+        elif not img_id and not src_vol_id and not snap_id:
+            # Create volume
+            self.log.debug('Creating cinder volume...')
+            bootable = 'false'
+        else:
+            # Impossible combination of parameters
+            msg = ('Invalid method use - name:{} size:{} img_id:{} '
+                   'src_vol_id:{} snap_id:{}'.format(vol_name, vol_size,
+                                                     img_id, src_vol_id,
+                                                     snap_id))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Create new volume
+        try:
+            vol_new = cinder.volumes.create(display_name=vol_name,
+                                            imageRef=img_id,
+                                            size=vol_size,
+                                            source_volid=src_vol_id,
+                                            snapshot_id=snap_id)
+            vol_id = vol_new.id
+        except Exception as e:
+            msg = 'Failed to create volume: {}'.format(e)
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Wait for volume to reach available status
+        ret = self.resource_reaches_status(cinder.volumes, vol_id,
+                                           expected_stat="available",
+                                           msg="Volume status wait")
+        if not ret:
+            msg = 'Cinder volume failed to reach expected state.'
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Re-validate new volume
+        self.log.debug('Validating volume attributes...')
+        val_vol_name = cinder.volumes.get(vol_id).display_name
+        val_vol_boot = cinder.volumes.get(vol_id).bootable
+        val_vol_stat = cinder.volumes.get(vol_id).status
+        val_vol_size = cinder.volumes.get(vol_id).size
+        msg_attr = ('Volume attributes - name:{} id:{} stat:{} boot:'
+                    '{} size:{}'.format(val_vol_name, vol_id,
+                                        val_vol_stat, val_vol_boot,
+                                        val_vol_size))
+
+        if val_vol_boot == bootable and val_vol_stat == 'available' \
+                and val_vol_name == vol_name and val_vol_size == vol_size:
+            self.log.debug(msg_attr)
+        else:
+            msg = ('Volume validation failed, {}'.format(msg_attr))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        return vol_new
+
     def delete_resource(self, resource, resource_id,
                         msg="resource", max_wait=120):
         """Delete one openstack resource, such as one instance, keypair,
@@ -350,6 +455,8 @@ class OpenStackAmuletUtils(AmuletUtils):
         :param max_wait: maximum wait time in seconds
         :returns: True if successful, otherwise False
         """
+        self.log.debug('Deleting OpenStack resource '
+                       '{} ({})'.format(resource_id, msg))
         num_before = len(list(resource.list()))
         resource.delete(resource_id)
 
@@ -411,3 +518,87 @@ class OpenStackAmuletUtils(AmuletUtils):
             self.log.debug('{} never reached expected status: '
                            '{}'.format(resource_id, expected_stat))
             return False
+
+    def get_ceph_osd_id_cmd(self, index):
+        """Produce a shell command that will return a ceph-osd id."""
+        return ("`initctl list | grep 'ceph-osd ' | "
+                "awk 'NR=={} {{ print $2 }}' | "
+                "grep -o '[0-9]*'`".format(index + 1))
+
+    def get_ceph_pools(self, sentry_unit):
+        """Return a dict of ceph pools from a single ceph unit, with
+        pool name as keys, pool id as vals."""
+        pools = {}
+        cmd = 'sudo ceph osd lspools'
+        output, code = sentry_unit.run(cmd)
+        if code != 0:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance,
+        for pool in str(output).split(','):
+            pool_id_name = pool.split(' ')
+            if len(pool_id_name) == 2:
+                pool_id = pool_id_name[0]
+                pool_name = pool_id_name[1]
+                pools[pool_name] = int(pool_id)
+
+        self.log.debug('Pools on {}: {}'.format(sentry_unit.info['unit_name'],
+                                                pools))
+        return pools
+
+    def get_ceph_df(self, sentry_unit):
+        """Return dict of ceph df json output, including ceph pool state.
+
+        :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+        :returns: Dict of ceph df output
+        """
+        cmd = 'sudo ceph df --format=json'
+        output, code = sentry_unit.run(cmd)
+        if code != 0:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+        return json.loads(output)
+
+    def get_ceph_pool_sample(self, sentry_unit, pool_id=0):
+        """Take a sample of attributes of a ceph pool, returning ceph
+        pool name, object count and disk space used for the specified
+        pool ID number.
+
+        :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+        :param pool_id: Ceph pool ID
+        :returns: List of pool name, object count, kb disk space used
+        """
+        df = self.get_ceph_df(sentry_unit)
+        pool_name = df['pools'][pool_id]['name']
+        obj_count = df['pools'][pool_id]['stats']['objects']
+        kb_used = df['pools'][pool_id]['stats']['kb_used']
+        self.log.debug('Ceph {} pool (ID {}): {} objects, '
+                       '{} kb used'.format(pool_name, pool_id,
+                                           obj_count, kb_used))
+        return pool_name, obj_count, kb_used
+
+    def validate_ceph_pool_samples(self, samples, sample_type="resource pool"):
+        """Validate ceph pool samples taken over time, such as pool
+        object counts or pool kb used, before adding, after adding, and
+        after deleting items which affect those pool attributes.  The
+        2nd element is expected to be greater than the 1st; 3rd is expected
+        to be less than the 2nd.
+
+        :param samples: List containing 3 data samples
+        :param sample_type: String for logging and usage context
+        :returns: None if successful, Failure message otherwise
+        """
+        original, created, deleted = range(3)
+        if samples[created] <= samples[original] or \
+                samples[deleted] >= samples[created]:
+            return ('Ceph {} samples ({}) '
+                    'unexpected.'.format(sample_type, samples))
+        else:
+            self.log.debug('Ceph {} samples (OK): '
+                           '{}'.format(sample_type, samples))
+            return None
diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py
index ab400060..8f3f1b15 100644
--- a/hooks/charmhelpers/contrib/openstack/context.py
+++ b/hooks/charmhelpers/contrib/openstack/context.py
@@ -122,21 +122,24 @@ def config_flags_parser(config_flags):
          of specifying multiple key value pairs within the same string. For
          example, a string in the format of 'key1=value1, key2=value2' will
          return a dict of:
-         {'key1': 'value1',
-          'key2': 'value2'}.
+
+             {'key1': 'value1',
+              'key2': 'value2'}.
 
       2. A string in the above format, but supporting a comma-delimited list
          of values for the same key. For example, a string in the format of
          'key1=value1, key2=value3,value4,value5' will return a dict of:
-         {'key1', 'value1',
-          'key2', 'value2,value3,value4'}
+
+             {'key1', 'value1',
+              'key2', 'value2,value3,value4'}
 
       3. A string containing a colon character (:) prior to an equal
          character (=) will be treated as yaml and parsed as such. This can be
          used to specify more complex key value pairs. For example,
          a string in the format of 'key1: subkey1=value1, subkey2=value2' will
          return a dict of:
-         {'key1', 'subkey1=value1, subkey2=value2'}
+
+             {'key1', 'subkey1=value1, subkey2=value2'}
 
     The provided config_flags string may be a list of comma-separated values
     which themselves may be comma-separated list of values.
@@ -891,8 +894,6 @@ class NeutronContext(OSContextGenerator):
         return ctxt
 
     def __call__(self):
-        self._ensure_packages()
-
         if self.network_manager not in ['quantum', 'neutron']:
             return {}
 
diff --git a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf
index 81a9719f..b99851cc 100644
--- a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf
+++ b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf
@@ -5,11 +5,11 @@
 ###############################################################################
 [global]
 {% if auth -%}
- auth_supported = {{ auth }}
- keyring = /etc/ceph/$cluster.$name.keyring
- mon host = {{ mon_hosts }}
+auth_supported = {{ auth }}
+keyring = /etc/ceph/$cluster.$name.keyring
+mon host = {{ mon_hosts }}
 {% endif -%}
- log to syslog = {{ use_syslog }}
- err to syslog = {{ use_syslog }}
- clog to syslog = {{ use_syslog }}
+log to syslog = {{ use_syslog }}
+err to syslog = {{ use_syslog }}
+clog to syslog = {{ use_syslog }}
 
diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index 28532c98..4dd000c3 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -522,6 +522,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1):
     Clone/install all specified OpenStack repositories.
 
     The expected format of projects_yaml is:
+
         repositories:
           - {name: keystone,
              repository: 'git://git.openstack.org/openstack/keystone.git',
@@ -529,11 +530,13 @@ def git_clone_and_install(projects_yaml, core_project, depth=1):
           - {name: requirements,
              repository: 'git://git.openstack.org/openstack/requirements.git',
              branch: 'stable/icehouse'}
+
         directory: /mnt/openstack-git
         http_proxy: squid-proxy-url
         https_proxy: squid-proxy-url
 
-        The directory, http_proxy, and https_proxy keys are optional.
+    The directory, http_proxy, and https_proxy keys are optional.
+
     """
     global requirements_dir
     parent_dir = '/mnt/openstack-git'
@@ -555,10 +558,11 @@ def git_clone_and_install(projects_yaml, core_project, depth=1):
 
     pip_create_virtualenv(os.path.join(parent_dir, 'venv'))
 
-    # Upgrade setuptools from default virtualenv version. The default version
-    # in trusty breaks update.py in global requirements master branch.
-    pip_install('setuptools', upgrade=True, proxy=http_proxy,
-                venv=os.path.join(parent_dir, 'venv'))
+    # Upgrade setuptools and pip from default virtualenv versions. The default
+    # versions in trusty break master OpenStack branch deployments.
+    for p in ['pip', 'setuptools']:
+        pip_install(p, upgrade=True, proxy=http_proxy,
+                    venv=os.path.join(parent_dir, 'venv'))
 
     for p in projects['repositories']:
         repo = p['repository']
diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py
index 31ea7f9e..00dbffb4 100644
--- a/hooks/charmhelpers/contrib/storage/linux/ceph.py
+++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py
@@ -60,12 +60,12 @@ KEYRING = '/etc/ceph/ceph.client.{}.keyring'
 KEYFILE = '/etc/ceph/ceph.client.{}.key'
 
 CEPH_CONF = """[global]
- auth supported = {auth}
- keyring = {keyring}
- mon host = {mon_hosts}
- log to syslog = {use_syslog}
- err to syslog = {use_syslog}
- clog to syslog = {use_syslog}
+auth supported = {auth}
+keyring = {keyring}
+mon host = {mon_hosts}
+log to syslog = {use_syslog}
+err to syslog = {use_syslog}
+clog to syslog = {use_syslog}
 """
 
 
diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py
index 117429fd..dd8def9a 100644
--- a/hooks/charmhelpers/core/hookenv.py
+++ b/hooks/charmhelpers/core/hookenv.py
@@ -21,7 +21,9 @@
 #  Charm Helpers Developers <juju@lists.ubuntu.com>
 
 from __future__ import print_function
+from distutils.version import LooseVersion
 from functools import wraps
+import glob
 import os
 import json
 import yaml
@@ -242,29 +244,7 @@ class Config(dict):
         self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
         if os.path.exists(self.path):
             self.load_previous()
-
-    def __getitem__(self, key):
-        """For regular dict lookups, check the current juju config first,
-        then the previous (saved) copy. This ensures that user-saved values
-        will be returned by a dict lookup.
-
-        """
-        try:
-            return dict.__getitem__(self, key)
-        except KeyError:
-            return (self._prev_dict or {})[key]
-
-    def get(self, key, default=None):
-        try:
-            return self[key]
-        except KeyError:
-            return default
-
-    def keys(self):
-        prev_keys = []
-        if self._prev_dict is not None:
-            prev_keys = self._prev_dict.keys()
-        return list(set(prev_keys + list(dict.keys(self))))
+        atexit(self._implicit_save)
 
     def load_previous(self, path=None):
         """Load previous copy of config from disk.
@@ -283,6 +263,9 @@ class Config(dict):
         self.path = path or self.path
         with open(self.path) as f:
             self._prev_dict = json.load(f)
+        for k, v in self._prev_dict.items():
+            if k not in self:
+                self[k] = v
 
     def changed(self, key):
         """Return True if the current value for this key is different from
@@ -314,13 +297,13 @@ class Config(dict):
         instance.
 
         """
-        if self._prev_dict:
-            for k, v in six.iteritems(self._prev_dict):
-                if k not in self:
-                    self[k] = v
         with open(self.path, 'w') as f:
             json.dump(self, f)
 
+    def _implicit_save(self):
+        if self.implicit_save:
+            self.save()
+
 
 @cached
 def config(scope=None):
@@ -587,10 +570,14 @@ class Hooks(object):
             hooks.execute(sys.argv)
     """
 
-    def __init__(self, config_save=True):
+    def __init__(self, config_save=None):
         super(Hooks, self).__init__()
         self._hooks = {}
-        self._config_save = config_save
+
+        # For unknown reasons, we allow the Hooks constructor to override
+        # config().implicit_save.
+        if config_save is not None:
+            config().implicit_save = config_save
 
     def register(self, name, function):
         """Register a hook"""
@@ -598,13 +585,16 @@ class Hooks(object):
 
     def execute(self, args):
         """Execute a registered hook based on args[0]"""
+        _run_atstart()
         hook_name = os.path.basename(args[0])
         if hook_name in self._hooks:
-            self._hooks[hook_name]()
-            if self._config_save:
-                cfg = config()
-                if cfg.implicit_save:
-                    cfg.save()
+            try:
+                self._hooks[hook_name]()
+            except SystemExit as x:
+                if x.code is None or x.code == 0:
+                    _run_atexit()
+                raise
+            _run_atexit()
         else:
             raise UnregisteredHookError(hook_name)
 
@@ -732,13 +722,80 @@ def leader_get(attribute=None):
 @translate_exc(from_exc=OSError, to_exc=NotImplementedError)
 def leader_set(settings=None, **kwargs):
     """Juju leader set value(s)"""
-    log("Juju leader-set '%s'" % (settings), level=DEBUG)
+    # Don't log secrets.
+    # log("Juju leader-set '%s'" % (settings), level=DEBUG)
     cmd = ['leader-set']
     settings = settings or {}
     settings.update(kwargs)
-    for k, v in settings.iteritems():
+    for k, v in settings.items():
         if v is None:
             cmd.append('{}='.format(k))
         else:
             cmd.append('{}={}'.format(k, v))
     subprocess.check_call(cmd)
+
+
+@cached
+def juju_version():
+    """Full version string (eg. '1.23.3.1-trusty-amd64')"""
+    # Per https://bugs.launchpad.net/juju-core/+bug/1455368/comments/1
+    jujud = glob.glob('/var/lib/juju/tools/machine-*/jujud')[0]
+    return subprocess.check_output([jujud, 'version'],
+                                   universal_newlines=True).strip()
+
+
+@cached
+def has_juju_version(minimum_version):
+    """Return True if the Juju version is at least the provided version"""
+    return LooseVersion(juju_version()) >= LooseVersion(minimum_version)
+
+
+_atexit = []
+_atstart = []
+
+
+def atstart(callback, *args, **kwargs):
+    '''Schedule a callback to run before the main hook.
+
+    Callbacks are run in the order they were added.
+
+    This is useful for modules and classes to perform initialization
+    and inject behavior. In particular:
+
+        - Run common code before all of your hooks, such as logging
+          the hook name or interesting relation data.
+        - Defer object or module initialization that requires a hook
+          context until we know there actually is a hook context,
+          making testing easier.
+        - Rather than requiring charm authors to include boilerplate to
+          invoke your helper's behavior, have it run automatically if
+          your object is instantiated or module imported.
+
+    This is not at all useful after your hook framework as been launched.
+    '''
+    global _atstart
+    _atstart.append((callback, args, kwargs))
+
+
+def atexit(callback, *args, **kwargs):
+    '''Schedule a callback to run on successful hook completion.
+
+    Callbacks are run in the reverse order that they were added.'''
+    _atexit.append((callback, args, kwargs))
+
+
+def _run_atstart():
+    '''Hook frameworks must invoke this before running the main hook body.'''
+    global _atstart
+    for callback, args, kwargs in _atstart:
+        callback(*args, **kwargs)
+    del _atstart[:]
+
+
+def _run_atexit():
+    '''Hook frameworks must invoke this after the main hook body has
+    successfully completed. Do not invoke it if the hook fails.'''
+    global _atexit
+    for callback, args, kwargs in reversed(_atexit):
+        callback(*args, **kwargs)
+    del _atexit[:]
diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py
index 901a4cfe..8ae8ef86 100644
--- a/hooks/charmhelpers/core/host.py
+++ b/hooks/charmhelpers/core/host.py
@@ -63,6 +63,36 @@ def service_reload(service_name, restart_on_failure=False):
     return service_result
 
 
+def service_pause(service_name, init_dir=None):
+    """Pause a system service.
+
+    Stop it, and prevent it from starting again at boot."""
+    if init_dir is None:
+        init_dir = "/etc/init"
+    stopped = service_stop(service_name)
+    # XXX: Support systemd too
+    override_path = os.path.join(
+        init_dir, '{}.conf.override'.format(service_name))
+    with open(override_path, 'w') as fh:
+        fh.write("manual\n")
+    return stopped
+
+
+def service_resume(service_name, init_dir=None):
+    """Resume a system service.
+
+    Reenable starting again at boot. Start the service"""
+    # XXX: Support systemd too
+    if init_dir is None:
+        init_dir = "/etc/init"
+    override_path = os.path.join(
+        init_dir, '{}.conf.override'.format(service_name))
+    if os.path.exists(override_path):
+        os.unlink(override_path)
+    started = service_start(service_name)
+    return started
+
+
 def service(action, service_name):
     """Control a system service"""
     cmd = ['service', service_name, action]
@@ -140,11 +170,7 @@ def add_group(group_name, system_group=False):
 
 def add_user_to_group(username, group):
     """Add a user to a group"""
-    cmd = [
-        'gpasswd', '-a',
-        username,
-        group
-    ]
+    cmd = ['gpasswd', '-a', username, group]
     log("Adding user {} to group {}".format(username, group))
     subprocess.check_call(cmd)
 
diff --git a/hooks/charmhelpers/core/services/base.py b/hooks/charmhelpers/core/services/base.py
index 98d344e1..a42660ca 100644
--- a/hooks/charmhelpers/core/services/base.py
+++ b/hooks/charmhelpers/core/services/base.py
@@ -128,15 +128,18 @@ class ServiceManager(object):
         """
         Handle the current hook by doing The Right Thing with the registered services.
         """
-        hook_name = hookenv.hook_name()
-        if hook_name == 'stop':
-            self.stop_services()
-        else:
-            self.reconfigure_services()
-            self.provide_data()
-        cfg = hookenv.config()
-        if cfg.implicit_save:
-            cfg.save()
+        hookenv._run_atstart()
+        try:
+            hook_name = hookenv.hook_name()
+            if hook_name == 'stop':
+                self.stop_services()
+            else:
+                self.reconfigure_services()
+                self.provide_data()
+        except SystemExit as x:
+            if x.code is None or x.code == 0:
+                hookenv._run_atexit()
+        hookenv._run_atexit()
 
     def provide_data(self):
         """
diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py
index 3eb5fb44..8005c415 100644
--- a/hooks/charmhelpers/core/services/helpers.py
+++ b/hooks/charmhelpers/core/services/helpers.py
@@ -239,12 +239,12 @@ class TemplateCallback(ManagerCallback):
     action.
 
     :param str source: The template source file, relative to
-    `$CHARM_DIR/templates`
-
+        `$CHARM_DIR/templates`
     :param str target: The target to write the rendered template to
     :param str owner: The owner of the rendered file
     :param str group: The group of the rendered file
     :param int perms: The permissions of the rendered file
+
     """
     def __init__(self, source, target,
                  owner='root', group='root', perms=0o444):
diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py
index 9a1a2515..0a3bb969 100644
--- a/hooks/charmhelpers/fetch/__init__.py
+++ b/hooks/charmhelpers/fetch/__init__.py
@@ -215,9 +215,9 @@ def apt_purge(packages, fatal=False):
     _run_apt_command(cmd, fatal)
 
 
-def apt_hold(packages, fatal=False):
-    """Hold one or more packages"""
-    cmd = ['apt-mark', 'hold']
+def apt_mark(packages, mark, fatal=False):
+    """Flag one or more packages using apt-mark"""
+    cmd = ['apt-mark', mark]
     if isinstance(packages, six.string_types):
         cmd.append(packages)
     else:
@@ -225,9 +225,17 @@ def apt_hold(packages, fatal=False):
     log("Holding {}".format(packages))
 
     if fatal:
-        subprocess.check_call(cmd)
+        subprocess.check_call(cmd, universal_newlines=True)
     else:
-        subprocess.call(cmd)
+        subprocess.call(cmd, universal_newlines=True)
+
+
+def apt_hold(packages, fatal=False):
+    return apt_mark(packages, 'hold', fatal=fatal)
+
+
+def apt_unhold(packages, fatal=False):
+    return apt_mark(packages, 'unhold', fatal=fatal)
 
 
 def add_source(source, key=None):
@@ -370,8 +378,9 @@ def install_remote(source, *args, **kwargs):
     for handler in handlers:
         try:
             installed_to = handler.install(source, *args, **kwargs)
-        except UnhandledSource:
-            pass
+        except UnhandledSource as e:
+            log('Install source attempt unsuccessful: {}'.format(e),
+                level='WARNING')
     if not installed_to:
         raise UnhandledSource("No handler found for source {}".format(source))
     return installed_to
diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py
index 8dfce505..efd7f9f0 100644
--- a/hooks/charmhelpers/fetch/archiveurl.py
+++ b/hooks/charmhelpers/fetch/archiveurl.py
@@ -77,6 +77,8 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
     def can_handle(self, source):
         url_parts = self.parse_url(source)
         if url_parts.scheme not in ('http', 'https', 'ftp', 'file'):
+            # XXX: Why is this returning a boolean and a string? It's
+            # doomed to fail since "bool(can_handle('foo://'))"  will be True.
             return "Wrong source type"
         if get_archive_handler(self.base_url(source)):
             return True
@@ -155,7 +157,11 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
             else:
                 algorithms = hashlib.algorithms_available
             if key in algorithms:
-                check_hash(dld_file, value, key)
+                if len(value) != 1:
+                    raise TypeError(
+                        "Expected 1 hash value, not %d" % len(value))
+                expected = value[0]
+                check_hash(dld_file, expected, key)
         if checksum:
             check_hash(dld_file, checksum, hash_type)
         return extract(dld_file, dest)
diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py
index ddc25b7e..f023b26d 100644
--- a/hooks/charmhelpers/fetch/giturl.py
+++ b/hooks/charmhelpers/fetch/giturl.py
@@ -67,7 +67,7 @@ class GitUrlFetchHandler(BaseFetchHandler):
         try:
             self.clone(source, dest_dir, branch, depth)
         except GitCommandError as e:
-            raise UnhandledSource(e.message)
+            raise UnhandledSource(e)
         except OSError as e:
             raise UnhandledSource(e.strerror)
         return dest_dir
diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py
index e8c4a274..3de26afd 100644
--- a/tests/charmhelpers/contrib/amulet/utils.py
+++ b/tests/charmhelpers/contrib/amulet/utils.py
@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import amulet
 import ConfigParser
 import distro_info
 import io
@@ -173,6 +174,11 @@ class AmuletUtils(object):
 
            Verify that the specified section of the config file contains
            the expected option key:value pairs.
+
+           Compare expected dictionary data vs actual dictionary data.
+           The values in the 'expected' dictionary can be strings, bools, ints,
+           longs, or can be a function that evaluates a variable and returns a
+           bool.
            """
         self.log.debug('Validating config file data ({} in {} on {})'
                        '...'.format(section, config_file,
@@ -185,9 +191,20 @@ class AmuletUtils(object):
         for k in expected.keys():
             if not config.has_option(section, k):
                 return "section [{}] is missing option {}".format(section, k)
-            if config.get(section, k) != expected[k]:
+
+            actual = config.get(section, k)
+            v = expected[k]
+            if (isinstance(v, six.string_types) or
+                    isinstance(v, bool) or
+                    isinstance(v, six.integer_types)):
+                # handle explicit values
+                if actual != v:
+                    return "section [{}] {}:{} != expected {}:{}".format(
+                           section, k, actual, k, expected[k])
+            # handle function pointers, such as not_null or valid_ip
+            elif not v(actual):
                 return "section [{}] {}:{} != expected {}:{}".format(
-                       section, k, config.get(section, k), k, expected[k])
+                       section, k, actual, k, expected[k])
         return None
 
     def _validate_dict_data(self, expected, actual):
@@ -195,7 +212,7 @@ class AmuletUtils(object):
 
            Compare expected dictionary data vs actual dictionary data.
            The values in the 'expected' dictionary can be strings, bools, ints,
-           longs, or can be a function that evaluate a variable and returns a
+           longs, or can be a function that evaluates a variable and returns a
            bool.
            """
         self.log.debug('actual: {}'.format(repr(actual)))
@@ -206,8 +223,10 @@ class AmuletUtils(object):
                 if (isinstance(v, six.string_types) or
                         isinstance(v, bool) or
                         isinstance(v, six.integer_types)):
+                    # handle explicit values
                     if v != actual[k]:
                         return "{}:{}".format(k, actual[k])
+                # handle function pointers, such as not_null or valid_ip
                 elif not v(actual[k]):
                     return "{}:{}".format(k, actual[k])
             else:
@@ -406,3 +425,109 @@ class AmuletUtils(object):
         """Convert a relative file path to a file URL."""
         _abs_path = os.path.abspath(file_rel_path)
         return urlparse.urlparse(_abs_path, scheme='file').geturl()
+
+    def check_commands_on_units(self, commands, sentry_units):
+        """Check that all commands in a list exit zero on all
+        sentry units in a list.
+
+        :param commands:  list of bash commands
+        :param sentry_units:  list of sentry unit pointers
+        :returns: None if successful; Failure message otherwise
+        """
+        self.log.debug('Checking exit codes for {} commands on {} '
+                       'sentry units...'.format(len(commands),
+                                                len(sentry_units)))
+        for sentry_unit in sentry_units:
+            for cmd in commands:
+                output, code = sentry_unit.run(cmd)
+                if code == 0:
+                    self.log.debug('{} `{}` returned {} '
+                                   '(OK)'.format(sentry_unit.info['unit_name'],
+                                                 cmd, code))
+                else:
+                    return ('{} `{}` returned {} '
+                            '{}'.format(sentry_unit.info['unit_name'],
+                                        cmd, code, output))
+        return None
+
+    def get_process_id_list(self, sentry_unit, process_name):
+        """Get a list of process ID(s) from a single sentry juju unit
+        for a single process name.
+
+        :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+        :param process_name: Process name
+        :returns: List of process IDs
+        """
+        cmd = 'pidof {}'.format(process_name)
+        output, code = sentry_unit.run(cmd)
+        if code != 0:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+        return str(output).split()
+
+    def get_unit_process_ids(self, unit_processes):
+        """Construct a dict containing unit sentries, process names, and
+        process IDs."""
+        pid_dict = {}
+        for sentry_unit, process_list in unit_processes.iteritems():
+            pid_dict[sentry_unit] = {}
+            for process in process_list:
+                pids = self.get_process_id_list(sentry_unit, process)
+                pid_dict[sentry_unit].update({process: pids})
+        return pid_dict
+
+    def validate_unit_process_ids(self, expected, actual):
+        """Validate process id quantities for services on units."""
+        self.log.debug('Checking units for running processes...')
+        self.log.debug('Expected PIDs: {}'.format(expected))
+        self.log.debug('Actual PIDs: {}'.format(actual))
+
+        if len(actual) != len(expected):
+            return ('Unit count mismatch.  expected, actual: {}, '
+                    '{} '.format(len(expected), len(actual)))
+
+        for (e_sentry, e_proc_names) in expected.iteritems():
+            e_sentry_name = e_sentry.info['unit_name']
+            if e_sentry in actual.keys():
+                a_proc_names = actual[e_sentry]
+            else:
+                return ('Expected sentry ({}) not found in actual dict data.'
+                        '{}'.format(e_sentry_name, e_sentry))
+
+            if len(e_proc_names.keys()) != len(a_proc_names.keys()):
+                return ('Process name count mismatch.  expected, actual: {}, '
+                        '{}'.format(len(expected), len(actual)))
+
+            for (e_proc_name, e_pids_length), (a_proc_name, a_pids) in \
+                    zip(e_proc_names.items(), a_proc_names.items()):
+                if e_proc_name != a_proc_name:
+                    return ('Process name mismatch.  expected, actual: {}, '
+                            '{}'.format(e_proc_name, a_proc_name))
+
+                a_pids_length = len(a_pids)
+                if e_pids_length != a_pids_length:
+                    return ('PID count mismatch. {} ({}) expected, actual: '
+                            '{}, {} ({})'.format(e_sentry_name, e_proc_name,
+                                                 e_pids_length, a_pids_length,
+                                                 a_pids))
+                else:
+                    self.log.debug('PID check OK: {} {} {}: '
+                                   '{}'.format(e_sentry_name, e_proc_name,
+                                               e_pids_length, a_pids))
+        return None
+
+    def validate_list_of_identical_dicts(self, list_of_dicts):
+        """Check that all dicts within a list are identical."""
+        hashes = []
+        for _dict in list_of_dicts:
+            hashes.append(hash(frozenset(_dict.items())))
+
+        self.log.debug('Hashes: {}'.format(hashes))
+        if len(set(hashes)) == 1:
+            self.log.debug('Dicts within list are identical')
+        else:
+            return 'Dicts within list are not identical'
+
+        return None
diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
index c664c9d0..b01e6cb8 100644
--- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -79,9 +79,9 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         services.append(this_service)
         use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
                       'ceph-osd', 'ceph-radosgw']
-        # Openstack subordinate charms do not expose an origin option as that
-        # is controlled by the principle
-        ignore = ['neutron-openvswitch']
+        # Most OpenStack subordinate charms do not expose an origin option
+        # as that is controlled by the principle.
+        ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch']
 
         if self.openstack:
             for svc in services:
@@ -148,3 +148,36 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             return os_origin.split('%s-' % self.series)[1].split('/')[0]
         else:
             return releases[self.series]
+
+    def get_ceph_expected_pools(self, radosgw=False):
+        """Return a list of expected ceph pools in a ceph + cinder + glance
+        test scenario, based on OpenStack release and whether ceph radosgw
+        is flagged as present or not."""
+
+        if self._get_openstack_release() >= self.trusty_kilo:
+            # Kilo or later
+            pools = [
+                'rbd',
+                'cinder',
+                'glance'
+            ]
+        else:
+            # Juno or earlier
+            pools = [
+                'data',
+                'metadata',
+                'rbd',
+                'cinder',
+                'glance'
+            ]
+
+        if radosgw:
+            pools.extend([
+                '.rgw.root',
+                '.rgw.control',
+                '.rgw',
+                '.rgw.gc',
+                '.users.uid'
+            ])
+
+        return pools
diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py
index 576bf0b5..03f79277 100644
--- a/tests/charmhelpers/contrib/openstack/amulet/utils.py
+++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py
@@ -14,16 +14,20 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import amulet
+import json
 import logging
 import os
 import six
 import time
 import urllib
 
+import cinderclient.v1.client as cinder_client
 import glanceclient.v1.client as glance_client
 import heatclient.v1.client as heat_client
 import keystoneclient.v2_0 as keystone_client
 import novaclient.v1_1.client as nova_client
+import swiftclient
 
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
@@ -171,6 +175,16 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
+    def authenticate_cinder_admin(self, keystone_sentry, username,
+                                  password, tenant):
+        """Authenticates admin user with cinder."""
+        # NOTE(beisner): cinder python client doesn't accept tokens.
+        service_ip = \
+            keystone_sentry.relation('shared-db',
+                                     'mysql:shared-db')['private-address']
+        ept = "http://{}:5000/v2.0".format(service_ip.strip().decode('utf-8'))
+        return cinder_client.Client(username, password, tenant, ept)
+
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
                                     tenant):
         """Authenticates admin user with the keystone admin endpoint."""
@@ -212,9 +226,29 @@ class OpenStackAmuletUtils(AmuletUtils):
         return nova_client.Client(username=user, api_key=password,
                                   project_id=tenant, auth_url=ep)
 
+    def authenticate_swift_user(self, keystone, user, password, tenant):
+        """Authenticates a regular user with swift api."""
+        self.log.debug('Authenticating swift user ({})...'.format(user))
+        ep = keystone.service_catalog.url_for(service_type='identity',
+                                              endpoint_type='publicURL')
+        return swiftclient.Connection(authurl=ep,
+                                      user=user,
+                                      key=password,
+                                      tenant_name=tenant,
+                                      auth_version='2.0')
+
     def create_cirros_image(self, glance, image_name):
-        """Download the latest cirros image and upload it to glance."""
-        self.log.debug('Creating glance image ({})...'.format(image_name))
+        """Download the latest cirros image and upload it to glance,
+        validate and return a resource pointer.
+
+        :param glance: pointer to authenticated glance connection
+        :param image_name: display name for new image
+        :returns: glance image pointer
+        """
+        self.log.debug('Creating glance cirros image '
+                       '({})...'.format(image_name))
+
+        # Download cirros image
         http_proxy = os.getenv('AMULET_HTTP_PROXY')
         self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
         if http_proxy:
@@ -223,33 +257,51 @@ class OpenStackAmuletUtils(AmuletUtils):
         else:
             opener = urllib.FancyURLopener()
 
-        f = opener.open("http://download.cirros-cloud.net/version/released")
+        f = opener.open('http://download.cirros-cloud.net/version/released')
         version = f.read().strip()
-        cirros_img = "cirros-{}-x86_64-disk.img".format(version)
+        cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
         local_path = os.path.join('tests', cirros_img)
 
         if not os.path.exists(local_path):
-            cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net",
+            cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
                                                   version, cirros_img)
             opener.retrieve(cirros_url, local_path)
         f.close()
 
+        # Create glance image
         with open(local_path) as f:
             image = glance.images.create(name=image_name, is_public=True,
                                          disk_format='qcow2',
                                          container_format='bare', data=f)
-        count = 1
-        status = image.status
-        while status != 'active' and count < 10:
-            time.sleep(3)
-            image = glance.images.get(image.id)
-            status = image.status
-            self.log.debug('image status: {}'.format(status))
-            count += 1
 
-        if status != 'active':
-            self.log.error('image creation timed out')
-            return None
+        # Wait for image to reach active status
+        img_id = image.id
+        ret = self.resource_reaches_status(glance.images, img_id,
+                                           expected_stat='active',
+                                           msg='Image status wait')
+        if not ret:
+            msg = 'Glance image failed to reach expected state.'
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Re-validate new image
+        self.log.debug('Validating image attributes...')
+        val_img_name = glance.images.get(img_id).name
+        val_img_stat = glance.images.get(img_id).status
+        val_img_pub = glance.images.get(img_id).is_public
+        val_img_cfmt = glance.images.get(img_id).container_format
+        val_img_dfmt = glance.images.get(img_id).disk_format
+        msg_attr = ('Image attributes - name:{} public:{} id:{} stat:{} '
+                    'container fmt:{} disk fmt:{}'.format(
+                        val_img_name, val_img_pub, img_id,
+                        val_img_stat, val_img_cfmt, val_img_dfmt))
+
+        if val_img_name == image_name and val_img_stat == 'active' \
+                and val_img_pub is True and val_img_cfmt == 'bare' \
+                and val_img_dfmt == 'qcow2':
+            self.log.debug(msg_attr)
+        else:
+            msg = ('Volume validation failed, {}'.format(msg_attr))
+            amulet.raise_status(amulet.FAIL, msg=msg)
 
         return image
 
@@ -260,22 +312,7 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.warn('/!\\ DEPRECATION WARNING:  use '
                       'delete_resource instead of delete_image.')
         self.log.debug('Deleting glance image ({})...'.format(image))
-        num_before = len(list(glance.images.list()))
-        glance.images.delete(image)
-
-        count = 1
-        num_after = len(list(glance.images.list()))
-        while num_after != (num_before - 1) and count < 10:
-            time.sleep(3)
-            num_after = len(list(glance.images.list()))
-            self.log.debug('number of images: {}'.format(num_after))
-            count += 1
-
-        if num_after != (num_before - 1):
-            self.log.error('image deletion timed out')
-            return False
-
-        return True
+        return self.delete_resource(glance.images, image, msg='glance image')
 
     def create_instance(self, nova, image_name, instance_name, flavor):
         """Create the specified instance."""
@@ -308,22 +345,8 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.warn('/!\\ DEPRECATION WARNING:  use '
                       'delete_resource instead of delete_instance.')
         self.log.debug('Deleting instance ({})...'.format(instance))
-        num_before = len(list(nova.servers.list()))
-        nova.servers.delete(instance)
-
-        count = 1
-        num_after = len(list(nova.servers.list()))
-        while num_after != (num_before - 1) and count < 10:
-            time.sleep(3)
-            num_after = len(list(nova.servers.list()))
-            self.log.debug('number of instances: {}'.format(num_after))
-            count += 1
-
-        if num_after != (num_before - 1):
-            self.log.error('instance deletion timed out')
-            return False
-
-        return True
+        return self.delete_resource(nova.servers, instance,
+                                    msg='nova instance')
 
     def create_or_get_keypair(self, nova, keypair_name="testkey"):
         """Create a new keypair, or return pointer if it already exists."""
@@ -339,6 +362,88 @@ class OpenStackAmuletUtils(AmuletUtils):
         _keypair = nova.keypairs.create(name=keypair_name)
         return _keypair
 
+    def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1,
+                             img_id=None, src_vol_id=None, snap_id=None):
+        """Create cinder volume, optionally from a glance image, OR
+        optionally as a clone of an existing volume, OR optionally
+        from a snapshot.  Wait for the new volume status to reach
+        the expected status, validate and return a resource pointer.
+
+        :param vol_name: cinder volume display name
+        :param vol_size: size in gigabytes
+        :param img_id: optional glance image id
+        :param src_vol_id: optional source volume id to clone
+        :param snap_id: optional snapshot id to use
+        :returns: cinder volume pointer
+        """
+        # Handle parameter input and avoid impossible combinations
+        if img_id and not src_vol_id and not snap_id:
+            # Create volume from image
+            self.log.debug('Creating cinder volume from glance image...')
+            bootable = 'true'
+        elif src_vol_id and not img_id and not snap_id:
+            # Clone an existing volume
+            self.log.debug('Cloning cinder volume...')
+            bootable = cinder.volumes.get(src_vol_id).bootable
+        elif snap_id and not src_vol_id and not img_id:
+            # Create volume from snapshot
+            self.log.debug('Creating cinder volume from snapshot...')
+            snap = cinder.volume_snapshots.find(id=snap_id)
+            vol_size = snap.size
+            snap_vol_id = cinder.volume_snapshots.get(snap_id).volume_id
+            bootable = cinder.volumes.get(snap_vol_id).bootable
+        elif not img_id and not src_vol_id and not snap_id:
+            # Create volume
+            self.log.debug('Creating cinder volume...')
+            bootable = 'false'
+        else:
+            # Impossible combination of parameters
+            msg = ('Invalid method use - name:{} size:{} img_id:{} '
+                   'src_vol_id:{} snap_id:{}'.format(vol_name, vol_size,
+                                                     img_id, src_vol_id,
+                                                     snap_id))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Create new volume
+        try:
+            vol_new = cinder.volumes.create(display_name=vol_name,
+                                            imageRef=img_id,
+                                            size=vol_size,
+                                            source_volid=src_vol_id,
+                                            snapshot_id=snap_id)
+            vol_id = vol_new.id
+        except Exception as e:
+            msg = 'Failed to create volume: {}'.format(e)
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Wait for volume to reach available status
+        ret = self.resource_reaches_status(cinder.volumes, vol_id,
+                                           expected_stat="available",
+                                           msg="Volume status wait")
+        if not ret:
+            msg = 'Cinder volume failed to reach expected state.'
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Re-validate new volume
+        self.log.debug('Validating volume attributes...')
+        val_vol_name = cinder.volumes.get(vol_id).display_name
+        val_vol_boot = cinder.volumes.get(vol_id).bootable
+        val_vol_stat = cinder.volumes.get(vol_id).status
+        val_vol_size = cinder.volumes.get(vol_id).size
+        msg_attr = ('Volume attributes - name:{} id:{} stat:{} boot:'
+                    '{} size:{}'.format(val_vol_name, vol_id,
+                                        val_vol_stat, val_vol_boot,
+                                        val_vol_size))
+
+        if val_vol_boot == bootable and val_vol_stat == 'available' \
+                and val_vol_name == vol_name and val_vol_size == vol_size:
+            self.log.debug(msg_attr)
+        else:
+            msg = ('Volume validation failed, {}'.format(msg_attr))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        return vol_new
+
     def delete_resource(self, resource, resource_id,
                         msg="resource", max_wait=120):
         """Delete one openstack resource, such as one instance, keypair,
@@ -350,6 +455,8 @@ class OpenStackAmuletUtils(AmuletUtils):
         :param max_wait: maximum wait time in seconds
         :returns: True if successful, otherwise False
         """
+        self.log.debug('Deleting OpenStack resource '
+                       '{} ({})'.format(resource_id, msg))
         num_before = len(list(resource.list()))
         resource.delete(resource_id)
 
@@ -411,3 +518,87 @@ class OpenStackAmuletUtils(AmuletUtils):
             self.log.debug('{} never reached expected status: '
                            '{}'.format(resource_id, expected_stat))
             return False
+
+    def get_ceph_osd_id_cmd(self, index):
+        """Produce a shell command that will return a ceph-osd id."""
+        return ("`initctl list | grep 'ceph-osd ' | "
+                "awk 'NR=={} {{ print $2 }}' | "
+                "grep -o '[0-9]*'`".format(index + 1))
+
+    def get_ceph_pools(self, sentry_unit):
+        """Return a dict of ceph pools from a single ceph unit, with
+        pool name as keys, pool id as vals."""
+        pools = {}
+        cmd = 'sudo ceph osd lspools'
+        output, code = sentry_unit.run(cmd)
+        if code != 0:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance,
+        for pool in str(output).split(','):
+            pool_id_name = pool.split(' ')
+            if len(pool_id_name) == 2:
+                pool_id = pool_id_name[0]
+                pool_name = pool_id_name[1]
+                pools[pool_name] = int(pool_id)
+
+        self.log.debug('Pools on {}: {}'.format(sentry_unit.info['unit_name'],
+                                                pools))
+        return pools
+
+    def get_ceph_df(self, sentry_unit):
+        """Return dict of ceph df json output, including ceph pool state.
+
+        :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+        :returns: Dict of ceph df output
+        """
+        cmd = 'sudo ceph df --format=json'
+        output, code = sentry_unit.run(cmd)
+        if code != 0:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+        return json.loads(output)
+
+    def get_ceph_pool_sample(self, sentry_unit, pool_id=0):
+        """Take a sample of attributes of a ceph pool, returning ceph
+        pool name, object count and disk space used for the specified
+        pool ID number.
+
+        :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+        :param pool_id: Ceph pool ID
+        :returns: List of pool name, object count, kb disk space used
+        """
+        df = self.get_ceph_df(sentry_unit)
+        pool_name = df['pools'][pool_id]['name']
+        obj_count = df['pools'][pool_id]['stats']['objects']
+        kb_used = df['pools'][pool_id]['stats']['kb_used']
+        self.log.debug('Ceph {} pool (ID {}): {} objects, '
+                       '{} kb used'.format(pool_name, pool_id,
+                                           obj_count, kb_used))
+        return pool_name, obj_count, kb_used
+
+    def validate_ceph_pool_samples(self, samples, sample_type="resource pool"):
+        """Validate ceph pool samples taken over time, such as pool
+        object counts or pool kb used, before adding, after adding, and
+        after deleting items which affect those pool attributes.  The
+        2nd element is expected to be greater than the 1st; 3rd is expected
+        to be less than the 2nd.
+
+        :param samples: List containing 3 data samples
+        :param sample_type: String for logging and usage context
+        :returns: None if successful, Failure message otherwise
+        """
+        original, created, deleted = range(3)
+        if samples[created] <= samples[original] or \
+                samples[deleted] >= samples[created]:
+            return ('Ceph {} samples ({}) '
+                    'unexpected.'.format(sample_type, samples))
+        else:
+            self.log.debug('Ceph {} samples (OK): '
+                           '{}'.format(sample_type, samples))
+            return None

From 40fde79e55b9ca37fb6624b7bac08f458d75ebdd Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Tue, 21 Jul 2015 15:31:39 +0000
Subject: [PATCH 75/88] Temp ppa override

---
 files/patched-icehouse     | 4 ++++
 hooks/neutron_api_hooks.py | 6 ++++++
 2 files changed, 10 insertions(+)
 create mode 100644 files/patched-icehouse

diff --git a/files/patched-icehouse b/files/patched-icehouse
new file mode 100644
index 00000000..c7bea34c
--- /dev/null
+++ b/files/patched-icehouse
@@ -0,0 +1,4 @@
+[ /etc/apt/preferences.d/patched-icehouse ]
+Package: *
+Pin: release o=LP-PPA-sdn-charmers-cisco-vpp-testing
+Pin-Priority: 990
diff --git a/hooks/neutron_api_hooks.py b/hooks/neutron_api_hooks.py
index b79571e1..15b79ab3 100755
--- a/hooks/neutron_api_hooks.py
+++ b/hooks/neutron_api_hooks.py
@@ -9,6 +9,7 @@ from subprocess import (
 from charmhelpers.core.hookenv import (
     Hooks,
     UnregisteredHookError,
+    charm_dir,
     config,
     is_relation_made,
     local_unit,
@@ -143,6 +144,11 @@ def configure_https():
 def install():
     execd_preinstall()
     configure_installation_source(config('openstack-origin'))
+    # XXX Remove me when patched nova and neutron are in the main ppa
+    configure_installation_source('ppa:sdn-charmers/cisco-vpp-testing')
+    apt_pin_file = charm_dir() + '/files/patched-icehouse'
+    import shutil
+    shutil.copyfile(apt_pin_file, '/etc/apt/preferences.d/patched-icehouse')
 
     apt_update()
     apt_install(determine_packages(config('openstack-origin')),

From c8c7beb1e39d08607d0f39d8bebce0fd7f61117b Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Sun, 26 Jul 2015 14:59:38 +0100
Subject: [PATCH 76/88] Remove temporary ppa hack

---
 hooks/neutron_api_hooks.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/hooks/neutron_api_hooks.py b/hooks/neutron_api_hooks.py
index 15b79ab3..8edc9b09 100755
--- a/hooks/neutron_api_hooks.py
+++ b/hooks/neutron_api_hooks.py
@@ -144,11 +144,6 @@ def configure_https():
 def install():
     execd_preinstall()
     configure_installation_source(config('openstack-origin'))
-    # XXX Remove me when patched nova and neutron are in the main ppa
-    configure_installation_source('ppa:sdn-charmers/cisco-vpp-testing')
-    apt_pin_file = charm_dir() + '/files/patched-icehouse'
-    import shutil
-    shutil.copyfile(apt_pin_file, '/etc/apt/preferences.d/patched-icehouse')
 
     apt_update()
     apt_install(determine_packages(config('openstack-origin')),

From 9d88d29605807cbacccde34f6ccfa4576c073bf6 Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Wed, 29 Jul 2015 11:13:49 +0100
Subject: [PATCH 77/88] Add ability to except subordinate config in icehouse
 and juno. Other fixes from jamespages review comments

---
 config.yaml                     |  2 +-
 templates/icehouse/neutron.conf |  6 ++++++
 templates/juno/neutron.conf     |  6 ++++++
 templates/kilo/neutron.conf     | 20 --------------------
 4 files changed, 13 insertions(+), 21 deletions(-)

diff --git a/config.yaml b/config.yaml
index 3affd361..76e52f3a 100644
--- a/config.yaml
+++ b/config.yaml
@@ -360,4 +360,4 @@ options:
     default: True
     description: |
       If True neutron-server will install neutron packages for the plugin
-      stipulated.
+      configured.
diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf
index 08d14fc7..177eaea3 100644
--- a/templates/icehouse/neutron.conf
+++ b/templates/icehouse/neutron.conf
@@ -54,6 +54,12 @@ nova_admin_password = {{ admin_password }}
 nova_admin_auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v2.0
 {% endif -%}
 
+{% if sections and 'DEFAULT' in sections -%}
+{% for key, value in sections['DEFAULT'] -%}
+{{ key }} = {{ value }}
+{% endfor -%}
+{% endif %}
+
 [quotas]
 quota_driver = neutron.db.quota_db.DbQuotaDriver
 {% if neutron_security_groups -%}
diff --git a/templates/juno/neutron.conf b/templates/juno/neutron.conf
index 05d3e212..a4c97285 100644
--- a/templates/juno/neutron.conf
+++ b/templates/juno/neutron.conf
@@ -54,6 +54,12 @@ nova_admin_password = {{ admin_password }}
 nova_admin_auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v2.0
 {% endif -%}
 
+{% if sections and 'DEFAULT' in sections -%}
+{% for key, value in sections['DEFAULT'] -%}
+{{ key }} = {{ value }}
+{% endfor -%}
+{% endif %}
+
 [quotas]
 quota_driver = neutron.db.quota_db.DbQuotaDriver
 {% if neutron_security_groups -%}
diff --git a/templates/kilo/neutron.conf b/templates/kilo/neutron.conf
index bf62a467..5e067f00 100644
--- a/templates/kilo/neutron.conf
+++ b/templates/kilo/neutron.conf
@@ -83,11 +83,6 @@ quota_member = {{ quota_member }}
 quota_health_monitors = {{ quota_health_monitors }}
 quota_router = {{ quota_router }}
 quota_floatingip = {{ quota_floatingip }}
-{% if sections and 'quotas' in sections -%}
-{% for key, value in sections['quotas'] -%}
-{{ key }} = {{ value }}
-{% endfor -%}
-{% endif %}
 
 [agent]
 root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
@@ -97,26 +92,11 @@ root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
 {% include "parts/section-database" %}
 
 {% include "section-rabbitmq-oslo" %}
-{% if sections and 'agent' in sections -%}
-{% for key, value in sections['agent'] -%}
-{{ key }} = {{ value }}
-{% endfor -%}
-{% endif %}
 
 [service_providers]
 service_provider=LOADBALANCER:Haproxy:neutron_lbaas.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
 service_provider=VPN:openswan:neutron_vpnaas.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default
 service_provider=FIREWALL:Iptables:neutron_fwaas.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver:default
-{% if sections and 'service_providers' in sections -%}
-{% for key, value in sections['service_providers'] -%}
-{{ key }} = {{ value }}
-{% endfor -%}
-{% endif %}
 
 [oslo_concurrency]
 lock_path = $state_path/lock
-{% if sections and 'oslo_concurrency' in sections -%}
-{% for key, value in sections['oslo_concurrency'] -%}
-{{ key }} = {{ value }}
-{% endfor -%}
-{% endif %}

From e34570ebc9e9a3d876d5062174504153f06bbf7c Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Wed, 29 Jul 2015 11:46:57 +0100
Subject: [PATCH 78/88] [gnuoy,trivial] Pre-release charmhelper sync

---
 .../charmhelpers/contrib/openstack/context.py | 18 +++++---
 .../contrib/openstack/templating.py           |  4 +-
 .../contrib/storage/linux/utils.py            |  2 +-
 hooks/charmhelpers/core/files.py              | 45 +++++++++++++++++++
 hooks/charmhelpers/core/hookenv.py            |  3 +-
 5 files changed, 63 insertions(+), 9 deletions(-)
 create mode 100644 hooks/charmhelpers/core/files.py

diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py
index 8f3f1b15..bbf4722b 100644
--- a/hooks/charmhelpers/contrib/openstack/context.py
+++ b/hooks/charmhelpers/contrib/openstack/context.py
@@ -1053,11 +1053,17 @@ class SubordinateConfigContext(OSContextGenerator):
         """
         self.service = service
         self.config_file = config_file
-        self.interface = interface
+        if isinstance(interface, list):
+            self.interfaces = interface
+        else:
+            self.interfaces = [interface]
 
     def __call__(self):
         ctxt = {'sections': {}}
-        for rid in relation_ids(self.interface):
+        rids = []
+        for interface in self.interfaces:
+            rids.extend(relation_ids(interface))
+        for rid in rids:
             for unit in related_units(rid):
                 sub_config = relation_get('subordinate_configuration',
                                           rid=rid, unit=unit)
@@ -1085,13 +1091,15 @@ class SubordinateConfigContext(OSContextGenerator):
                     sub_config = sub_config[self.config_file]
                     for k, v in six.iteritems(sub_config):
                         if k == 'sections':
-                            for section, config_dict in six.iteritems(v):
+                            for section, config_list in six.iteritems(v):
                                 log("adding section '%s'" % (section),
                                     level=DEBUG)
-                                ctxt[k][section] = config_dict
+                                if ctxt[k].get(section):
+                                    ctxt[k][section].extend(config_list)
+                                else:
+                                    ctxt[k][section] = config_list
                         else:
                             ctxt[k] = v
-
         log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)
         return ctxt
 
diff --git a/hooks/charmhelpers/contrib/openstack/templating.py b/hooks/charmhelpers/contrib/openstack/templating.py
index 24cb272b..021d8cf9 100644
--- a/hooks/charmhelpers/contrib/openstack/templating.py
+++ b/hooks/charmhelpers/contrib/openstack/templating.py
@@ -29,8 +29,8 @@ from charmhelpers.contrib.openstack.utils import OPENSTACK_CODENAMES
 try:
     from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 except ImportError:
-    # python-jinja2 may not be installed yet, or we're running unittests.
-    FileSystemLoader = ChoiceLoader = Environment = exceptions = None
+    apt_install('python-jinja2', fatal=True)
+    from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 
 
 class OSConfigException(Exception):
diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py
index c8373b72..e2769e49 100644
--- a/hooks/charmhelpers/contrib/storage/linux/utils.py
+++ b/hooks/charmhelpers/contrib/storage/linux/utils.py
@@ -67,4 +67,4 @@ def is_device_mounted(device):
     out = check_output(['mount']).decode('UTF-8')
     if is_partition:
         return bool(re.search(device + r"\b", out))
-    return bool(re.search(device + r"[0-9]+\b", out))
+    return bool(re.search(device + r"[0-9]*\b", out))
diff --git a/hooks/charmhelpers/core/files.py b/hooks/charmhelpers/core/files.py
new file mode 100644
index 00000000..0f12d321
--- /dev/null
+++ b/hooks/charmhelpers/core/files.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = 'Jorge Niedbalski <niedbalski@ubuntu.com>'
+
+import os
+import subprocess
+
+
+def sed(filename, before, after, flags='g'):
+    """
+    Search and replaces the given pattern on filename.
+
+    :param filename: relative or absolute file path.
+    :param before: expression to be replaced (see 'man sed')
+    :param after: expression to replace with (see 'man sed')
+    :param flags: sed-compatible regex flags in example, to make
+    the  search and replace case insensitive, specify ``flags="i"``.
+    The ``g`` flag is always specified regardless, so you do not
+    need to remember to include it when overriding this parameter.
+    :returns: If the sed command exit code was zero then return,
+    otherwise raise CalledProcessError.
+    """
+    expression = r's/{0}/{1}/{2}'.format(before,
+                                         after, flags)
+
+    return subprocess.check_call(["sed", "-i", "-r", "-e",
+                                  expression,
+                                  os.path.expanduser(filename)])
diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py
index dd8def9a..15b09d11 100644
--- a/hooks/charmhelpers/core/hookenv.py
+++ b/hooks/charmhelpers/core/hookenv.py
@@ -21,6 +21,7 @@
 #  Charm Helpers Developers <juju@lists.ubuntu.com>
 
 from __future__ import print_function
+import copy
 from distutils.version import LooseVersion
 from functools import wraps
 import glob
@@ -263,7 +264,7 @@ class Config(dict):
         self.path = path or self.path
         with open(self.path) as f:
             self._prev_dict = json.load(f)
-        for k, v in self._prev_dict.items():
+        for k, v in copy.deepcopy(self._prev_dict).items():
             if k not in self:
                 self[k] = v
 

From 4e41fcb505dd3b6f876ca6b5930d4abd138b02bd Mon Sep 17 00:00:00 2001
From: Ryan Beisner <ryan.beisner@canonical.com>
Date: Wed, 29 Jul 2015 17:23:27 +0000
Subject: [PATCH 79/88] remove amulet tests for unsupported releases

---
 tests/018-basic-utopic-juno | 9 ---------
 1 file changed, 9 deletions(-)
 delete mode 100755 tests/018-basic-utopic-juno

diff --git a/tests/018-basic-utopic-juno b/tests/018-basic-utopic-juno
deleted file mode 100755
index 219af149..00000000
--- a/tests/018-basic-utopic-juno
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/usr/bin/python
-
-"""Amulet tests on a basic neutron-api deployment on utopic-juno."""
-
-from basic_deployment import NeutronAPIBasicDeployment
-
-if __name__ == '__main__':
-    deployment = NeutronAPIBasicDeployment(series='utopic')
-    deployment.run_tests()

From fdaa26dfe9de788ea501b91e5b1fcefaa175acfa Mon Sep 17 00:00:00 2001
From: Ryan Beisner <ryan.beisner@canonical.com>
Date: Wed, 29 Jul 2015 17:23:43 +0000
Subject: [PATCH 80/88] rename quantum to neutron in amulet tests

---
 tests/basic_deployment.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index dced4d4c..b3c34b56 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -81,7 +81,7 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
                           {'name': 'rabbitmq-server'}, {'name': 'keystone'},
                           {'name': 'neutron-openvswitch'},
                           {'name': 'nova-cloud-controller'},
-                          {'name': 'quantum-gateway'},
+                          {'name': 'neutron-gateway'},
                           {'name': 'nova-compute'}]
         super(NeutronAPIBasicDeployment, self)._add_services(this_service,
                                                              other_services)
@@ -92,7 +92,7 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
             'neutron-api:shared-db': 'mysql:shared-db',
             'neutron-api:amqp': 'rabbitmq-server:amqp',
             'neutron-api:neutron-api': 'nova-cloud-controller:neutron-api',
-            'neutron-api:neutron-plugin-api': 'quantum-gateway:'
+            'neutron-api:neutron-plugin-api': 'neutron-gateway:'
                                               'neutron-plugin-api',
             'neutron-api:neutron-plugin-api': 'neutron-openvswitch:'
                                               'neutron-plugin-api',
@@ -171,7 +171,7 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
         self.keystone_sentry = self.d.sentry.unit['keystone/0']
         self.rabbitmq_sentry = self.d.sentry.unit['rabbitmq-server/0']
         self.nova_cc_sentry = self.d.sentry.unit['nova-cloud-controller/0']
-        self.quantum_gateway_sentry = self.d.sentry.unit['quantum-gateway/0']
+        self.neutron_gateway_sentry = self.d.sentry.unit['neutron-gateway/0']
         self.neutron_api_sentry = self.d.sentry.unit['neutron-api/0']
         self.nova_compute_sentry = self.d.sentry.unit['nova-compute/0']
         u.log.debug('openstack release val: {}'.format(
@@ -212,7 +212,7 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
             self.mysql_sentry: ['status mysql'],
             self.keystone_sentry: ['status keystone'],
             self.nova_cc_sentry: nova_cc_services,
-            self.quantum_gateway_sentry: neutron_services,
+            self.neutron_gateway_sentry: neutron_services,
             self.neutron_api_sentry: neutron_api_services,
         }
 

From 46dda298c811c61dc57504445f3e7d79c153c219 Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Fri, 31 Jul 2015 14:10:47 +0100
Subject: [PATCH 81/88] [gnuoy,trivial] Pre-release charmhelper sync to pickup
 cli module

---
 charm-helpers-hooks.yaml                      |   1 +
 hooks/charmhelpers/cli/__init__.py            | 195 ++++++++++++++++++
 hooks/charmhelpers/cli/benchmark.py           |  36 ++++
 hooks/charmhelpers/cli/commands.py            |  32 +++
 hooks/charmhelpers/cli/host.py                |  31 +++
 hooks/charmhelpers/cli/unitdata.py            |  39 ++++
 .../charmhelpers/contrib/openstack/context.py |  52 ++---
 hooks/charmhelpers/core/hookenv.py            | 106 +++++++++-
 hooks/charmhelpers/core/unitdata.py           |  78 +++++--
 9 files changed, 525 insertions(+), 45 deletions(-)
 create mode 100644 hooks/charmhelpers/cli/__init__.py
 create mode 100644 hooks/charmhelpers/cli/benchmark.py
 create mode 100644 hooks/charmhelpers/cli/commands.py
 create mode 100644 hooks/charmhelpers/cli/host.py
 create mode 100644 hooks/charmhelpers/cli/unitdata.py

diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml
index 917cf211..ff6c3546 100644
--- a/charm-helpers-hooks.yaml
+++ b/charm-helpers-hooks.yaml
@@ -2,6 +2,7 @@ branch: lp:charm-helpers
 destination: hooks/charmhelpers
 include:
     - core
+    - cli
     - fetch
     - contrib.openstack|inc=*
     - contrib.hahelpers
diff --git a/hooks/charmhelpers/cli/__init__.py b/hooks/charmhelpers/cli/__init__.py
new file mode 100644
index 00000000..7118daf5
--- /dev/null
+++ b/hooks/charmhelpers/cli/__init__.py
@@ -0,0 +1,195 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import inspect
+import argparse
+import sys
+
+from six.moves import zip
+
+from charmhelpers.core import unitdata
+
+
+class OutputFormatter(object):
+    def __init__(self, outfile=sys.stdout):
+        self.formats = (
+            "raw",
+            "json",
+            "py",
+            "yaml",
+            "csv",
+            "tab",
+        )
+        self.outfile = outfile
+
+    def add_arguments(self, argument_parser):
+        formatgroup = argument_parser.add_mutually_exclusive_group()
+        choices = self.supported_formats
+        formatgroup.add_argument("--format", metavar='FMT',
+                                 help="Select output format for returned data, "
+                                      "where FMT is one of: {}".format(choices),
+                                 choices=choices, default='raw')
+        for fmt in self.formats:
+            fmtfunc = getattr(self, fmt)
+            formatgroup.add_argument("-{}".format(fmt[0]),
+                                     "--{}".format(fmt), action='store_const',
+                                     const=fmt, dest='format',
+                                     help=fmtfunc.__doc__)
+
+    @property
+    def supported_formats(self):
+        return self.formats
+
+    def raw(self, output):
+        """Output data as raw string (default)"""
+        if isinstance(output, (list, tuple)):
+            output = '\n'.join(map(str, output))
+        self.outfile.write(str(output))
+
+    def py(self, output):
+        """Output data as a nicely-formatted python data structure"""
+        import pprint
+        pprint.pprint(output, stream=self.outfile)
+
+    def json(self, output):
+        """Output data in JSON format"""
+        import json
+        json.dump(output, self.outfile)
+
+    def yaml(self, output):
+        """Output data in YAML format"""
+        import yaml
+        yaml.safe_dump(output, self.outfile)
+
+    def csv(self, output):
+        """Output data as excel-compatible CSV"""
+        import csv
+        csvwriter = csv.writer(self.outfile)
+        csvwriter.writerows(output)
+
+    def tab(self, output):
+        """Output data in excel-compatible tab-delimited format"""
+        import csv
+        csvwriter = csv.writer(self.outfile, dialect=csv.excel_tab)
+        csvwriter.writerows(output)
+
+    def format_output(self, output, fmt='raw'):
+        fmtfunc = getattr(self, fmt)
+        fmtfunc(output)
+
+
+class CommandLine(object):
+    argument_parser = None
+    subparsers = None
+    formatter = None
+    exit_code = 0
+
+    def __init__(self):
+        if not self.argument_parser:
+            self.argument_parser = argparse.ArgumentParser(description='Perform common charm tasks')
+        if not self.formatter:
+            self.formatter = OutputFormatter()
+            self.formatter.add_arguments(self.argument_parser)
+        if not self.subparsers:
+            self.subparsers = self.argument_parser.add_subparsers(help='Commands')
+
+    def subcommand(self, command_name=None):
+        """
+        Decorate a function as a subcommand. Use its arguments as the
+        command-line arguments"""
+        def wrapper(decorated):
+            cmd_name = command_name or decorated.__name__
+            subparser = self.subparsers.add_parser(cmd_name,
+                                                   description=decorated.__doc__)
+            for args, kwargs in describe_arguments(decorated):
+                subparser.add_argument(*args, **kwargs)
+            subparser.set_defaults(func=decorated)
+            return decorated
+        return wrapper
+
+    def test_command(self, decorated):
+        """
+        Subcommand is a boolean test function, so bool return values should be
+        converted to a 0/1 exit code.
+        """
+        decorated._cli_test_command = True
+        return decorated
+
+    def no_output(self, decorated):
+        """
+        Subcommand is not expected to return a value, so don't print a spurious None.
+        """
+        decorated._cli_no_output = True
+        return decorated
+
+    def subcommand_builder(self, command_name, description=None):
+        """
+        Decorate a function that builds a subcommand. Builders should accept a
+        single argument (the subparser instance) and return the function to be
+        run as the command."""
+        def wrapper(decorated):
+            subparser = self.subparsers.add_parser(command_name)
+            func = decorated(subparser)
+            subparser.set_defaults(func=func)
+            subparser.description = description or func.__doc__
+        return wrapper
+
+    def run(self):
+        "Run cli, processing arguments and executing subcommands."
+        arguments = self.argument_parser.parse_args()
+        argspec = inspect.getargspec(arguments.func)
+        vargs = []
+        kwargs = {}
+        for arg in argspec.args:
+            vargs.append(getattr(arguments, arg))
+        if argspec.varargs:
+            vargs.extend(getattr(arguments, argspec.varargs))
+        if argspec.keywords:
+            for kwarg in argspec.keywords.items():
+                kwargs[kwarg] = getattr(arguments, kwarg)
+        output = arguments.func(*vargs, **kwargs)
+        if getattr(arguments.func, '_cli_test_command', False):
+            self.exit_code = 0 if output else 1
+            output = ''
+        if getattr(arguments.func, '_cli_no_output', False):
+            output = ''
+        self.formatter.format_output(output, arguments.format)
+        if unitdata._KV:
+            unitdata._KV.flush()
+
+
+cmdline = CommandLine()
+
+
+def describe_arguments(func):
+    """
+    Analyze a function's signature and return a data structure suitable for
+    passing in as arguments to an argparse parser's add_argument() method."""
+
+    argspec = inspect.getargspec(func)
+    # we should probably raise an exception somewhere if func includes **kwargs
+    if argspec.defaults:
+        positional_args = argspec.args[:-len(argspec.defaults)]
+        keyword_names = argspec.args[-len(argspec.defaults):]
+        for arg, default in zip(keyword_names, argspec.defaults):
+            yield ('--{}'.format(arg),), {'default': default}
+    else:
+        positional_args = argspec.args
+
+    for arg in positional_args:
+        yield (arg,), {}
+    if argspec.varargs:
+        yield (argspec.varargs,), {'nargs': '*'}
diff --git a/hooks/charmhelpers/cli/benchmark.py b/hooks/charmhelpers/cli/benchmark.py
new file mode 100644
index 00000000..b23c16ce
--- /dev/null
+++ b/hooks/charmhelpers/cli/benchmark.py
@@ -0,0 +1,36 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from . import cmdline
+from charmhelpers.contrib.benchmark import Benchmark
+
+
+@cmdline.subcommand(command_name='benchmark-start')
+def start():
+    Benchmark.start()
+
+
+@cmdline.subcommand(command_name='benchmark-finish')
+def finish():
+    Benchmark.finish()
+
+
+@cmdline.subcommand_builder('benchmark-composite', description="Set the benchmark composite score")
+def service(subparser):
+    subparser.add_argument("value", help="The composite score.")
+    subparser.add_argument("units", help="The units the composite score represents, i.e., 'reads/sec'.")
+    subparser.add_argument("direction", help="'asc' if a lower score is better, 'desc' if a higher score is better.")
+    return Benchmark.set_composite_score
diff --git a/hooks/charmhelpers/cli/commands.py b/hooks/charmhelpers/cli/commands.py
new file mode 100644
index 00000000..443ff05d
--- /dev/null
+++ b/hooks/charmhelpers/cli/commands.py
@@ -0,0 +1,32 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+This module loads sub-modules into the python runtime so they can be
+discovered via the inspect module. In order to prevent flake8 from (rightfully)
+telling us these are unused modules, throw a ' # noqa' at the end of each import
+so that the warning is suppressed.
+"""
+
+from . import CommandLine  # noqa
+
+"""
+Import the sub-modules which have decorated subcommands to register with chlp.
+"""
+import host  # noqa
+import benchmark  # noqa
+import unitdata  # noqa
+from charmhelpers.core import hookenv  # noqa
diff --git a/hooks/charmhelpers/cli/host.py b/hooks/charmhelpers/cli/host.py
new file mode 100644
index 00000000..58e78d6b
--- /dev/null
+++ b/hooks/charmhelpers/cli/host.py
@@ -0,0 +1,31 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from . import cmdline
+from charmhelpers.core import host
+
+
+@cmdline.subcommand()
+def mounts():
+    "List mounts"
+    return host.mounts()
+
+
+@cmdline.subcommand_builder('service', description="Control system services")
+def service(subparser):
+    subparser.add_argument("action", help="The action to perform (start, stop, etc...)")
+    subparser.add_argument("service_name", help="Name of the service to control")
+    return host.service
diff --git a/hooks/charmhelpers/cli/unitdata.py b/hooks/charmhelpers/cli/unitdata.py
new file mode 100644
index 00000000..d1cd95bf
--- /dev/null
+++ b/hooks/charmhelpers/cli/unitdata.py
@@ -0,0 +1,39 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from . import cmdline
+from charmhelpers.core import unitdata
+
+
+@cmdline.subcommand_builder('unitdata', description="Store and retrieve data")
+def unitdata_cmd(subparser):
+    nested = subparser.add_subparsers()
+    get_cmd = nested.add_parser('get', help='Retrieve data')
+    get_cmd.add_argument('key', help='Key to retrieve the value of')
+    get_cmd.set_defaults(action='get', value=None)
+    set_cmd = nested.add_parser('set', help='Store data')
+    set_cmd.add_argument('key', help='Key to set')
+    set_cmd.add_argument('value', help='Value to store')
+    set_cmd.set_defaults(action='set')
+
+    def _unitdata_cmd(action, key, value):
+        if action == 'get':
+            return unitdata.kv().get(key)
+        elif action == 'set':
+            unitdata.kv().set(key, value)
+            unitdata.kv().flush()
+            return ''
+    return _unitdata_cmd
diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py
index bbf4722b..ab2ebac1 100644
--- a/hooks/charmhelpers/contrib/openstack/context.py
+++ b/hooks/charmhelpers/contrib/openstack/context.py
@@ -1051,8 +1051,11 @@ class SubordinateConfigContext(OSContextGenerator):
         :param config_file : Service's config file to query sections
         :param interface   : Subordinate interface to inspect
         """
-        self.service = service
         self.config_file = config_file
+        if isinstance(service, list):
+            self.services = service
+        else:
+            self.services = [service]
         if isinstance(interface, list):
             self.interfaces = interface
         else:
@@ -1075,31 +1078,32 @@ class SubordinateConfigContext(OSContextGenerator):
                             'setting from %s' % rid, level=ERROR)
                         continue
 
-                    if self.service not in sub_config:
-                        log('Found subordinate_config on %s but it contained'
-                            'nothing for %s service' % (rid, self.service),
-                            level=INFO)
-                        continue
+                    for service in self.services:
+                        if service not in sub_config:
+                            log('Found subordinate_config on %s but it contained'
+                                'nothing for %s service' % (rid, service),
+                                level=INFO)
+                            continue
 
-                    sub_config = sub_config[self.service]
-                    if self.config_file not in sub_config:
-                        log('Found subordinate_config on %s but it contained'
-                            'nothing for %s' % (rid, self.config_file),
-                            level=INFO)
-                        continue
+                        sub_config = sub_config[service]
+                        if self.config_file not in sub_config:
+                            log('Found subordinate_config on %s but it contained'
+                                'nothing for %s' % (rid, self.config_file),
+                                level=INFO)
+                            continue
 
-                    sub_config = sub_config[self.config_file]
-                    for k, v in six.iteritems(sub_config):
-                        if k == 'sections':
-                            for section, config_list in six.iteritems(v):
-                                log("adding section '%s'" % (section),
-                                    level=DEBUG)
-                                if ctxt[k].get(section):
-                                    ctxt[k][section].extend(config_list)
-                                else:
-                                    ctxt[k][section] = config_list
-                        else:
-                            ctxt[k] = v
+                        sub_config = sub_config[self.config_file]
+                        for k, v in six.iteritems(sub_config):
+                            if k == 'sections':
+                                for section, config_list in six.iteritems(v):
+                                    log("adding section '%s'" % (section),
+                                        level=DEBUG)
+                                    if ctxt[k].get(section):
+                                        ctxt[k][section].extend(config_list)
+                                    else:
+                                        ctxt[k][section] = config_list
+                            else:
+                                ctxt[k] = v
         log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)
         return ctxt
 
diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py
index 15b09d11..6e4fb686 100644
--- a/hooks/charmhelpers/core/hookenv.py
+++ b/hooks/charmhelpers/core/hookenv.py
@@ -34,6 +34,8 @@ import errno
 import tempfile
 from subprocess import CalledProcessError
 
+from charmhelpers.cli import cmdline
+
 import six
 if not six.PY3:
     from UserDict import UserDict
@@ -173,9 +175,20 @@ def relation_type():
     return os.environ.get('JUJU_RELATION', None)
 
 
-def relation_id():
-    """The relation ID for the current relation hook"""
-    return os.environ.get('JUJU_RELATION_ID', None)
+@cmdline.subcommand()
+@cached
+def relation_id(relation_name=None, service_or_unit=None):
+    """The relation ID for the current or a specified relation"""
+    if not relation_name and not service_or_unit:
+        return os.environ.get('JUJU_RELATION_ID', None)
+    elif relation_name and service_or_unit:
+        service_name = service_or_unit.split('/')[0]
+        for relid in relation_ids(relation_name):
+            remote_service = remote_service_name(relid)
+            if remote_service == service_name:
+                return relid
+    else:
+        raise ValueError('Must specify neither or both of relation_name and service_or_unit')
 
 
 def local_unit():
@@ -188,14 +201,27 @@ def remote_unit():
     return os.environ.get('JUJU_REMOTE_UNIT', None)
 
 
+@cmdline.subcommand()
 def service_name():
     """The name service group this unit belongs to"""
     return local_unit().split('/')[0]
 
 
+@cmdline.subcommand()
+@cached
+def remote_service_name(relid=None):
+    """The remote service name for a given relation-id (or the current relation)"""
+    if relid is None:
+        unit = remote_unit()
+    else:
+        units = related_units(relid)
+        unit = units[0] if units else None
+    return unit.split('/')[0] if unit else None
+
+
 def hook_name():
     """The name of the currently executing hook"""
-    return os.path.basename(sys.argv[0])
+    return os.environ.get('JUJU_HOOK_NAME', os.path.basename(sys.argv[0]))
 
 
 class Config(dict):
@@ -468,6 +494,63 @@ def relation_types():
     return rel_types
 
 
+@cached
+def relation_to_interface(relation_name):
+    """
+    Given the name of a relation, return the interface that relation uses.
+
+    :returns: The interface name, or ``None``.
+    """
+    return relation_to_role_and_interface(relation_name)[1]
+
+
+@cached
+def relation_to_role_and_interface(relation_name):
+    """
+    Given the name of a relation, return the role and the name of the interface
+    that relation uses (where role is one of ``provides``, ``requires``, or ``peer``).
+
+    :returns: A tuple containing ``(role, interface)``, or ``(None, None)``.
+    """
+    _metadata = metadata()
+    for role in ('provides', 'requires', 'peer'):
+        interface = _metadata.get(role, {}).get(relation_name, {}).get('interface')
+        if interface:
+            return role, interface
+    return None, None
+
+
+@cached
+def role_and_interface_to_relations(role, interface_name):
+    """
+    Given a role and interface name, return a list of relation names for the
+    current charm that use that interface under that role (where role is one
+    of ``provides``, ``requires``, or ``peer``).
+
+    :returns: A list of relation names.
+    """
+    _metadata = metadata()
+    results = []
+    for relation_name, relation in _metadata.get(role, {}).items():
+        if relation['interface'] == interface_name:
+            results.append(relation_name)
+    return results
+
+
+@cached
+def interface_to_relations(interface_name):
+    """
+    Given an interface, return a list of relation names for the current
+    charm that use that interface.
+
+    :returns: A list of relation names.
+    """
+    results = []
+    for role in ('provides', 'requires', 'peer'):
+        results.extend(role_and_interface_to_relations(role, interface_name))
+    return results
+
+
 @cached
 def charm_name():
     """Get the name of the current charm as is specified on metadata.yaml"""
@@ -644,6 +727,21 @@ def action_fail(message):
     subprocess.check_call(['action-fail', message])
 
 
+def action_name():
+    """Get the name of the currently executing action."""
+    return os.environ.get('JUJU_ACTION_NAME')
+
+
+def action_uuid():
+    """Get the UUID of the currently executing action."""
+    return os.environ.get('JUJU_ACTION_UUID')
+
+
+def action_tag():
+    """Get the tag for the currently executing action."""
+    return os.environ.get('JUJU_ACTION_TAG')
+
+
 def status_set(workload_state, message):
     """Set the workload state with a message
 
diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py
index 406a35c5..338104e0 100644
--- a/hooks/charmhelpers/core/unitdata.py
+++ b/hooks/charmhelpers/core/unitdata.py
@@ -152,6 +152,7 @@ associated to the hookname.
 import collections
 import contextlib
 import datetime
+import itertools
 import json
 import os
 import pprint
@@ -164,8 +165,7 @@ __author__ = 'Kapil Thangavelu <kapil.foss@gmail.com>'
 class Storage(object):
     """Simple key value database for local unit state within charms.
 
-    Modifications are automatically committed at hook exit. That's
-    currently regardless of exit code.
+    Modifications are not persisted unless :meth:`flush` is called.
 
     To support dicts, lists, integer, floats, and booleans values
     are automatically json encoded/decoded.
@@ -173,8 +173,11 @@ class Storage(object):
     def __init__(self, path=None):
         self.db_path = path
         if path is None:
-            self.db_path = os.path.join(
-                os.environ.get('CHARM_DIR', ''), '.unit-state.db')
+            if 'UNIT_STATE_DB' in os.environ:
+                self.db_path = os.environ['UNIT_STATE_DB']
+            else:
+                self.db_path = os.path.join(
+                    os.environ.get('CHARM_DIR', ''), '.unit-state.db')
         self.conn = sqlite3.connect('%s' % self.db_path)
         self.cursor = self.conn.cursor()
         self.revision = None
@@ -189,15 +192,8 @@ class Storage(object):
         self.conn.close()
         self._closed = True
 
-    def _scoped_query(self, stmt, params=None):
-        if params is None:
-            params = []
-        return stmt, params
-
     def get(self, key, default=None, record=False):
-        self.cursor.execute(
-            *self._scoped_query(
-                'select data from kv where key=?', [key]))
+        self.cursor.execute('select data from kv where key=?', [key])
         result = self.cursor.fetchone()
         if not result:
             return default
@@ -206,33 +202,81 @@ class Storage(object):
         return json.loads(result[0])
 
     def getrange(self, key_prefix, strip=False):
-        stmt = "select key, data from kv where key like '%s%%'" % key_prefix
-        self.cursor.execute(*self._scoped_query(stmt))
+        """
+        Get a range of keys starting with a common prefix as a mapping of
+        keys to values.
+
+        :param str key_prefix: Common prefix among all keys
+        :param bool strip: Optionally strip the common prefix from the key
+            names in the returned dict
+        :return dict: A (possibly empty) dict of key-value mappings
+        """
+        self.cursor.execute("select key, data from kv where key like ?",
+                            ['%s%%' % key_prefix])
         result = self.cursor.fetchall()
 
         if not result:
-            return None
+            return {}
         if not strip:
             key_prefix = ''
         return dict([
             (k[len(key_prefix):], json.loads(v)) for k, v in result])
 
     def update(self, mapping, prefix=""):
+        """
+        Set the values of multiple keys at once.
+
+        :param dict mapping: Mapping of keys to values
+        :param str prefix: Optional prefix to apply to all keys in `mapping`
+            before setting
+        """
         for k, v in mapping.items():
             self.set("%s%s" % (prefix, k), v)
 
     def unset(self, key):
+        """
+        Remove a key from the database entirely.
+        """
         self.cursor.execute('delete from kv where key=?', [key])
         if self.revision and self.cursor.rowcount:
             self.cursor.execute(
                 'insert into kv_revisions values (?, ?, ?)',
                 [key, self.revision, json.dumps('DELETED')])
 
+    def unsetrange(self, keys=None, prefix=""):
+        """
+        Remove a range of keys starting with a common prefix, from the database
+        entirely.
+
+        :param list keys: List of keys to remove.
+        :param str prefix: Optional prefix to apply to all keys in ``keys``
+            before removing.
+        """
+        if keys is not None:
+            keys = ['%s%s' % (prefix, key) for key in keys]
+            self.cursor.execute('delete from kv where key in (%s)' % ','.join(['?'] * len(keys)), keys)
+            if self.revision and self.cursor.rowcount:
+                self.cursor.execute(
+                    'insert into kv_revisions values %s' % ','.join(['(?, ?, ?)'] * len(keys)),
+                    list(itertools.chain.from_iterable((key, self.revision, json.dumps('DELETED')) for key in keys)))
+        else:
+            self.cursor.execute('delete from kv where key like ?',
+                                ['%s%%' % prefix])
+            if self.revision and self.cursor.rowcount:
+                self.cursor.execute(
+                    'insert into kv_revisions values (?, ?, ?)',
+                    ['%s%%' % prefix, self.revision, json.dumps('DELETED')])
+
     def set(self, key, value):
+        """
+        Set a value in the database.
+
+        :param str key: Key to set the value for
+        :param value: Any JSON-serializable value to be set
+        """
         serialized = json.dumps(value)
 
-        self.cursor.execute(
-            'select data from kv where key=?', [key])
+        self.cursor.execute('select data from kv where key=?', [key])
         exists = self.cursor.fetchone()
 
         # Skip mutations to the same value

From dad203ed1ef6c622cdf8be13fbe3f90f777dc0d8 Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Mon, 3 Aug 2015 14:59:32 +0100
Subject: [PATCH 82/88] [gnuoy,trivial] Pre-release charmhelper sync to pickup
 leadership election peer migration fix

---
 hooks/charmhelpers/core/hookenv.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py
index 6e4fb686..18860f59 100644
--- a/hooks/charmhelpers/core/hookenv.py
+++ b/hooks/charmhelpers/core/hookenv.py
@@ -34,7 +34,22 @@ import errno
 import tempfile
 from subprocess import CalledProcessError
 
-from charmhelpers.cli import cmdline
+try:
+    from charmhelpers.cli import cmdline
+except ImportError as e:
+    # due to the anti-pattern of partially synching charmhelpers directly
+    # into charms, it's possible that charmhelpers.cli is not available;
+    # if that's the case, they don't really care about using the cli anyway,
+    # so mock it out
+    if str(e) == 'No module named cli':
+        class cmdline(object):
+            @classmethod
+            def subcommand(cls, *args, **kwargs):
+                def _wrap(func):
+                    return func
+                return _wrap
+    else:
+        raise
 
 import six
 if not six.PY3:

From b01f51dcbf854db5b173732e21702d42ff9a6038 Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Wed, 12 Aug 2015 06:22:30 +0100
Subject: [PATCH 83/88] Remove leftover pin package

---
 files/patched-icehouse | 4 ----
 1 file changed, 4 deletions(-)
 delete mode 100644 files/patched-icehouse

diff --git a/files/patched-icehouse b/files/patched-icehouse
deleted file mode 100644
index c7bea34c..00000000
--- a/files/patched-icehouse
+++ /dev/null
@@ -1,4 +0,0 @@
-[ /etc/apt/preferences.d/patched-icehouse ]
-Package: *
-Pin: release o=LP-PPA-sdn-charmers-cisco-vpp-testing
-Pin-Priority: 990

From 2c11941949b8c15beec3a9f97f95b1140abc561e Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Wed, 12 Aug 2015 11:14:00 +0100
Subject: [PATCH 84/88] Fix kilo template ordering

---
 templates/kilo/neutron.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/templates/kilo/neutron.conf b/templates/kilo/neutron.conf
index 5e067f00..ec54b395 100644
--- a/templates/kilo/neutron.conf
+++ b/templates/kilo/neutron.conf
@@ -57,14 +57,14 @@ nova_admin_password = {{ admin_password }}
 nova_admin_auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v2.0
 {% endif -%}
 
-{% include "section-zeromq" %}
-
 {% if sections and 'DEFAULT' in sections -%}
 {% for key, value in sections['DEFAULT'] -%}
 {{ key }} = {{ value }}
 {% endfor -%}
 {% endif %}
 
+{% include "section-zeromq" %}
+
 [quotas]
 quota_driver = neutron.db.quota_db.DbQuotaDriver
 {% if neutron_security_groups -%}

From e78f7d5f7b119c7577e4b501b506c3c6c30da02b Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Wed, 12 Aug 2015 11:18:00 +0100
Subject: [PATCH 85/88] Fix lint and remove pointless comment from template

---
 hooks/neutron_api_hooks.py  | 1 -
 templates/kilo/neutron.conf | 1 -
 2 files changed, 2 deletions(-)

diff --git a/hooks/neutron_api_hooks.py b/hooks/neutron_api_hooks.py
index 8edc9b09..b79571e1 100755
--- a/hooks/neutron_api_hooks.py
+++ b/hooks/neutron_api_hooks.py
@@ -9,7 +9,6 @@ from subprocess import (
 from charmhelpers.core.hookenv import (
     Hooks,
     UnregisteredHookError,
-    charm_dir,
     config,
     is_relation_made,
     local_unit,
diff --git a/templates/kilo/neutron.conf b/templates/kilo/neutron.conf
index ec54b395..07a0ed88 100644
--- a/templates/kilo/neutron.conf
+++ b/templates/kilo/neutron.conf
@@ -29,7 +29,6 @@ bind_port = {{ neutron_bind_port }}
 bind_port = 9696
 {% endif -%}
 
-# {{ service_plugins }}
 {% if core_plugin -%}
 core_plugin = {{ core_plugin }}
 {% if service_plugins -%}

From efffe3785f380e91259acf3f833daa25e28fd8bd Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Wed, 12 Aug 2015 11:35:59 +0100
Subject: [PATCH 86/88] Fix unit test

---
 unit_tests/test_neutron_api_context.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/unit_tests/test_neutron_api_context.py b/unit_tests/test_neutron_api_context.py
index a0ef6af4..3d12e482 100644
--- a/unit_tests/test_neutron_api_context.py
+++ b/unit_tests/test_neutron_api_context.py
@@ -447,7 +447,7 @@ class NeutronApiSDNContextTest(CharmTestCase):
     def test_init(self):
         napisdn_ctxt = context.NeutronApiSDNContext()
         self.assertEquals(
-            napisdn_ctxt.interface,
+            napisdn_ctxt.interfaces,
             'neutron-plugin-api-subordinate'
         )
         self.assertEquals(napisdn_ctxt.service, 'neutron-api')

From 7482777425fc782cbaf10641e1a93e563ccdcf2c Mon Sep 17 00:00:00 2001
From: Corey Bryant <corey.bryant@canonical.com>
Date: Tue, 18 Aug 2015 13:34:35 -0400
Subject: [PATCH 87/88] [corey.bryant,r=trivial] Sync charm-helpers to pick up
 Liberty support.

---
 hooks/charmhelpers/cli/__init__.py            |   6 +-
 hooks/charmhelpers/cli/commands.py            |   8 +-
 hooks/charmhelpers/cli/hookenv.py             |  23 ++++
 .../contrib/openstack/amulet/deployment.py    |   4 +-
 hooks/charmhelpers/contrib/openstack/utils.py |  65 ++++++++---
 .../contrib/storage/linux/utils.py            |   5 +-
 hooks/charmhelpers/core/hookenv.py            |  21 +---
 hooks/charmhelpers/core/host.py               |  25 ++++-
 hooks/charmhelpers/core/services/helpers.py   |  20 +++-
 hooks/charmhelpers/fetch/__init__.py          |   8 ++
 tests/charmhelpers/contrib/amulet/utils.py    | 105 ++++++++++++++----
 .../contrib/openstack/amulet/deployment.py    |   4 +-
 12 files changed, 220 insertions(+), 74 deletions(-)
 create mode 100644 hooks/charmhelpers/cli/hookenv.py

diff --git a/hooks/charmhelpers/cli/__init__.py b/hooks/charmhelpers/cli/__init__.py
index 7118daf5..16d52cc4 100644
--- a/hooks/charmhelpers/cli/__init__.py
+++ b/hooks/charmhelpers/cli/__init__.py
@@ -152,15 +152,11 @@ class CommandLine(object):
         arguments = self.argument_parser.parse_args()
         argspec = inspect.getargspec(arguments.func)
         vargs = []
-        kwargs = {}
         for arg in argspec.args:
             vargs.append(getattr(arguments, arg))
         if argspec.varargs:
             vargs.extend(getattr(arguments, argspec.varargs))
-        if argspec.keywords:
-            for kwarg in argspec.keywords.items():
-                kwargs[kwarg] = getattr(arguments, kwarg)
-        output = arguments.func(*vargs, **kwargs)
+        output = arguments.func(*vargs)
         if getattr(arguments.func, '_cli_test_command', False):
             self.exit_code = 0 if output else 1
             output = ''
diff --git a/hooks/charmhelpers/cli/commands.py b/hooks/charmhelpers/cli/commands.py
index 443ff05d..7e91db00 100644
--- a/hooks/charmhelpers/cli/commands.py
+++ b/hooks/charmhelpers/cli/commands.py
@@ -26,7 +26,7 @@ from . import CommandLine  # noqa
 """
 Import the sub-modules which have decorated subcommands to register with chlp.
 """
-import host  # noqa
-import benchmark  # noqa
-import unitdata  # noqa
-from charmhelpers.core import hookenv  # noqa
+from . import host  # noqa
+from . import benchmark  # noqa
+from . import unitdata  # noqa
+from . import hookenv  # noqa
diff --git a/hooks/charmhelpers/cli/hookenv.py b/hooks/charmhelpers/cli/hookenv.py
new file mode 100644
index 00000000..265c816e
--- /dev/null
+++ b/hooks/charmhelpers/cli/hookenv.py
@@ -0,0 +1,23 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from . import cmdline
+from charmhelpers.core import hookenv
+
+
+cmdline.subcommand('relation-id')(hookenv.relation_id._wrapped)
+cmdline.subcommand('service-name')(hookenv.service_name)
+cmdline.subcommand('remote-service-name')(hookenv.remote_service_name._wrapped)
diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
index b01e6cb8..07ee2ef1 100644
--- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -44,7 +44,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            Determine if the local branch being tested is derived from its
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
-        base_charms = ['mysql', 'mongodb']
+        base_charms = ['mysql', 'mongodb', 'nrpe']
 
         if self.series in ['precise', 'trusty']:
             base_series = self.series
@@ -81,7 +81,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
                       'ceph-osd', 'ceph-radosgw']
         # Most OpenStack subordinate charms do not expose an origin option
         # as that is controlled by the principle.
-        ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch']
+        ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
 
         if self.openstack:
             for svc in services:
diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index 4dd000c3..c9fd68f7 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -24,6 +24,7 @@ import subprocess
 import json
 import os
 import sys
+import re
 
 import six
 import yaml
@@ -69,7 +70,6 @@ CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
 DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed '
                    'restricted main multiverse universe')
 
-
 UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('oneiric', 'diablo'),
     ('precise', 'essex'),
@@ -118,6 +118,34 @@ SWIFT_CODENAMES = OrderedDict([
     ('2.3.0', 'liberty'),
 ])
 
+# >= Liberty version->codename mapping
+PACKAGE_CODENAMES = {
+    'nova-common': OrderedDict([
+        ('12.0.0', 'liberty'),
+    ]),
+    'neutron-common': OrderedDict([
+        ('7.0.0', 'liberty'),
+    ]),
+    'cinder-common': OrderedDict([
+        ('7.0.0', 'liberty'),
+    ]),
+    'keystone': OrderedDict([
+        ('8.0.0', 'liberty'),
+    ]),
+    'horizon-common': OrderedDict([
+        ('8.0.0', 'liberty'),
+    ]),
+    'ceilometer-common': OrderedDict([
+        ('5.0.0', 'liberty'),
+    ]),
+    'heat-common': OrderedDict([
+        ('5.0.0', 'liberty'),
+    ]),
+    'glance-common': OrderedDict([
+        ('11.0.0', 'liberty'),
+    ]),
+}
+
 DEFAULT_LOOPBACK_SIZE = '5G'
 
 
@@ -201,20 +229,29 @@ def get_os_codename_package(package, fatal=True):
         error_out(e)
 
     vers = apt.upstream_version(pkg.current_ver.ver_str)
+    match = re.match('^(\d)\.(\d)\.(\d)', vers)
+    if match:
+        vers = match.group(0)
 
-    try:
-        if 'swift' in pkg.name:
-            swift_vers = vers[:5]
-            if swift_vers not in SWIFT_CODENAMES:
-                # Deal with 1.10.0 upward
-                swift_vers = vers[:6]
-            return SWIFT_CODENAMES[swift_vers]
-        else:
-            vers = vers[:6]
-            return OPENSTACK_CODENAMES[vers]
-    except KeyError:
-        e = 'Could not determine OpenStack codename for version %s' % vers
-        error_out(e)
+    # >= Liberty independent project versions
+    if (package in PACKAGE_CODENAMES and
+            vers in PACKAGE_CODENAMES[package]):
+        return PACKAGE_CODENAMES[package][vers]
+    else:
+        # < Liberty co-ordinated project versions
+        try:
+            if 'swift' in pkg.name:
+                swift_vers = vers[:5]
+                if swift_vers not in SWIFT_CODENAMES:
+                    # Deal with 1.10.0 upward
+                    swift_vers = vers[:6]
+                return SWIFT_CODENAMES[swift_vers]
+            else:
+                vers = vers[:6]
+                return OPENSTACK_CODENAMES[vers]
+        except KeyError:
+            e = 'Could not determine OpenStack codename for version %s' % vers
+            error_out(e)
 
 
 def get_os_version_package(pkg, fatal=True):
diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py
index e2769e49..1e57941a 100644
--- a/hooks/charmhelpers/contrib/storage/linux/utils.py
+++ b/hooks/charmhelpers/contrib/storage/linux/utils.py
@@ -43,9 +43,10 @@ def zap_disk(block_device):
 
     :param block_device: str: Full path of block device to clean.
     '''
+    # https://github.com/ceph/ceph/commit/fdd7f8d83afa25c4e09aaedd90ab93f3b64a677b
     # sometimes sgdisk exits non-zero; this is OK, dd will clean up
-    call(['sgdisk', '--zap-all', '--mbrtogpt',
-          '--clear', block_device])
+    call(['sgdisk', '--zap-all', '--', block_device])
+    call(['sgdisk', '--clear', '--mbrtogpt', '--', block_device])
     dev_end = check_output(['blockdev', '--getsz',
                             block_device]).decode('UTF-8')
     gpt_end = int(dev_end.split()[0]) - 100
diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py
index 18860f59..a35d006b 100644
--- a/hooks/charmhelpers/core/hookenv.py
+++ b/hooks/charmhelpers/core/hookenv.py
@@ -34,23 +34,6 @@ import errno
 import tempfile
 from subprocess import CalledProcessError
 
-try:
-    from charmhelpers.cli import cmdline
-except ImportError as e:
-    # due to the anti-pattern of partially synching charmhelpers directly
-    # into charms, it's possible that charmhelpers.cli is not available;
-    # if that's the case, they don't really care about using the cli anyway,
-    # so mock it out
-    if str(e) == 'No module named cli':
-        class cmdline(object):
-            @classmethod
-            def subcommand(cls, *args, **kwargs):
-                def _wrap(func):
-                    return func
-                return _wrap
-    else:
-        raise
-
 import six
 if not six.PY3:
     from UserDict import UserDict
@@ -91,6 +74,7 @@ def cached(func):
         res = func(*args, **kwargs)
         cache[key] = res
         return res
+    wrapper._wrapped = func
     return wrapper
 
 
@@ -190,7 +174,6 @@ def relation_type():
     return os.environ.get('JUJU_RELATION', None)
 
 
-@cmdline.subcommand()
 @cached
 def relation_id(relation_name=None, service_or_unit=None):
     """The relation ID for the current or a specified relation"""
@@ -216,13 +199,11 @@ def remote_unit():
     return os.environ.get('JUJU_REMOTE_UNIT', None)
 
 
-@cmdline.subcommand()
 def service_name():
     """The name service group this unit belongs to"""
     return local_unit().split('/')[0]
 
 
-@cmdline.subcommand()
 @cached
 def remote_service_name(relid=None):
     """The remote service name for a given relation-id (or the current relation)"""
diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py
index 8ae8ef86..ec659eef 100644
--- a/hooks/charmhelpers/core/host.py
+++ b/hooks/charmhelpers/core/host.py
@@ -72,7 +72,7 @@ def service_pause(service_name, init_dir=None):
     stopped = service_stop(service_name)
     # XXX: Support systemd too
     override_path = os.path.join(
-        init_dir, '{}.conf.override'.format(service_name))
+        init_dir, '{}.override'.format(service_name))
     with open(override_path, 'w') as fh:
         fh.write("manual\n")
     return stopped
@@ -86,7 +86,7 @@ def service_resume(service_name, init_dir=None):
     if init_dir is None:
         init_dir = "/etc/init"
     override_path = os.path.join(
-        init_dir, '{}.conf.override'.format(service_name))
+        init_dir, '{}.override'.format(service_name))
     if os.path.exists(override_path):
         os.unlink(override_path)
     started = service_start(service_name)
@@ -148,6 +148,16 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False):
     return user_info
 
 
+def user_exists(username):
+    """Check if a user exists"""
+    try:
+        pwd.getpwnam(username)
+        user_exists = True
+    except KeyError:
+        user_exists = False
+    return user_exists
+
+
 def add_group(group_name, system_group=False):
     """Add a group to the system"""
     try:
@@ -280,6 +290,17 @@ def mounts():
     return system_mounts
 
 
+def fstab_mount(mountpoint):
+    """Mount filesystem using fstab"""
+    cmd_args = ['mount', mountpoint]
+    try:
+        subprocess.check_output(cmd_args)
+    except subprocess.CalledProcessError as e:
+        log('Error unmounting {}\n{}'.format(mountpoint, e.output))
+        return False
+    return True
+
+
 def file_hash(path, hash_type='md5'):
     """
     Generate a hash checksum of the contents of 'path' or None if not found.
diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py
index 8005c415..3f677833 100644
--- a/hooks/charmhelpers/core/services/helpers.py
+++ b/hooks/charmhelpers/core/services/helpers.py
@@ -16,7 +16,9 @@
 
 import os
 import yaml
+
 from charmhelpers.core import hookenv
+from charmhelpers.core import host
 from charmhelpers.core import templating
 
 from charmhelpers.core.services.base import ManagerCallback
@@ -240,27 +242,41 @@ class TemplateCallback(ManagerCallback):
 
     :param str source: The template source file, relative to
         `$CHARM_DIR/templates`
+
     :param str target: The target to write the rendered template to
     :param str owner: The owner of the rendered file
     :param str group: The group of the rendered file
     :param int perms: The permissions of the rendered file
-
+    :param partial on_change_action: functools partial to be executed when
+                                     rendered file changes
     """
     def __init__(self, source, target,
-                 owner='root', group='root', perms=0o444):
+                 owner='root', group='root', perms=0o444,
+                 on_change_action=None):
         self.source = source
         self.target = target
         self.owner = owner
         self.group = group
         self.perms = perms
+        self.on_change_action = on_change_action
 
     def __call__(self, manager, service_name, event_name):
+        pre_checksum = ''
+        if self.on_change_action and os.path.isfile(self.target):
+            pre_checksum = host.file_hash(self.target)
         service = manager.get_service(service_name)
         context = {}
         for ctx in service.get('required_data', []):
             context.update(ctx)
         templating.render(self.source, self.target, context,
                           self.owner, self.group, self.perms)
+        if self.on_change_action:
+            if pre_checksum == host.file_hash(self.target):
+                hookenv.log(
+                    'No change detected: {}'.format(self.target),
+                    hookenv.DEBUG)
+            else:
+                self.on_change_action()
 
 
 # Convenience aliases for templates
diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py
index 0a3bb969..cd0b783c 100644
--- a/hooks/charmhelpers/fetch/__init__.py
+++ b/hooks/charmhelpers/fetch/__init__.py
@@ -90,6 +90,14 @@ CLOUD_ARCHIVE_POCKETS = {
     'kilo/proposed': 'trusty-proposed/kilo',
     'trusty-kilo/proposed': 'trusty-proposed/kilo',
     'trusty-proposed/kilo': 'trusty-proposed/kilo',
+    # Liberty
+    'liberty': 'trusty-updates/liberty',
+    'trusty-liberty': 'trusty-updates/liberty',
+    'trusty-liberty/updates': 'trusty-updates/liberty',
+    'trusty-updates/liberty': 'trusty-updates/liberty',
+    'liberty/proposed': 'trusty-proposed/liberty',
+    'trusty-liberty/proposed': 'trusty-proposed/liberty',
+    'trusty-proposed/liberty': 'trusty-proposed/liberty',
 }
 
 # The order of this list is very important. Handlers should be listed in from
diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py
index 3de26afd..7816c934 100644
--- a/tests/charmhelpers/contrib/amulet/utils.py
+++ b/tests/charmhelpers/contrib/amulet/utils.py
@@ -14,17 +14,23 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
-import amulet
-import ConfigParser
-import distro_info
 import io
+import json
 import logging
 import os
 import re
-import six
+import subprocess
 import sys
 import time
-import urlparse
+
+import amulet
+import distro_info
+import six
+from six.moves import configparser
+if six.PY3:
+    from urllib import parse as urlparse
+else:
+    import urlparse
 
 
 class AmuletUtils(object):
@@ -142,19 +148,23 @@ class AmuletUtils(object):
 
             for service_name in services_list:
                 if (self.ubuntu_releases.index(release) >= systemd_switch or
-                        service_name == "rabbitmq-server"):
-                    # init is systemd
+                        service_name in ['rabbitmq-server', 'apache2']):
+                    # init is systemd (or regular sysv)
                     cmd = 'sudo service {} status'.format(service_name)
+                    output, code = sentry_unit.run(cmd)
+                    service_running = code == 0
                 elif self.ubuntu_releases.index(release) < systemd_switch:
                     # init is upstart
                     cmd = 'sudo status {}'.format(service_name)
+                    output, code = sentry_unit.run(cmd)
+                    service_running = code == 0 and "start/running" in output
 
-                output, code = sentry_unit.run(cmd)
                 self.log.debug('{} `{}` returned '
                                '{}'.format(sentry_unit.info['unit_name'],
                                            cmd, code))
-                if code != 0:
-                    return "command `{}` returned {}".format(cmd, str(code))
+                if not service_running:
+                    return u"command `{}` returned {} {}".format(
+                        cmd, output, str(code))
         return None
 
     def _get_config(self, unit, filename):
@@ -164,7 +174,7 @@ class AmuletUtils(object):
         # NOTE(beisner):  by default, ConfigParser does not handle options
         # with no value, such as the flags used in the mysql my.cnf file.
         # https://bugs.python.org/issue7005
-        config = ConfigParser.ConfigParser(allow_no_value=True)
+        config = configparser.ConfigParser(allow_no_value=True)
         config.readfp(io.StringIO(file_contents))
         return config
 
@@ -450,15 +460,20 @@ class AmuletUtils(object):
                                         cmd, code, output))
         return None
 
-    def get_process_id_list(self, sentry_unit, process_name):
+    def get_process_id_list(self, sentry_unit, process_name,
+                            expect_success=True):
         """Get a list of process ID(s) from a single sentry juju unit
         for a single process name.
 
-        :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+        :param sentry_unit: Amulet sentry instance (juju unit)
         :param process_name: Process name
+        :param expect_success: If False, expect the PID to be missing,
+            raise if it is present.
         :returns: List of process IDs
         """
-        cmd = 'pidof {}'.format(process_name)
+        cmd = 'pidof -x {}'.format(process_name)
+        if not expect_success:
+            cmd += " || exit 0 && exit 1"
         output, code = sentry_unit.run(cmd)
         if code != 0:
             msg = ('{} `{}` returned {} '
@@ -467,14 +482,23 @@ class AmuletUtils(object):
             amulet.raise_status(amulet.FAIL, msg=msg)
         return str(output).split()
 
-    def get_unit_process_ids(self, unit_processes):
+    def get_unit_process_ids(self, unit_processes, expect_success=True):
         """Construct a dict containing unit sentries, process names, and
-        process IDs."""
+        process IDs.
+
+        :param unit_processes: A dictionary of Amulet sentry instance
+            to list of process names.
+        :param expect_success: if False expect the processes to not be
+            running, raise if they are.
+        :returns: Dictionary of Amulet sentry instance to dictionary
+            of process names to PIDs.
+        """
         pid_dict = {}
-        for sentry_unit, process_list in unit_processes.iteritems():
+        for sentry_unit, process_list in six.iteritems(unit_processes):
             pid_dict[sentry_unit] = {}
             for process in process_list:
-                pids = self.get_process_id_list(sentry_unit, process)
+                pids = self.get_process_id_list(
+                    sentry_unit, process, expect_success=expect_success)
                 pid_dict[sentry_unit].update({process: pids})
         return pid_dict
 
@@ -488,7 +512,7 @@ class AmuletUtils(object):
             return ('Unit count mismatch.  expected, actual: {}, '
                     '{} '.format(len(expected), len(actual)))
 
-        for (e_sentry, e_proc_names) in expected.iteritems():
+        for (e_sentry, e_proc_names) in six.iteritems(expected):
             e_sentry_name = e_sentry.info['unit_name']
             if e_sentry in actual.keys():
                 a_proc_names = actual[e_sentry]
@@ -507,11 +531,23 @@ class AmuletUtils(object):
                             '{}'.format(e_proc_name, a_proc_name))
 
                 a_pids_length = len(a_pids)
-                if e_pids_length != a_pids_length:
-                    return ('PID count mismatch. {} ({}) expected, actual: '
+                fail_msg = ('PID count mismatch. {} ({}) expected, actual: '
                             '{}, {} ({})'.format(e_sentry_name, e_proc_name,
                                                  e_pids_length, a_pids_length,
                                                  a_pids))
+
+                # If expected is not bool, ensure PID quantities match
+                if not isinstance(e_pids_length, bool) and \
+                        a_pids_length != e_pids_length:
+                    return fail_msg
+                # If expected is bool True, ensure 1 or more PIDs exist
+                elif isinstance(e_pids_length, bool) and \
+                        e_pids_length is True and a_pids_length < 1:
+                    return fail_msg
+                # If expected is bool False, ensure 0 PIDs exist
+                elif isinstance(e_pids_length, bool) and \
+                        e_pids_length is False and a_pids_length != 0:
+                    return fail_msg
                 else:
                     self.log.debug('PID check OK: {} {} {}: '
                                    '{}'.format(e_sentry_name, e_proc_name,
@@ -531,3 +567,30 @@ class AmuletUtils(object):
             return 'Dicts within list are not identical'
 
         return None
+
+    def run_action(self, unit_sentry, action,
+                   _check_output=subprocess.check_output):
+        """Run the named action on a given unit sentry.
+
+        _check_output parameter is used for dependency injection.
+
+        @return action_id.
+        """
+        unit_id = unit_sentry.info["unit_name"]
+        command = ["juju", "action", "do", "--format=json", unit_id, action]
+        self.log.info("Running command: %s\n" % " ".join(command))
+        output = _check_output(command, universal_newlines=True)
+        data = json.loads(output)
+        action_id = data[u'Action queued with id']
+        return action_id
+
+    def wait_on_action(self, action_id, _check_output=subprocess.check_output):
+        """Wait for a given action, returning if it completed or not.
+
+        _check_output parameter is used for dependency injection.
+        """
+        command = ["juju", "action", "fetch", "--format=json", "--wait=0",
+                   action_id]
+        output = _check_output(command, universal_newlines=True)
+        data = json.loads(output)
+        return data.get(u"status") == "completed"
diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
index b01e6cb8..07ee2ef1 100644
--- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -44,7 +44,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            Determine if the local branch being tested is derived from its
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
-        base_charms = ['mysql', 'mongodb']
+        base_charms = ['mysql', 'mongodb', 'nrpe']
 
         if self.series in ['precise', 'trusty']:
             base_series = self.series
@@ -81,7 +81,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
                       'ceph-osd', 'ceph-radosgw']
         # Most OpenStack subordinate charms do not expose an origin option
         # as that is controlled by the principle.
-        ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch']
+        ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
 
         if self.openstack:
             for svc in services:

From 5e415f774311ae9ceffd0a554b86eeaa559cc8f1 Mon Sep 17 00:00:00 2001
From: Liam Young <liam.young@canonical.com>
Date: Wed, 19 Aug 2015 14:48:47 +0100
Subject: [PATCH 88/88] [gnuoy,trivial] Charmhelper sync (+1'd by mojo)

---
 .../charmhelpers/contrib/openstack/context.py | 27 +++++--
 .../charmhelpers/contrib/openstack/neutron.py | 43 +++++++----
 hooks/charmhelpers/core/host.py               | 77 ++++++++++++++++---
 hooks/charmhelpers/core/hugepage.py           | 62 +++++++++++++++
 4 files changed, 178 insertions(+), 31 deletions(-)
 create mode 100644 hooks/charmhelpers/core/hugepage.py

diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py
index ab2ebac1..9a33a035 100644
--- a/hooks/charmhelpers/contrib/openstack/context.py
+++ b/hooks/charmhelpers/contrib/openstack/context.py
@@ -50,6 +50,8 @@ from charmhelpers.core.sysctl import create as sysctl_create
 from charmhelpers.core.strutils import bool_from_string
 
 from charmhelpers.core.host import (
+    get_bond_master,
+    is_phy_iface,
     list_nics,
     get_nic_hwaddr,
     mkdir,
@@ -923,7 +925,6 @@ class NeutronContext(OSContextGenerator):
 
 
 class NeutronPortContext(OSContextGenerator):
-    NIC_PREFIXES = ['eth', 'bond']
 
     def resolve_ports(self, ports):
         """Resolve NICs not yet bound to bridge(s)
@@ -935,7 +936,18 @@ class NeutronPortContext(OSContextGenerator):
 
         hwaddr_to_nic = {}
         hwaddr_to_ip = {}
-        for nic in list_nics(self.NIC_PREFIXES):
+        for nic in list_nics():
+            # Ignore virtual interfaces (bond masters will be identified from
+            # their slaves)
+            if not is_phy_iface(nic):
+                continue
+
+            _nic = get_bond_master(nic)
+            if _nic:
+                log("Replacing iface '%s' with bond master '%s'" % (nic, _nic),
+                    level=DEBUG)
+                nic = _nic
+
             hwaddr = get_nic_hwaddr(nic)
             hwaddr_to_nic[hwaddr] = nic
             addresses = get_ipv4_addr(nic, fatal=False)
@@ -961,7 +973,8 @@ class NeutronPortContext(OSContextGenerator):
                 # trust it to be the real external network).
                 resolved.append(entry)
 
-        return resolved
+        # Ensure no duplicates
+        return list(set(resolved))
 
 
 class OSConfigFlagContext(OSContextGenerator):
@@ -1280,15 +1293,19 @@ class DataPortContext(NeutronPortContext):
     def __call__(self):
         ports = config('data-port')
         if ports:
+            # Map of {port/mac:bridge}
             portmap = parse_data_port_mappings(ports)
-            ports = portmap.values()
+            ports = portmap.keys()
+            # Resolve provided ports or mac addresses and filter out those
+            # already attached to a bridge.
             resolved = self.resolve_ports(ports)
+            # FIXME: is this necessary?
             normalized = {get_nic_hwaddr(port): port for port in resolved
                           if port not in ports}
             normalized.update({port: port for port in resolved
                                if port in ports})
             if resolved:
-                return {bridge: normalized[port] for bridge, port in
+                return {bridge: normalized[port] for port, bridge in
                         six.iteritems(portmap) if port in normalized.keys()}
 
         return None
diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py
index f7b72352..c3d5c28e 100644
--- a/hooks/charmhelpers/contrib/openstack/neutron.py
+++ b/hooks/charmhelpers/contrib/openstack/neutron.py
@@ -255,17 +255,30 @@ def network_manager():
         return 'neutron'
 
 
-def parse_mappings(mappings):
+def parse_mappings(mappings, key_rvalue=False):
+    """By default mappings are lvalue keyed.
+
+    If key_rvalue is True, the mapping will be reversed to allow multiple
+    configs for the same lvalue.
+    """
     parsed = {}
     if mappings:
         mappings = mappings.split()
         for m in mappings:
             p = m.partition(':')
-            key = p[0].strip()
-            if p[1]:
-                parsed[key] = p[2].strip()
+
+            if key_rvalue:
+                key_index = 2
+                val_index = 0
+                # if there is no rvalue skip to next
+                if not p[1]:
+                    continue
             else:
-                parsed[key] = ''
+                key_index = 0
+                val_index = 2
+
+            key = p[key_index].strip()
+            parsed[key] = p[val_index].strip()
 
     return parsed
 
@@ -283,25 +296,25 @@ def parse_bridge_mappings(mappings):
 def parse_data_port_mappings(mappings, default_bridge='br-data'):
     """Parse data port mappings.
 
-    Mappings must be a space-delimited list of bridge:port mappings.
+    Mappings must be a space-delimited list of port:bridge mappings.
 
-    Returns dict of the form {bridge:port}.
+    Returns dict of the form {port:bridge} where port may be an mac address or
+    interface name.
     """
-    _mappings = parse_mappings(mappings)
+
+    # NOTE(dosaboy): we use rvalue for key to allow multiple values to be
+    # proposed for <port> since it may be a mac address which will differ
+    # across units this allowing first-known-good to be chosen.
+    _mappings = parse_mappings(mappings, key_rvalue=True)
     if not _mappings or list(_mappings.values()) == ['']:
         if not mappings:
             return {}
 
         # For backwards-compatibility we need to support port-only provided in
         # config.
-        _mappings = {default_bridge: mappings.split()[0]}
-
-    bridges = _mappings.keys()
-    ports = _mappings.values()
-    if len(set(bridges)) != len(bridges):
-        raise Exception("It is not allowed to have more than one port "
-                        "configured on the same bridge")
+        _mappings = {mappings.split()[0]: default_bridge}
 
+    ports = _mappings.keys()
     if len(set(ports)) != len(ports):
         raise Exception("It is not allowed to have the same port configured "
                         "on more than one bridge")
diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py
index ec659eef..29e8fee0 100644
--- a/hooks/charmhelpers/core/host.py
+++ b/hooks/charmhelpers/core/host.py
@@ -417,25 +417,80 @@ def pwgen(length=None):
     return(''.join(random_chars))
 
 
-def list_nics(nic_type):
+def is_phy_iface(interface):
+    """Returns True if interface is not virtual, otherwise False."""
+    if interface:
+        sys_net = '/sys/class/net'
+        if os.path.isdir(sys_net):
+            for iface in glob.glob(os.path.join(sys_net, '*')):
+                if '/virtual/' in os.path.realpath(iface):
+                    continue
+
+                if interface == os.path.basename(iface):
+                    return True
+
+    return False
+
+
+def get_bond_master(interface):
+    """Returns bond master if interface is bond slave otherwise None.
+
+    NOTE: the provided interface is expected to be physical
+    """
+    if interface:
+        iface_path = '/sys/class/net/%s' % (interface)
+        if os.path.exists(iface_path):
+            if '/virtual/' in os.path.realpath(iface_path):
+                return None
+
+            master = os.path.join(iface_path, 'master')
+            if os.path.exists(master):
+                master = os.path.realpath(master)
+                # make sure it is a bond master
+                if os.path.exists(os.path.join(master, 'bonding')):
+                    return os.path.basename(master)
+
+    return None
+
+
+def list_nics(nic_type=None):
     '''Return a list of nics of given type(s)'''
     if isinstance(nic_type, six.string_types):
         int_types = [nic_type]
     else:
         int_types = nic_type
+
     interfaces = []
-    for int_type in int_types:
-        cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
+    if nic_type:
+        for int_type in int_types:
+            cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
+            ip_output = subprocess.check_output(cmd).decode('UTF-8')
+            ip_output = ip_output.split('\n')
+            ip_output = (line for line in ip_output if line)
+            for line in ip_output:
+                if line.split()[1].startswith(int_type):
+                    matched = re.search('.*: (' + int_type +
+                                        r'[0-9]+\.[0-9]+)@.*', line)
+                    if matched:
+                        iface = matched.groups()[0]
+                    else:
+                        iface = line.split()[1].replace(":", "")
+
+                    if iface not in interfaces:
+                        interfaces.append(iface)
+    else:
+        cmd = ['ip', 'a']
         ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
-        ip_output = (line for line in ip_output if line)
+        ip_output = (line.strip() for line in ip_output if line)
+
+        key = re.compile('^[0-9]+:\s+(.+):')
         for line in ip_output:
-            if line.split()[1].startswith(int_type):
-                matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line)
-                if matched:
-                    interface = matched.groups()[0]
-                else:
-                    interface = line.split()[1].replace(":", "")
-                interfaces.append(interface)
+            matched = re.search(key, line)
+            if matched:
+                iface = matched.group(1)
+                iface = iface.partition("@")[0]
+                if iface not in interfaces:
+                    interfaces.append(iface)
 
     return interfaces
 
diff --git a/hooks/charmhelpers/core/hugepage.py b/hooks/charmhelpers/core/hugepage.py
new file mode 100644
index 00000000..ba4340ff
--- /dev/null
+++ b/hooks/charmhelpers/core/hugepage.py
@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import yaml
+from charmhelpers.core import fstab
+from charmhelpers.core import sysctl
+from charmhelpers.core.host import (
+    add_group,
+    add_user_to_group,
+    fstab_mount,
+    mkdir,
+)
+
+
+def hugepage_support(user, group='hugetlb', nr_hugepages=256,
+                     max_map_count=65536, mnt_point='/run/hugepages/kvm',
+                     pagesize='2MB', mount=True):
+    """Enable hugepages on system.
+
+    Args:
+    user (str)  -- Username to allow access to hugepages to
+    group (str) -- Group name to own hugepages
+    nr_hugepages (int) -- Number of pages to reserve
+    max_map_count (int) -- Number of Virtual Memory Areas a process can own
+    mnt_point (str) -- Directory to mount hugepages on
+    pagesize (str) -- Size of hugepages
+    mount (bool) -- Whether to Mount hugepages
+    """
+    group_info = add_group(group)
+    gid = group_info.gr_gid
+    add_user_to_group(user, group)
+    sysctl_settings = {
+        'vm.nr_hugepages': nr_hugepages,
+        'vm.max_map_count': max_map_count,
+        'vm.hugetlb_shm_group': gid,
+    }
+    sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf')
+    mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False)
+    lfstab = fstab.Fstab()
+    fstab_entry = lfstab.get_entry_by_attr('mountpoint', mnt_point)
+    if fstab_entry:
+        lfstab.remove_entry(fstab_entry)
+    entry = lfstab.Entry('nodev', mnt_point, 'hugetlbfs',
+                         'mode=1770,gid={},pagesize={}'.format(gid, pagesize), 0, 0)
+    lfstab.add_entry(entry)
+    if mount:
+        fstab_mount(mnt_point)