Merge from lp:~openstack-charmers/charms/trusty/.../next/

Makefile

@@ -2,12 +2,18 @@
 PYTHON := /usr/bin/env python
 
 lint:
-	@flake8 --exclude hooks/charmhelpers actions hooks unit_tests tests
+	@flake8 --exclude hooks/charmhelpers,tests/charmhelpers \
+        actions hooks unit_tests tests
 	@charm proof
 
-unit_test:
-	@echo Starting tests...
-	@$(PYTHON) /usr/bin/nosetests --nologcapture unit_tests
+test:
+	@# Bundletester expects unit tests here.
+	@echo Starting unit tests...
+	@$(PYTHON) /usr/bin/nosetests -v --nologcapture --with-coverage unit_tests
+
+functional_test:
+	@echo Starting Amulet tests...
+	@juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700
 
 bin/charm_helpers_sync.py:
 	@mkdir -p bin
@@ -18,13 +24,6 @@ sync: bin/charm_helpers_sync.py
 	@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-hooks.yaml
 	@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-tests.yaml
 
-test:
-	@echo Starting Amulet tests...
-	# coreycb note: The -v should only be temporary until Amulet sends
-	# raise_status() messages to stderr:
-	#   https://bugs.launchpad.net/amulet/+bug/1320357
-	@juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700
-
-publish: lint unit_test
+publish: lint test
 	bzr push lp:charms/neutron-api
 	bzr push lp:charms/trusty/neutron-api

actions.yaml

@@ -1,2 +1,4 @@
 git-reinstall:
     description: Reinstall neutron-api from the openstack-origin-git repositories.
+openstack-upgrade:
+  description: Perform openstack upgrades. Config option action-managed-upgrade must be set to True.

actions/openstack-upgrade

@@ -0,0 +1 @@
+openstack_upgrade.py

actions/openstack_upgrade.py

@@ -0,0 +1,34 @@
+#!/usr/bin/python
+import sys
+
+sys.path.append('hooks/')
+
+from charmhelpers.contrib.openstack.utils import (
+    do_action_openstack_upgrade,
+)
+
+from neutron_api_hooks import (
+    config_changed,
+    CONFIGS,
+)
+
+from neutron_api_utils import (
+    do_openstack_upgrade,
+)
+
+
+def openstack_upgrade():
+    """Upgrade packages to config-set Openstack version.
+
+    If the charm was installed from source we cannot upgrade it.
+    For backwards compatibility a config flag must be set for this
+    code to run, otherwise a full service level upgrade will fire
+    on config-changed."""
+
+    if (do_action_openstack_upgrade('neutron-common',
+                                    do_openstack_upgrade,
+                                    CONFIGS)):
+        config_changed()
+
+if __name__ == '__main__':
+    openstack_upgrade()

config.yaml

@@ -100,6 +100,7 @@ options:
         ovs - OpenvSwitch Plugin
         vsp - Nuage Networks VSP
         nsx - VMWare NSX
+        Calico - Project Calico Networking
       .
   overlay-network-type:
     default: gre
@@ -110,6 +111,7 @@ options:
         gre
         vxlan
       .
+      Multiple types can be provided - field is space delimited.
   flat-network-providers:
     type: string
     default:
@@ -402,6 +404,20 @@ options:
     description: |
       A comma-separated list of nagios servicegroups.
       If left empty, the nagios_context will be used as the servicegroup
+  # PLUMgrid Plugin configuration
+  plumgrid-username:
+    default: plumgrid
+    type: string
+    description: Username to access PLUMgrid Director
+  plumgrid-password:
+    default: plumgrid
+    type: string
+    description: Password to access PLUMgrid Director
+  plumgrid-virtual-ip:
+    default:
+    type: string
+    description: IP address of PLUMgrid Director
+  # end of PLUMgrid configuration
   manage-neutron-plugin-legacy-mode:
     type: boolean
     default: True
@@ -418,3 +434,22 @@ options:
     default:
     description: Optional source for archive containing additional packages.
 
+  # Calico plugin configuration
+  calico-origin:
+    default:
+    type: string
+    description: |
+      Repository from which to install Calico packages. If set, must be
+      a PPA URL, of the form ppa:somecustom/ppa. Changing this value
+      after installation will force an immediate software upgrade.
+  # End of Calico plugin configuration
+  action-managed-upgrade:
+    type: boolean
+    default: False
+    description: |
+      If True enables openstack upgrades for this charm via juju actions.
+      You will still need to set openstack-origin to the new repository but
+      instead of an upgrade running automatically across all units, it will
+      wait for you to execute the openstack-upgrade action for this charm on
+      each unit. If False it will revert to existing behavior of upgrading
+      all units on config change.

hooks/charmhelpers/contrib/network/ip.py

@@ -23,7 +23,7 @@ import socket
 from functools import partial
 
 from charmhelpers.core.hookenv import unit_get
-from charmhelpers.fetch import apt_install
+from charmhelpers.fetch import apt_install, apt_update
 from charmhelpers.core.hookenv import (
     log,
     WARNING,
@@ -32,13 +32,15 @@ from charmhelpers.core.hookenv import (
 try:
     import netifaces
 except ImportError:
-    apt_install('python-netifaces')
+    apt_update(fatal=True)
+    apt_install('python-netifaces', fatal=True)
     import netifaces
 
 try:
     import netaddr
 except ImportError:
-    apt_install('python-netaddr')
+    apt_update(fatal=True)
+    apt_install('python-netaddr', fatal=True)
     import netaddr
 
 
@@ -435,8 +437,12 @@ def get_hostname(address, fqdn=True):
 
         rev = dns.reversename.from_address(address)
         result = ns_query(rev)
+
         if not result:
-            return None
+            try:
+                result = socket.gethostbyaddr(address)[0]
+            except:
+                return None
     else:
         result = address
 

hooks/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import re
 import six
 from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
@@ -44,20 +45,31 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            Determine if the local branch being tested is derived from its
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
+
+        # Charms outside the lp:~openstack-charmers namespace
         base_charms = ['mysql', 'mongodb', 'nrpe']
 
+        # Force these charms to current series even when using an older series.
+        # ie. Use trusty/nrpe even when series is precise, as the P charm
+        # does not possess the necessary external master config and hooks.
+        force_series_current = ['nrpe']
+
         if self.series in ['precise', 'trusty']:
             base_series = self.series
         else:
             base_series = self.current_next
 
-        if self.stable:
-            for svc in other_services:
+        for svc in other_services:
+            if svc['name'] in force_series_current:
+                base_series = self.current_next
+            # If a location has been explicitly set, use it
+            if svc.get('location'):
+                continue
+            if self.stable:
                 temp = 'lp:charms/{}/{}'
                 svc['location'] = temp.format(base_series,
                                               svc['name'])
-        else:
-            for svc in other_services:
+            else:
                 if svc['name'] in base_charms:
                     temp = 'lp:charms/{}/{}'
                     svc['location'] = temp.format(base_series,
@@ -66,6 +78,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
                     temp = 'lp:~openstack-charmers/charms/{}/{}/next'
                     svc['location'] = temp.format(self.current_next,
                                                   svc['name'])
+
         return other_services
 
     def _add_services(self, this_service, other_services):
@@ -77,21 +90,23 @@ class OpenStackAmuletDeployment(AmuletDeployment):
 
         services = other_services
         services.append(this_service)
+
+        # Charms which should use the source config option
         use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
                       'ceph-osd', 'ceph-radosgw']
-        # Most OpenStack subordinate charms do not expose an origin option
-        # as that is controlled by the principle.
-        ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
+
+        # Charms which can not use openstack-origin, ie. many subordinates
+        no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
 
         if self.openstack:
             for svc in services:
-                if svc['name'] not in use_source + ignore:
+                if svc['name'] not in use_source + no_origin:
                     config = {'openstack-origin': self.openstack}
                     self.d.configure(svc['name'], config)
 
         if self.source:
             for svc in services:
-                if svc['name'] in use_source and svc['name'] not in ignore:
+                if svc['name'] in use_source and svc['name'] not in no_origin:
                     config = {'source': self.source}
                     self.d.configure(svc['name'], config)
 
@@ -100,6 +115,45 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         for service, config in six.iteritems(configs):
             self.d.configure(service, config)
 
+    def _auto_wait_for_status(self, message=None, exclude_services=None,
+                              timeout=1800):
+        """Wait for all units to have a specific extended status, except
+        for any defined as excluded.  Unless specified via message, any
+        status containing any case of 'ready' will be considered a match.
+
+        Examples of message usage:
+
+          Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
+              message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
+
+          Wait for all units to reach this status (exact match):
+              message = 'Unit is ready'
+
+          Wait for all units to reach any one of these (exact match):
+              message = re.compile('Unit is ready|OK|Ready')
+
+          Wait for at least one unit to reach this status (exact match):
+              message = {'ready'}
+
+        See Amulet's sentry.wait_for_messages() for message usage detail.
+        https://github.com/juju/amulet/blob/master/amulet/sentry.py
+
+        :param message: Expected status match
+        :param exclude_services: List of juju service names to ignore
+        :param timeout: Maximum time in seconds to wait for status match
+        :returns: None.  Raises if timeout is hit.
+        """
+
+        if not message:
+            message = re.compile('.*ready.*', re.IGNORECASE)
+
+        if not exclude_services:
+            exclude_services = []
+
+        services = list(set(self.d.services.keys()) - set(exclude_services))
+        service_messages = {service: message for service in services}
+        self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+
     def _get_openstack_release(self):
         """Get openstack release.
 

hooks/charmhelpers/contrib/openstack/amulet/utils.py

@@ -27,6 +27,7 @@ import glanceclient.v1.client as glance_client
 import heatclient.v1.client as heat_client
 import keystoneclient.v2_0 as keystone_client
 import novaclient.v1_1.client as nova_client
+import pika
 import swiftclient
 
 from charmhelpers.contrib.amulet.utils import (
@@ -602,3 +603,361 @@ class OpenStackAmuletUtils(AmuletUtils):
             self.log.debug('Ceph {} samples (OK): '
                            '{}'.format(sample_type, samples))
             return None
+
+# rabbitmq/amqp specific helpers:
+    def add_rmq_test_user(self, sentry_units,
+                          username="testuser1", password="changeme"):
+        """Add a test user via the first rmq juju unit, check connection as
+        the new user against all sentry units.
+
+        :param sentry_units: list of sentry unit pointers
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug('Adding rmq user ({})...'.format(username))
+
+        # Check that user does not already exist
+        cmd_user_list = 'rabbitmqctl list_users'
+        output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
+        if username in output:
+            self.log.warning('User ({}) already exists, returning '
+                             'gracefully.'.format(username))
+            return
+
+        perms = '".*" ".*" ".*"'
+        cmds = ['rabbitmqctl add_user {} {}'.format(username, password),
+                'rabbitmqctl set_permissions {} {}'.format(username, perms)]
+
+        # Add user via first unit
+        for cmd in cmds:
+            output, _ = self.run_cmd_unit(sentry_units[0], cmd)
+
+        # Check connection against the other sentry_units
+        self.log.debug('Checking user connect against units...')
+        for sentry_unit in sentry_units:
+            connection = self.connect_amqp_by_unit(sentry_unit, ssl=False,
+                                                   username=username,
+                                                   password=password)
+            connection.close()
+
+    def delete_rmq_test_user(self, sentry_units, username="testuser1"):
+        """Delete a rabbitmq user via the first rmq juju unit.
+
+        :param sentry_units: list of sentry unit pointers
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :returns: None if successful or no such user.
+        """
+        self.log.debug('Deleting rmq user ({})...'.format(username))
+
+        # Check that the user exists
+        cmd_user_list = 'rabbitmqctl list_users'
+        output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
+
+        if username not in output:
+            self.log.warning('User ({}) does not exist, returning '
+                             'gracefully.'.format(username))
+            return
+
+        # Delete the user
+        cmd_user_del = 'rabbitmqctl delete_user {}'.format(username)
+        output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_del)
+
+    def get_rmq_cluster_status(self, sentry_unit):
+        """Execute rabbitmq cluster status command on a unit and return
+        the full output.
+
+        :param unit: sentry unit
+        :returns: String containing console output of cluster status command
+        """
+        cmd = 'rabbitmqctl cluster_status'
+        output, _ = self.run_cmd_unit(sentry_unit, cmd)
+        self.log.debug('{} cluster_status:\n{}'.format(
+            sentry_unit.info['unit_name'], output))
+        return str(output)
+
+    def get_rmq_cluster_running_nodes(self, sentry_unit):
+        """Parse rabbitmqctl cluster_status output string, return list of
+        running rabbitmq cluster nodes.
+
+        :param unit: sentry unit
+        :returns: List containing node names of running nodes
+        """
+        # NOTE(beisner): rabbitmqctl cluster_status output is not
+        # json-parsable, do string chop foo, then json.loads that.
+        str_stat = self.get_rmq_cluster_status(sentry_unit)
+        if 'running_nodes' in str_stat:
+            pos_start = str_stat.find("{running_nodes,") + 15
+            pos_end = str_stat.find("]},", pos_start) + 1
+            str_run_nodes = str_stat[pos_start:pos_end].replace("'", '"')
+            run_nodes = json.loads(str_run_nodes)
+            return run_nodes
+        else:
+            return []
+
+    def validate_rmq_cluster_running_nodes(self, sentry_units):
+        """Check that all rmq unit hostnames are represented in the
+        cluster_status output of all units.
+
+        :param host_names: dict of juju unit names to host names
+        :param units: list of sentry unit pointers (all rmq units)
+        :returns: None if successful, otherwise return error message
+        """
+        host_names = self.get_unit_hostnames(sentry_units)
+        errors = []
+
+        # Query every unit for cluster_status running nodes
+        for query_unit in sentry_units:
+            query_unit_name = query_unit.info['unit_name']
+            running_nodes = self.get_rmq_cluster_running_nodes(query_unit)
+
+            # Confirm that every unit is represented in the queried unit's
+            # cluster_status running nodes output.
+            for validate_unit in sentry_units:
+                val_host_name = host_names[validate_unit.info['unit_name']]
+                val_node_name = 'rabbit@{}'.format(val_host_name)
+
+                if val_node_name not in running_nodes:
+                    errors.append('Cluster member check failed on {}: {} not '
+                                  'in {}\n'.format(query_unit_name,
+                                                   val_node_name,
+                                                   running_nodes))
+        if errors:
+            return ''.join(errors)
+
+    def rmq_ssl_is_enabled_on_unit(self, sentry_unit, port=None):
+        """Check a single juju rmq unit for ssl and port in the config file."""
+        host = sentry_unit.info['public-address']
+        unit_name = sentry_unit.info['unit_name']
+
+        conf_file = '/etc/rabbitmq/rabbitmq.config'
+        conf_contents = str(self.file_contents_safe(sentry_unit,
+                                                    conf_file, max_wait=16))
+        # Checks
+        conf_ssl = 'ssl' in conf_contents
+        conf_port = str(port) in conf_contents
+
+        # Port explicitly checked in config
+        if port and conf_port and conf_ssl:
+            self.log.debug('SSL is enabled  @{}:{} '
+                           '({})'.format(host, port, unit_name))
+            return True
+        elif port and not conf_port and conf_ssl:
+            self.log.debug('SSL is enabled @{} but not on port {} '
+                           '({})'.format(host, port, unit_name))
+            return False
+        # Port not checked (useful when checking that ssl is disabled)
+        elif not port and conf_ssl:
+            self.log.debug('SSL is enabled  @{}:{} '
+                           '({})'.format(host, port, unit_name))
+            return True
+        elif not conf_ssl:
+            self.log.debug('SSL not enabled @{}:{} '
+                           '({})'.format(host, port, unit_name))
+            return False
+        else:
+            msg = ('Unknown condition when checking SSL status @{}:{} '
+                   '({})'.format(host, port, unit_name))
+            amulet.raise_status(amulet.FAIL, msg)
+
+    def validate_rmq_ssl_enabled_units(self, sentry_units, port=None):
+        """Check that ssl is enabled on rmq juju sentry units.
+
+        :param sentry_units: list of all rmq sentry units
+        :param port: optional ssl port override to validate
+        :returns: None if successful, otherwise return error message
+        """
+        for sentry_unit in sentry_units:
+            if not self.rmq_ssl_is_enabled_on_unit(sentry_unit, port=port):
+                return ('Unexpected condition:  ssl is disabled on unit '
+                        '({})'.format(sentry_unit.info['unit_name']))
+        return None
+
+    def validate_rmq_ssl_disabled_units(self, sentry_units):
+        """Check that ssl is enabled on listed rmq juju sentry units.
+
+        :param sentry_units: list of all rmq sentry units
+        :returns: True if successful.  Raise on error.
+        """
+        for sentry_unit in sentry_units:
+            if self.rmq_ssl_is_enabled_on_unit(sentry_unit):
+                return ('Unexpected condition:  ssl is enabled on unit '
+                        '({})'.format(sentry_unit.info['unit_name']))
+        return None
+
+    def configure_rmq_ssl_on(self, sentry_units, deployment,
+                             port=None, max_wait=60):
+        """Turn ssl charm config option on, with optional non-default
+        ssl port specification.  Confirm that it is enabled on every
+        unit.
+
+        :param sentry_units: list of sentry units
+        :param deployment: amulet deployment object pointer
+        :param port: amqp port, use defaults if None
+        :param max_wait: maximum time to wait in seconds to confirm
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug('Setting ssl charm config option:  on')
+
+        # Enable RMQ SSL
+        config = {'ssl': 'on'}
+        if port:
+            config['ssl_port'] = port
+
+        deployment.configure('rabbitmq-server', config)
+
+        # Confirm
+        tries = 0
+        ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
+        while ret and tries < (max_wait / 4):
+            time.sleep(4)
+            self.log.debug('Attempt {}: {}'.format(tries, ret))
+            ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
+            tries += 1
+
+        if ret:
+            amulet.raise_status(amulet.FAIL, ret)
+
+    def configure_rmq_ssl_off(self, sentry_units, deployment, max_wait=60):
+        """Turn ssl charm config option off, confirm that it is disabled
+        on every unit.
+
+        :param sentry_units: list of sentry units
+        :param deployment: amulet deployment object pointer
+        :param max_wait: maximum time to wait in seconds to confirm
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug('Setting ssl charm config option:  off')
+
+        # Disable RMQ SSL
+        config = {'ssl': 'off'}
+        deployment.configure('rabbitmq-server', config)
+
+        # Confirm
+        tries = 0
+        ret = self.validate_rmq_ssl_disabled_units(sentry_units)
+        while ret and tries < (max_wait / 4):
+            time.sleep(4)
+            self.log.debug('Attempt {}: {}'.format(tries, ret))
+            ret = self.validate_rmq_ssl_disabled_units(sentry_units)
+            tries += 1
+
+        if ret:
+            amulet.raise_status(amulet.FAIL, ret)
+
+    def connect_amqp_by_unit(self, sentry_unit, ssl=False,
+                             port=None, fatal=True,
+                             username="testuser1", password="changeme"):
+        """Establish and return a pika amqp connection to the rabbitmq service
+        running on a rmq juju unit.
+
+        :param sentry_unit: sentry unit pointer
+        :param ssl: boolean, default to False
+        :param port: amqp port, use defaults if None
+        :param fatal: boolean, default to True (raises on connect error)
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :returns: pika amqp connection pointer or None if failed and non-fatal
+        """
+        host = sentry_unit.info['public-address']
+        unit_name = sentry_unit.info['unit_name']
+
+        # Default port logic if port is not specified
+        if ssl and not port:
+            port = 5671
+        elif not ssl and not port:
+            port = 5672
+
+        self.log.debug('Connecting to amqp on {}:{} ({}) as '
+                       '{}...'.format(host, port, unit_name, username))
+
+        try:
+            credentials = pika.PlainCredentials(username, password)
+            parameters = pika.ConnectionParameters(host=host, port=port,
+                                                   credentials=credentials,
+                                                   ssl=ssl,
+                                                   connection_attempts=3,
+                                                   retry_delay=5,
+                                                   socket_timeout=1)
+            connection = pika.BlockingConnection(parameters)
+            assert connection.server_properties['product'] == 'RabbitMQ'
+            self.log.debug('Connect OK')
+            return connection
+        except Exception as e:
+            msg = ('amqp connection failed to {}:{} as '
+                   '{} ({})'.format(host, port, username, str(e)))
+            if fatal:
+                amulet.raise_status(amulet.FAIL, msg)
+            else:
+                self.log.warn(msg)
+                return None
+
+    def publish_amqp_message_by_unit(self, sentry_unit, message,
+                                     queue="test", ssl=False,
+                                     username="testuser1",
+                                     password="changeme",
+                                     port=None):
+        """Publish an amqp message to a rmq juju unit.
+
+        :param sentry_unit: sentry unit pointer
+        :param message: amqp message string
+        :param queue: message queue, default to test
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :param ssl: boolean, default to False
+        :param port: amqp port, use defaults if None
+        :returns: None.  Raises exception if publish failed.
+        """
+        self.log.debug('Publishing message to {} queue:\n{}'.format(queue,
+                                                                    message))
+        connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
+                                               port=port,
+                                               username=username,
+                                               password=password)
+
+        # NOTE(beisner): extra debug here re: pika hang potential:
+        #   https://github.com/pika/pika/issues/297
+        #   https://groups.google.com/forum/#!topic/rabbitmq-users/Ja0iyfF0Szw
+        self.log.debug('Defining channel...')
+        channel = connection.channel()
+        self.log.debug('Declaring queue...')
+        channel.queue_declare(queue=queue, auto_delete=False, durable=True)
+        self.log.debug('Publishing message...')
+        channel.basic_publish(exchange='', routing_key=queue, body=message)
+        self.log.debug('Closing channel...')
+        channel.close()
+        self.log.debug('Closing connection...')
+        connection.close()
+
+    def get_amqp_message_by_unit(self, sentry_unit, queue="test",
+                                 username="testuser1",
+                                 password="changeme",
+                                 ssl=False, port=None):
+        """Get an amqp message from a rmq juju unit.
+
+        :param sentry_unit: sentry unit pointer
+        :param queue: message queue, default to test
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :param ssl: boolean, default to False
+        :param port: amqp port, use defaults if None
+        :returns: amqp message body as string.  Raise if get fails.
+        """
+        connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
+                                               port=port,
+                                               username=username,
+                                               password=password)
+        channel = connection.channel()
+        method_frame, _, body = channel.basic_get(queue)
+
+        if method_frame:
+            self.log.debug('Retreived message from {} queue:\n{}'.format(queue,
+                                                                         body))
+            channel.basic_ack(method_frame.delivery_tag)
+            channel.close()
+            connection.close()
+            return body
+        else:
+            msg = 'No message retrieved.'
+            amulet.raise_status(amulet.FAIL, msg)

hooks/charmhelpers/contrib/openstack/context.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import glob
 import json
 import os
 import re
@@ -194,10 +195,50 @@ def config_flags_parser(config_flags):
 class OSContextGenerator(object):
     """Base class for all context generators."""
     interfaces = []
+    related = False
+    complete = False
+    missing_data = []
 
     def __call__(self):
         raise NotImplementedError
 
+    def context_complete(self, ctxt):
+        """Check for missing data for the required context data.
+        Set self.missing_data if it exists and return False.
+        Set self.complete if no missing data and return True.
+        """
+        # Fresh start
+        self.complete = False
+        self.missing_data = []
+        for k, v in six.iteritems(ctxt):
+            if v is None or v == '':
+                if k not in self.missing_data:
+                    self.missing_data.append(k)
+
+        if self.missing_data:
+            self.complete = False
+            log('Missing required data: %s' % ' '.join(self.missing_data), level=INFO)
+        else:
+            self.complete = True
+        return self.complete
+
+    def get_related(self):
+        """Check if any of the context interfaces have relation ids.
+        Set self.related and return True if one of the interfaces
+        has relation ids.
+        """
+        # Fresh start
+        self.related = False
+        try:
+            for interface in self.interfaces:
+                if relation_ids(interface):
+                    self.related = True
+            return self.related
+        except AttributeError as e:
+            log("{} {}"
+                "".format(self, e), 'INFO')
+            return self.related
+
 
 class SharedDBContext(OSContextGenerator):
     interfaces = ['shared-db']
@@ -213,6 +254,7 @@ class SharedDBContext(OSContextGenerator):
         self.database = database
         self.user = user
         self.ssl_dir = ssl_dir
+        self.rel_name = self.interfaces[0]
 
     def __call__(self):
         self.database = self.database or config('database')
@@ -246,6 +288,7 @@ class SharedDBContext(OSContextGenerator):
             password_setting = self.relation_prefix + '_password'
 
         for rid in relation_ids(self.interfaces[0]):
+            self.related = True
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 host = rdata.get('db_host')
@@ -257,7 +300,7 @@ class SharedDBContext(OSContextGenerator):
                     'database_password': rdata.get(password_setting),
                     'database_type': 'mysql'
                 }
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     db_ssl(rdata, ctxt, self.ssl_dir)
                     return ctxt
         return {}
@@ -278,6 +321,7 @@ class PostgresqlDBContext(OSContextGenerator):
 
         ctxt = {}
         for rid in relation_ids(self.interfaces[0]):
+            self.related = True
             for unit in related_units(rid):
                 rel_host = relation_get('host', rid=rid, unit=unit)
                 rel_user = relation_get('user', rid=rid, unit=unit)
@@ -287,7 +331,7 @@ class PostgresqlDBContext(OSContextGenerator):
                         'database_user': rel_user,
                         'database_password': rel_passwd,
                         'database_type': 'postgresql'}
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     return ctxt
 
         return {}
@@ -348,6 +392,7 @@ class IdentityServiceContext(OSContextGenerator):
             ctxt['signing_dir'] = cachedir
 
         for rid in relation_ids(self.rel_name):
+            self.related = True
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 serv_host = rdata.get('service_host')
@@ -366,7 +411,7 @@ class IdentityServiceContext(OSContextGenerator):
                              'service_protocol': svc_protocol,
                              'auth_protocol': auth_protocol})
 
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     # NOTE(jamespage) this is required for >= icehouse
                     # so a missing value just indicates keystone needs
                     # upgrading
@@ -405,6 +450,7 @@ class AMQPContext(OSContextGenerator):
         ctxt = {}
         for rid in relation_ids(self.rel_name):
             ha_vip_only = False
+            self.related = True
             for unit in related_units(rid):
                 if relation_get('clustered', rid=rid, unit=unit):
                     ctxt['clustered'] = True
@@ -437,7 +483,7 @@ class AMQPContext(OSContextGenerator):
                 ha_vip_only = relation_get('ha-vip-only',
                                            rid=rid, unit=unit) is not None
 
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     if 'rabbit_ssl_ca' in ctxt:
                         if not self.ssl_dir:
                             log("Charm not setup for ssl support but ssl ca "
@@ -469,7 +515,7 @@ class AMQPContext(OSContextGenerator):
             ctxt['oslo_messaging_flags'] = config_flags_parser(
                 oslo_messaging_flags)
 
-        if not context_complete(ctxt):
+        if not self.complete:
             return {}
 
         return ctxt
@@ -485,13 +531,15 @@ class CephContext(OSContextGenerator):
 
         log('Generating template context for ceph', level=DEBUG)
         mon_hosts = []
-        auth = None
-        key = None
-        use_syslog = str(config('use-syslog')).lower()
+        ctxt = {
+            'use_syslog': str(config('use-syslog')).lower()
+        }
         for rid in relation_ids('ceph'):
             for unit in related_units(rid):
-                auth = relation_get('auth', rid=rid, unit=unit)
-                key = relation_get('key', rid=rid, unit=unit)
+                if not ctxt.get('auth'):
+                    ctxt['auth'] = relation_get('auth', rid=rid, unit=unit)
+                if not ctxt.get('key'):
+                    ctxt['key'] = relation_get('key', rid=rid, unit=unit)
                 ceph_pub_addr = relation_get('ceph-public-address', rid=rid,
                                              unit=unit)
                 unit_priv_addr = relation_get('private-address', rid=rid,
@@ -500,15 +548,12 @@ class CephContext(OSContextGenerator):
                 ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr
                 mon_hosts.append(ceph_addr)
 
-        ctxt = {'mon_hosts': ' '.join(sorted(mon_hosts)),
-                'auth': auth,
-                'key': key,
-                'use_syslog': use_syslog}
+        ctxt['mon_hosts'] = ' '.join(sorted(mon_hosts))
 
         if not os.path.isdir('/etc/ceph'):
             os.mkdir('/etc/ceph')
 
-        if not context_complete(ctxt):
+        if not self.context_complete(ctxt):
             return {}
 
         ensure_packages(['ceph-common'])
@@ -895,6 +940,31 @@ class NeutronContext(OSContextGenerator):
                 'neutron_url': '%s://%s:%s' % (proto, host, '9696')}
         return ctxt
 
+    def pg_ctxt(self):
+        driver = neutron_plugin_attribute(self.plugin, 'driver',
+                                          self.network_manager)
+        config = neutron_plugin_attribute(self.plugin, 'config',
+                                          self.network_manager)
+        ovs_ctxt = {'core_plugin': driver,
+                    'neutron_plugin': 'plumgrid',
+                    'neutron_security_groups': self.neutron_security_groups,
+                    'local_ip': unit_private_ip(),
+                    'config': config}
+        return ovs_ctxt
+
+    def midonet_ctxt(self):
+        driver = neutron_plugin_attribute(self.plugin, 'driver',
+                                          self.network_manager)
+        midonet_config = neutron_plugin_attribute(self.plugin, 'config',
+                                                  self.network_manager)
+        mido_ctxt = {'core_plugin': driver,
+                     'neutron_plugin': 'midonet',
+                     'neutron_security_groups': self.neutron_security_groups,
+                     'local_ip': unit_private_ip(),
+                     'config': midonet_config}
+
+        return mido_ctxt
+
     def __call__(self):
         if self.network_manager not in ['quantum', 'neutron']:
             return {}
@@ -914,6 +984,10 @@ class NeutronContext(OSContextGenerator):
             ctxt.update(self.calico_ctxt())
         elif self.plugin == 'vsp':
             ctxt.update(self.nuage_ctxt())
+        elif self.plugin == 'plumgrid':
+            ctxt.update(self.pg_ctxt())
+        elif self.plugin == 'midonet':
+            ctxt.update(self.midonet_ctxt())
 
         alchemy_flags = config('neutron-alchemy-flags')
         if alchemy_flags:
@@ -1046,7 +1120,7 @@ class SubordinateConfigContext(OSContextGenerator):
 
         ctxt = {
             ... other context ...
-            'subordinate_config': {
+            'subordinate_configuration': {
                 'DEFAULT': {
                     'key1': 'value1',
                 },
@@ -1087,22 +1161,23 @@ class SubordinateConfigContext(OSContextGenerator):
                     try:
                         sub_config = json.loads(sub_config)
                     except:
-                        log('Could not parse JSON from subordinate_config '
-                            'setting from %s' % rid, level=ERROR)
+                        log('Could not parse JSON from '
+                            'subordinate_configuration setting from %s'
+                            % rid, level=ERROR)
                         continue
 
                     for service in self.services:
                         if service not in sub_config:
-                            log('Found subordinate_config on %s but it contained'
-                                'nothing for %s service' % (rid, service),
-                                level=INFO)
+                            log('Found subordinate_configuration on %s but it '
+                                'contained nothing for %s service'
+                                % (rid, service), level=INFO)
                             continue
 
                         sub_config = sub_config[service]
                         if self.config_file not in sub_config:
-                            log('Found subordinate_config on %s but it contained'
-                                'nothing for %s' % (rid, self.config_file),
-                                level=INFO)
+                            log('Found subordinate_configuration on %s but it '
+                                'contained nothing for %s'
+                                % (rid, self.config_file), level=INFO)
                             continue
 
                         sub_config = sub_config[self.config_file]
@@ -1305,7 +1380,7 @@ class DataPortContext(NeutronPortContext):
             normalized.update({port: port for port in resolved
                                if port in ports})
             if resolved:
-                return {bridge: normalized[port] for port, bridge in
+                return {normalized[port]: bridge for port, bridge in
                         six.iteritems(portmap) if port in normalized.keys()}
 
         return None
@@ -1316,12 +1391,22 @@ class PhyNICMTUContext(DataPortContext):
     def __call__(self):
         ctxt = {}
         mappings = super(PhyNICMTUContext, self).__call__()
-        if mappings and mappings.values():
-            ports = mappings.values()
+        if mappings and mappings.keys():
+            ports = sorted(mappings.keys())
             napi_settings = NeutronAPIContext()()
             mtu = napi_settings.get('network_device_mtu')
+            all_ports = set()
+            # If any of ports is a vlan device, its underlying device must have
+            # mtu applied first.
+            for port in ports:
+                for lport in glob.glob("/sys/class/net/%s/lower_*" % port):
+                    lport = os.path.basename(lport)
+                    all_ports.add(lport.split('_')[1])
+
+            all_ports = list(all_ports)
+            all_ports.extend(ports)
             if mtu:
-                ctxt["devs"] = '\\n'.join(ports)
+                ctxt["devs"] = '\\n'.join(all_ports)
                 ctxt['mtu'] = mtu
 
         return ctxt
@@ -1353,6 +1438,6 @@ class NetworkServiceContext(OSContextGenerator):
                     'auth_protocol':
                     rdata.get('auth_protocol') or 'http',
                 }
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     return ctxt
         return {}

hooks/charmhelpers/contrib/openstack/neutron.py

@@ -195,6 +195,34 @@ def neutron_plugins():
             'packages': [],
             'server_packages': ['neutron-server', 'neutron-plugin-nuage'],
             'server_services': ['neutron-server']
+        },
+        'plumgrid': {
+            'config': '/etc/neutron/plugins/plumgrid/plumgrid.ini',
+            'driver': 'neutron.plugins.plumgrid.plumgrid_plugin.plumgrid_plugin.NeutronPluginPLUMgridV2',
+            'contexts': [
+                context.SharedDBContext(user=config('database-user'),
+                                        database=config('database'),
+                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'services': [],
+            'packages': [['plumgrid-lxc'],
+                         ['iovisor-dkms']],
+            'server_packages': ['neutron-server',
+                                'neutron-plugin-plumgrid'],
+            'server_services': ['neutron-server']
+        },
+        'midonet': {
+            'config': '/etc/neutron/plugins/midonet/midonet.ini',
+            'driver': 'midonet.neutron.plugin.MidonetPluginV2',
+            'contexts': [
+                context.SharedDBContext(user=config('neutron-database-user'),
+                                        database=config('neutron-database'),
+                                        relation_prefix='neutron',
+                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'services': [],
+            'packages': [[headers_package()] + determine_dkms_package()],
+            'server_packages': ['neutron-server',
+                                'python-neutron-plugin-midonet'],
+            'server_services': ['neutron-server']
         }
     }
     if release >= 'icehouse':
@@ -296,10 +324,10 @@ def parse_bridge_mappings(mappings):
 def parse_data_port_mappings(mappings, default_bridge='br-data'):
     """Parse data port mappings.
 
-    Mappings must be a space-delimited list of port:bridge mappings.
+    Mappings must be a space-delimited list of bridge:port.
 
-    Returns dict of the form {port:bridge} where port may be an mac address or
-    interface name.
+    Returns dict of the form {port:bridge} where ports may be mac addresses or
+    interface names.
     """
 
     # NOTE(dosaboy): we use rvalue for key to allow multiple values to be

hooks/charmhelpers/contrib/openstack/templates/ceph.conf

@@ -13,3 +13,9 @@ log to syslog = {{ use_syslog }}
 err to syslog = {{ use_syslog }}
 clog to syslog = {{ use_syslog }}
 
+[client]
+{% if rbd_client_cache_settings -%}
+{% for key, value in rbd_client_cache_settings.iteritems() -%}
+{{ key }} = {{ value }}
+{% endfor -%}
+{%- endif %}

hooks/charmhelpers/contrib/openstack/templating.py

@@ -18,7 +18,7 @@ import os
 
 import six
 
-from charmhelpers.fetch import apt_install
+from charmhelpers.fetch import apt_install, apt_update
 from charmhelpers.core.hookenv import (
     log,
     ERROR,
@@ -29,6 +29,7 @@ from charmhelpers.contrib.openstack.utils import OPENSTACK_CODENAMES
 try:
     from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 except ImportError:
+    apt_update(fatal=True)
     apt_install('python-jinja2', fatal=True)
     from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 
@@ -112,7 +113,7 @@ class OSConfigTemplate(object):
 
     def complete_contexts(self):
         '''
-        Return a list of interfaces that have atisfied contexts.
+        Return a list of interfaces that have satisfied contexts.
         '''
         if self._complete_contexts:
             return self._complete_contexts
@@ -293,3 +294,30 @@ class OSConfigRenderer(object):
         [interfaces.extend(i.complete_contexts())
          for i in six.itervalues(self.templates)]
         return interfaces
+
+    def get_incomplete_context_data(self, interfaces):
+        '''
+        Return dictionary of relation status of interfaces and any missing
+        required context data. Example:
+            {'amqp': {'missing_data': ['rabbitmq_password'], 'related': True},
+             'zeromq-configuration': {'related': False}}
+        '''
+        incomplete_context_data = {}
+
+        for i in six.itervalues(self.templates):
+            for context in i.contexts:
+                for interface in interfaces:
+                    related = False
+                    if interface in context.interfaces:
+                        related = context.get_related()
+                        missing_data = context.missing_data
+                        if missing_data:
+                            incomplete_context_data[interface] = {'missing_data': missing_data}
+                        if related:
+                            if incomplete_context_data.get(interface):
+                                incomplete_context_data[interface].update({'related': True})
+                            else:
+                                incomplete_context_data[interface] = {'related': True}
+                        else:
+                            incomplete_context_data[interface] = {'related': False}
+        return incomplete_context_data

hooks/charmhelpers/contrib/openstack/utils.py

@@ -1,5 +1,3 @@
-#!/usr/bin/python
-
 # Copyright 2014-2015 Canonical Limited.
 #
 # This file is part of charm-helpers.
@@ -27,6 +25,7 @@ import sys
 import re
 
 import six
+import traceback
 import yaml
 
 from charmhelpers.contrib.network import ip
@@ -36,12 +35,16 @@ from charmhelpers.core import (
 )
 
 from charmhelpers.core.hookenv import (
+    action_fail,
+    action_set,
     config,
     log as juju_log,
     charm_dir,
     INFO,
     relation_ids,
-    relation_set
+    relation_set,
+    status_set,
+    hook_name
 )
 
 from charmhelpers.contrib.storage.linux.lvm import (
@@ -51,7 +54,8 @@ from charmhelpers.contrib.storage.linux.lvm import (
 )
 
 from charmhelpers.contrib.network.ip import (
-    get_ipv6_addr
+    get_ipv6_addr,
+    is_ipv6,
 )
 
 from charmhelpers.contrib.python.packages import (
@@ -116,6 +120,8 @@ SWIFT_CODENAMES = OrderedDict([
     ('2.2.1', 'kilo'),
     ('2.2.2', 'kilo'),
     ('2.3.0', 'liberty'),
+    ('2.4.0', 'liberty'),
+    ('2.5.0', 'liberty'),
 ])
 
 # >= Liberty version->codename mapping
@@ -144,6 +150,9 @@ PACKAGE_CODENAMES = {
     'glance-common': OrderedDict([
         ('11.0.0', 'liberty'),
     ]),
+    'openstack-dashboard': OrderedDict([
+        ('8.0.0', 'liberty'),
+    ]),
 }
 
 DEFAULT_LOOPBACK_SIZE = '5G'
@@ -195,9 +204,9 @@ def get_os_codename_version(vers):
         error_out(e)
 
 
-def get_os_version_codename(codename):
+def get_os_version_codename(codename, version_map=OPENSTACK_CODENAMES):
     '''Determine OpenStack version number from codename.'''
-    for k, v in six.iteritems(OPENSTACK_CODENAMES):
+    for k, v in six.iteritems(version_map):
         if v == codename:
             return k
     e = 'Could not derive OpenStack version for '\
@@ -229,7 +238,7 @@ def get_os_codename_package(package, fatal=True):
         error_out(e)
 
     vers = apt.upstream_version(pkg.current_ver.ver_str)
-    match = re.match('^(\d)\.(\d)\.(\d)', vers)
+    match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
     if match:
         vers = match.group(0)
 
@@ -250,6 +259,8 @@ def get_os_codename_package(package, fatal=True):
                 vers = vers[:6]
                 return OPENSTACK_CODENAMES[vers]
         except KeyError:
+            if not fatal:
+                return None
             e = 'Could not determine OpenStack codename for version %s' % vers
             error_out(e)
 
@@ -429,7 +440,11 @@ def openstack_upgrade_available(package):
     import apt_pkg as apt
     src = config('openstack-origin')
     cur_vers = get_os_version_package(package)
-    available_vers = get_os_version_install_source(src)
+    if "swift" in package:
+        codename = get_os_codename_install_source(src)
+        available_vers = get_os_version_codename(codename, SWIFT_CODENAMES)
+    else:
+        available_vers = get_os_version_install_source(src)
     apt.init()
     return apt.version_compare(available_vers, cur_vers) == 1
 
@@ -506,6 +521,12 @@ def sync_db_with_multi_ipv6_addresses(database, database_user,
                                       relation_prefix=None):
     hosts = get_ipv6_addr(dynamic_only=False)
 
+    if config('vip'):
+        vips = config('vip').split()
+        for vip in vips:
+            if vip and is_ipv6(vip):
+                hosts.append(vip)
+
     kwargs = {'database': database,
               'username': database_user,
               'hostname': json.dumps(hosts)}
@@ -741,3 +762,217 @@ def git_yaml_value(projects_yaml, key):
         return projects[key]
 
     return None
+
+
+def os_workload_status(configs, required_interfaces, charm_func=None):
+    """
+    Decorator to set workload status based on complete contexts
+    """
+    def wrap(f):
+        @wraps(f)
+        def wrapped_f(*args, **kwargs):
+            # Run the original function first
+            f(*args, **kwargs)
+            # Set workload status now that contexts have been
+            # acted on
+            set_os_workload_status(configs, required_interfaces, charm_func)
+        return wrapped_f
+    return wrap
+
+
+def set_os_workload_status(configs, required_interfaces, charm_func=None):
+    """
+    Set workload status based on complete contexts.
+    status-set missing or incomplete contexts
+    and juju-log details of missing required data.
+    charm_func is a charm specific function to run checking
+    for charm specific requirements such as a VIP setting.
+    """
+    incomplete_rel_data = incomplete_relation_data(configs, required_interfaces)
+    state = 'active'
+    missing_relations = []
+    incomplete_relations = []
+    message = None
+    charm_state = None
+    charm_message = None
+
+    for generic_interface in incomplete_rel_data.keys():
+        related_interface = None
+        missing_data = {}
+        # Related or not?
+        for interface in incomplete_rel_data[generic_interface]:
+            if incomplete_rel_data[generic_interface][interface].get('related'):
+                related_interface = interface
+                missing_data = incomplete_rel_data[generic_interface][interface].get('missing_data')
+        # No relation ID for the generic_interface
+        if not related_interface:
+            juju_log("{} relation is missing and must be related for "
+                     "functionality. ".format(generic_interface), 'WARN')
+            state = 'blocked'
+            if generic_interface not in missing_relations:
+                missing_relations.append(generic_interface)
+        else:
+            # Relation ID exists but no related unit
+            if not missing_data:
+                # Edge case relation ID exists but departing
+                if ('departed' in hook_name() or 'broken' in hook_name()) \
+                        and related_interface in hook_name():
+                    state = 'blocked'
+                    if generic_interface not in missing_relations:
+                        missing_relations.append(generic_interface)
+                    juju_log("{} relation's interface, {}, "
+                             "relationship is departed or broken "
+                             "and is required for functionality."
+                             "".format(generic_interface, related_interface), "WARN")
+                # Normal case relation ID exists but no related unit
+                # (joining)
+                else:
+                    juju_log("{} relations's interface, {}, is related but has "
+                             "no units in the relation."
+                             "".format(generic_interface, related_interface), "INFO")
+            # Related unit exists and data missing on the relation
+            else:
+                juju_log("{} relation's interface, {}, is related awaiting "
+                         "the following data from the relationship: {}. "
+                         "".format(generic_interface, related_interface,
+                                   ", ".join(missing_data)), "INFO")
+            if state != 'blocked':
+                state = 'waiting'
+            if generic_interface not in incomplete_relations \
+                    and generic_interface not in missing_relations:
+                incomplete_relations.append(generic_interface)
+
+    if missing_relations:
+        message = "Missing relations: {}".format(", ".join(missing_relations))
+        if incomplete_relations:
+            message += "; incomplete relations: {}" \
+                       "".format(", ".join(incomplete_relations))
+        state = 'blocked'
+    elif incomplete_relations:
+        message = "Incomplete relations: {}" \
+                  "".format(", ".join(incomplete_relations))
+        state = 'waiting'
+
+    # Run charm specific checks
+    if charm_func:
+        charm_state, charm_message = charm_func(configs)
+        if charm_state != 'active' and charm_state != 'unknown':
+            state = workload_state_compare(state, charm_state)
+            if message:
+                message = "{} {}".format(message, charm_message)
+            else:
+                message = charm_message
+
+    # Set to active if all requirements have been met
+    if state == 'active':
+        message = "Unit is ready"
+        juju_log(message, "INFO")
+
+    status_set(state, message)
+
+
+def workload_state_compare(current_workload_state, workload_state):
+    """ Return highest priority of two states"""
+    hierarchy = {'unknown': -1,
+                 'active': 0,
+                 'maintenance': 1,
+                 'waiting': 2,
+                 'blocked': 3,
+                 }
+
+    if hierarchy.get(workload_state) is None:
+        workload_state = 'unknown'
+    if hierarchy.get(current_workload_state) is None:
+        current_workload_state = 'unknown'
+
+    # Set workload_state based on hierarchy of statuses
+    if hierarchy.get(current_workload_state) > hierarchy.get(workload_state):
+        return current_workload_state
+    else:
+        return workload_state
+
+
+def incomplete_relation_data(configs, required_interfaces):
+    """
+    Check complete contexts against required_interfaces
+    Return dictionary of incomplete relation data.
+
+    configs is an OSConfigRenderer object with configs registered
+
+    required_interfaces is a dictionary of required general interfaces
+    with dictionary values of possible specific interfaces.
+    Example:
+    required_interfaces = {'database': ['shared-db', 'pgsql-db']}
+
+    The interface is said to be satisfied if anyone of the interfaces in the
+    list has a complete context.
+
+    Return dictionary of incomplete or missing required contexts with relation
+    status of interfaces and any missing data points. Example:
+        {'message':
+             {'amqp': {'missing_data': ['rabbitmq_password'], 'related': True},
+              'zeromq-configuration': {'related': False}},
+         'identity':
+             {'identity-service': {'related': False}},
+         'database':
+             {'pgsql-db': {'related': False},
+              'shared-db': {'related': True}}}
+    """
+    complete_ctxts = configs.complete_contexts()
+    incomplete_relations = []
+    for svc_type in required_interfaces.keys():
+        # Avoid duplicates
+        found_ctxt = False
+        for interface in required_interfaces[svc_type]:
+            if interface in complete_ctxts:
+                found_ctxt = True
+        if not found_ctxt:
+            incomplete_relations.append(svc_type)
+    incomplete_context_data = {}
+    for i in incomplete_relations:
+        incomplete_context_data[i] = configs.get_incomplete_context_data(required_interfaces[i])
+    return incomplete_context_data
+
+
+def do_action_openstack_upgrade(package, upgrade_callback, configs):
+    """Perform action-managed OpenStack upgrade.
+
+    Upgrades packages to the configured openstack-origin version and sets
+    the corresponding action status as a result.
+
+    If the charm was installed from source we cannot upgrade it.
+    For backwards compatibility a config flag (action-managed-upgrade) must
+    be set for this code to run, otherwise a full service level upgrade will
+    fire on config-changed.
+
+    @param package: package name for determining if upgrade available
+    @param upgrade_callback: function callback to charm's upgrade function
+    @param configs: templating object derived from OSConfigRenderer class
+
+    @return: True if upgrade successful; False if upgrade failed or skipped
+    """
+    ret = False
+
+    if git_install_requested():
+        action_set({'outcome': 'installed from source, skipped upgrade.'})
+    else:
+        if openstack_upgrade_available(package):
+            if config('action-managed-upgrade'):
+                juju_log('Upgrading OpenStack release')
+
+                try:
+                    upgrade_callback(configs=configs)
+                    action_set({'outcome': 'success, upgrade completed.'})
+                    ret = True
+                except:
+                    action_set({'outcome': 'upgrade failed, see traceback.'})
+                    action_set({'traceback': traceback.format_exc()})
+                    action_fail('do_openstack_upgrade resulted in an '
+                                'unexpected error')
+            else:
+                action_set({'outcome': 'action-managed-upgrade config is '
+                                       'False, skipped upgrade.'})
+        else:
+            action_set({'outcome': 'no upgrade available.'})
+
+    return ret

hooks/charmhelpers/contrib/storage/linux/ceph.py

@@ -28,6 +28,7 @@ import os
 import shutil
 import json
 import time
+import uuid
 
 from subprocess import (
     check_call,
@@ -35,8 +36,10 @@ from subprocess import (
     CalledProcessError,
 )
 from charmhelpers.core.hookenv import (
+    local_unit,
     relation_get,
     relation_ids,
+    relation_set,
     related_units,
     log,
     DEBUG,
@@ -56,6 +59,8 @@ from charmhelpers.fetch import (
     apt_install,
 )
 
+from charmhelpers.core.kernel import modprobe
+
 KEYRING = '/etc/ceph/ceph.client.{}.keyring'
 KEYFILE = '/etc/ceph/ceph.client.{}.key'
 
@@ -288,17 +293,6 @@ def place_data_on_block_device(blk_device, data_src_dst):
     os.chown(data_src_dst, uid, gid)
 
 
-# TODO: re-use
-def modprobe(module):
-    """Load a kernel module and configure for auto-load on reboot."""
-    log('Loading kernel module', level=INFO)
-    cmd = ['modprobe', module]
-    check_call(cmd)
-    with open('/etc/modules', 'r+') as modules:
-        if module not in modules.read():
-            modules.write(module)
-
-
 def copy_files(src, dst, symlinks=False, ignore=None):
     """Copy files from src to dst."""
     for item in os.listdir(src):
@@ -411,17 +405,52 @@ class CephBrokerRq(object):
 
     The API is versioned and defaults to version 1.
     """
-    def __init__(self, api_version=1):
+    def __init__(self, api_version=1, request_id=None):
         self.api_version = api_version
+        if request_id:
+            self.request_id = request_id
+        else:
+            self.request_id = str(uuid.uuid1())
         self.ops = []
 
     def add_op_create_pool(self, name, replica_count=3):
         self.ops.append({'op': 'create-pool', 'name': name,
                          'replicas': replica_count})
 
+    def set_ops(self, ops):
+        """Set request ops to provided value.
+
+        Useful for injecting ops that come from a previous request
+        to allow comparisons to ensure validity.
+        """
+        self.ops = ops
+
     @property
     def request(self):
-        return json.dumps({'api-version': self.api_version, 'ops': self.ops})
+        return json.dumps({'api-version': self.api_version, 'ops': self.ops,
+                           'request-id': self.request_id})
+
+    def _ops_equal(self, other):
+        if len(self.ops) == len(other.ops):
+            for req_no in range(0, len(self.ops)):
+                for key in ['replicas', 'name', 'op']:
+                    if self.ops[req_no][key] != other.ops[req_no][key]:
+                        return False
+        else:
+            return False
+        return True
+
+    def __eq__(self, other):
+        if not isinstance(other, self.__class__):
+            return False
+        if self.api_version == other.api_version and \
+                self._ops_equal(other):
+            return True
+        else:
+            return False
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
 
 
 class CephBrokerRsp(object):
@@ -431,10 +460,15 @@ class CephBrokerRsp(object):
 
     The API is versioned and defaults to version 1.
     """
+
     def __init__(self, encoded_rsp):
         self.api_version = None
         self.rsp = json.loads(encoded_rsp)
 
+    @property
+    def request_id(self):
+        return self.rsp.get('request-id')
+
     @property
     def exit_code(self):
         return self.rsp.get('exit-code')
@@ -442,3 +476,182 @@ class CephBrokerRsp(object):
     @property
     def exit_msg(self):
         return self.rsp.get('stderr')
+
+
+# Ceph Broker Conversation:
+# If a charm needs an action to be taken by ceph it can create a CephBrokerRq
+# and send that request to ceph via the ceph relation. The CephBrokerRq has a
+# unique id so that the client can identity which CephBrokerRsp is associated
+# with the request. Ceph will also respond to each client unit individually
+# creating a response key per client unit eg glance/0 will get a CephBrokerRsp
+# via key broker-rsp-glance-0
+#
+# To use this the charm can just do something like:
+#
+# from charmhelpers.contrib.storage.linux.ceph import (
+#     send_request_if_needed,
+#     is_request_complete,
+#     CephBrokerRq,
+# )
+#
+# @hooks.hook('ceph-relation-changed')
+# def ceph_changed():
+#     rq = CephBrokerRq()
+#     rq.add_op_create_pool(name='poolname', replica_count=3)
+#
+#     if is_request_complete(rq):
+#         <Request complete actions>
+#     else:
+#         send_request_if_needed(get_ceph_request())
+#
+# CephBrokerRq and CephBrokerRsp are serialized into JSON. Below is an example
+# of glance having sent a request to ceph which ceph has successfully processed
+#  'ceph:8': {
+#      'ceph/0': {
+#          'auth': 'cephx',
+#          'broker-rsp-glance-0': '{"request-id": "0bc7dc54", "exit-code": 0}',
+#          'broker_rsp': '{"request-id": "0da543b8", "exit-code": 0}',
+#          'ceph-public-address': '10.5.44.103',
+#          'key': 'AQCLDttVuHXINhAAvI144CB09dYchhHyTUY9BQ==',
+#          'private-address': '10.5.44.103',
+#      },
+#      'glance/0': {
+#          'broker_req': ('{"api-version": 1, "request-id": "0bc7dc54", '
+#                         '"ops": [{"replicas": 3, "name": "glance", '
+#                         '"op": "create-pool"}]}'),
+#          'private-address': '10.5.44.109',
+#      },
+#  }
+
+def get_previous_request(rid):
+    """Return the last ceph broker request sent on a given relation
+
+    @param rid: Relation id to query for request
+    """
+    request = None
+    broker_req = relation_get(attribute='broker_req', rid=rid,
+                              unit=local_unit())
+    if broker_req:
+        request_data = json.loads(broker_req)
+        request = CephBrokerRq(api_version=request_data['api-version'],
+                               request_id=request_data['request-id'])
+        request.set_ops(request_data['ops'])
+
+    return request
+
+
+def get_request_states(request):
+    """Return a dict of requests per relation id with their corresponding
+       completion state.
+
+    This allows a charm, which has a request for ceph, to see whether there is
+    an equivalent request already being processed and if so what state that
+    request is in.
+
+    @param request: A CephBrokerRq object
+    """
+    complete = []
+    requests = {}
+    for rid in relation_ids('ceph'):
+        complete = False
+        previous_request = get_previous_request(rid)
+        if request == previous_request:
+            sent = True
+            complete = is_request_complete_for_rid(previous_request, rid)
+        else:
+            sent = False
+            complete = False
+
+        requests[rid] = {
+            'sent': sent,
+            'complete': complete,
+        }
+
+    return requests
+
+
+def is_request_sent(request):
+    """Check to see if a functionally equivalent request has already been sent
+
+    Returns True if a similair request has been sent
+
+    @param request: A CephBrokerRq object
+    """
+    states = get_request_states(request)
+    for rid in states.keys():
+        if not states[rid]['sent']:
+            return False
+
+    return True
+
+
+def is_request_complete(request):
+    """Check to see if a functionally equivalent request has already been
+    completed
+
+    Returns True if a similair request has been completed
+
+    @param request: A CephBrokerRq object
+    """
+    states = get_request_states(request)
+    for rid in states.keys():
+        if not states[rid]['complete']:
+            return False
+
+    return True
+
+
+def is_request_complete_for_rid(request, rid):
+    """Check if a given request has been completed on the given relation
+
+    @param request: A CephBrokerRq object
+    @param rid: Relation ID
+    """
+    broker_key = get_broker_rsp_key()
+    for unit in related_units(rid):
+        rdata = relation_get(rid=rid, unit=unit)
+        if rdata.get(broker_key):
+            rsp = CephBrokerRsp(rdata.get(broker_key))
+            if rsp.request_id == request.request_id:
+                if not rsp.exit_code:
+                    return True
+        else:
+            # The remote unit sent no reply targeted at this unit so either the
+            # remote ceph cluster does not support unit targeted replies or it
+            # has not processed our request yet.
+            if rdata.get('broker_rsp'):
+                request_data = json.loads(rdata['broker_rsp'])
+                if request_data.get('request-id'):
+                    log('Ignoring legacy broker_rsp without unit key as remote '
+                        'service supports unit specific replies', level=DEBUG)
+                else:
+                    log('Using legacy broker_rsp as remote service does not '
+                        'supports unit specific replies', level=DEBUG)
+                    rsp = CephBrokerRsp(rdata['broker_rsp'])
+                    if not rsp.exit_code:
+                        return True
+
+    return False
+
+
+def get_broker_rsp_key():
+    """Return broker response key for this unit
+
+    This is the key that ceph is going to use to pass request status
+    information back to this unit
+    """
+    return 'broker-rsp-' + local_unit().replace('/', '-')
+
+
+def send_request_if_needed(request):
+    """Send broker request if an equivalent request has not already been sent
+
+    @param request: A CephBrokerRq object
+    """
+    if is_request_sent(request):
+        log('Request already sent but not complete, not sending new request',
+            level=DEBUG)
+    else:
+        for rid in relation_ids('ceph'):
+            log('Sending request {}'.format(request.request_id), level=DEBUG)
+            relation_set(relation_id=rid, broker_req=request.request)

hooks/charmhelpers/core/hookenv.py

@@ -623,6 +623,38 @@ def unit_private_ip():
     return unit_get('private-address')
 
 
+@cached
+def storage_get(attribute="", storage_id=""):
+    """Get storage attributes"""
+    _args = ['storage-get', '--format=json']
+    if storage_id:
+        _args.extend(('-s', storage_id))
+    if attribute:
+        _args.append(attribute)
+    try:
+        return json.loads(subprocess.check_output(_args).decode('UTF-8'))
+    except ValueError:
+        return None
+
+
+@cached
+def storage_list(storage_name=""):
+    """List the storage IDs for the unit"""
+    _args = ['storage-list', '--format=json']
+    if storage_name:
+        _args.append(storage_name)
+    try:
+        return json.loads(subprocess.check_output(_args).decode('UTF-8'))
+    except ValueError:
+        return None
+    except OSError as e:
+        import errno
+        if e.errno == errno.ENOENT:
+            # storage-list does not exist
+            return []
+        raise
+
+
 class UnregisteredHookError(Exception):
     """Raised when an undefined hook is called"""
     pass
@@ -767,21 +799,23 @@ def status_set(workload_state, message):
 
 
 def status_get():
-    """Retrieve the previously set juju workload state
+    """Retrieve the previously set juju workload state and message
+
+    If the status-get command is not found then assume this is juju < 1.23 and
+    return 'unknown', ""
 
-    If the status-set command is not found then assume this is juju < 1.23 and
-    return 'unknown'
     """
-    cmd = ['status-get']
+    cmd = ['status-get', "--format=json", "--include-data"]
     try:
-        raw_status = subprocess.check_output(cmd, universal_newlines=True)
-        status = raw_status.rstrip()
-        return status
+        raw_status = subprocess.check_output(cmd)
     except OSError as e:
         if e.errno == errno.ENOENT:
-            return 'unknown'
+            return ('unknown', "")
         else:
             raise
+    else:
+        status = json.loads(raw_status.decode("UTF-8"))
+        return (status["status"], status["message"])
 
 
 def translate_exc(from_exc, to_exc):

hooks/charmhelpers/core/host.py

@@ -63,32 +63,48 @@ def service_reload(service_name, restart_on_failure=False):
     return service_result
 
 
-def service_pause(service_name, init_dir=None):
+def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"):
     """Pause a system service.
 
     Stop it, and prevent it from starting again at boot."""
-    if init_dir is None:
-        init_dir = "/etc/init"
     stopped = service_stop(service_name)
-    # XXX: Support systemd too
-    override_path = os.path.join(
-        init_dir, '{}.override'.format(service_name))
-    with open(override_path, 'w') as fh:
-        fh.write("manual\n")
+    upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
+    sysv_file = os.path.join(initd_dir, service_name)
+    if os.path.exists(upstart_file):
+        override_path = os.path.join(
+            init_dir, '{}.override'.format(service_name))
+        with open(override_path, 'w') as fh:
+            fh.write("manual\n")
+    elif os.path.exists(sysv_file):
+        subprocess.check_call(["update-rc.d", service_name, "disable"])
+    else:
+        # XXX: Support SystemD too
+        raise ValueError(
+            "Unable to detect {0} as either Upstart {1} or SysV {2}".format(
+                service_name, upstart_file, sysv_file))
     return stopped
 
 
-def service_resume(service_name, init_dir=None):
+def service_resume(service_name, init_dir="/etc/init",
+                   initd_dir="/etc/init.d"):
     """Resume a system service.
 
     Reenable starting again at boot. Start the service"""
-    # XXX: Support systemd too
-    if init_dir is None:
-        init_dir = "/etc/init"
-    override_path = os.path.join(
-        init_dir, '{}.override'.format(service_name))
-    if os.path.exists(override_path):
-        os.unlink(override_path)
+    upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
+    sysv_file = os.path.join(initd_dir, service_name)
+    if os.path.exists(upstart_file):
+        override_path = os.path.join(
+            init_dir, '{}.override'.format(service_name))
+        if os.path.exists(override_path):
+            os.unlink(override_path)
+    elif os.path.exists(sysv_file):
+        subprocess.check_call(["update-rc.d", service_name, "enable"])
+    else:
+        # XXX: Support SystemD too
+        raise ValueError(
+            "Unable to detect {0} as either Upstart {1} or SysV {2}".format(
+                service_name, upstart_file, sysv_file))
+
     started = service_start(service_name)
     return started
 
@@ -550,7 +566,14 @@ def chdir(d):
         os.chdir(cur)
 
 
-def chownr(path, owner, group, follow_links=True):
+def chownr(path, owner, group, follow_links=True, chowntopdir=False):
+    """
+    Recursively change user and group ownership of files and directories
+    in given path. Doesn't chown path itself by default, only its children.
+
+    :param bool follow_links: Also Chown links if True
+    :param bool chowntopdir: Also chown path itself if True
+    """
     uid = pwd.getpwnam(owner).pw_uid
     gid = grp.getgrnam(group).gr_gid
     if follow_links:
@@ -558,6 +581,10 @@ def chownr(path, owner, group, follow_links=True):
     else:
         chown = os.lchown
 
+    if chowntopdir:
+        broken_symlink = os.path.lexists(path) and not os.path.exists(path)
+        if not broken_symlink:
+            chown(path, uid, gid)
     for root, dirs, files in os.walk(path):
         for name in dirs + files:
             full = os.path.join(root, name)

hooks/charmhelpers/core/hugepage.py

@@ -25,11 +25,13 @@ from charmhelpers.core.host import (
     fstab_mount,
     mkdir,
 )
+from charmhelpers.core.strutils import bytes_from_string
+from subprocess import check_output
 
 
 def hugepage_support(user, group='hugetlb', nr_hugepages=256,
                      max_map_count=65536, mnt_point='/run/hugepages/kvm',
-                     pagesize='2MB', mount=True):
+                     pagesize='2MB', mount=True, set_shmmax=False):
     """Enable hugepages on system.
 
     Args:
@@ -49,6 +51,11 @@ def hugepage_support(user, group='hugetlb', nr_hugepages=256,
         'vm.max_map_count': max_map_count,
         'vm.hugetlb_shm_group': gid,
     }
+    if set_shmmax:
+        shmmax_current = int(check_output(['sysctl', '-n', 'kernel.shmmax']))
+        shmmax_minsize = bytes_from_string(pagesize) * nr_hugepages
+        if shmmax_minsize > shmmax_current:
+            sysctl_settings['kernel.shmmax'] = shmmax_minsize
     sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf')
     mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False)
     lfstab = fstab.Fstab()

hooks/charmhelpers/core/strutils.py

@@ -18,6 +18,7 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import six
+import re
 
 
 def bool_from_string(value):
@@ -40,3 +41,32 @@ def bool_from_string(value):
 
     msg = "Unable to interpret string value '%s' as boolean" % (value)
     raise ValueError(msg)
+
+
+def bytes_from_string(value):
+    """Interpret human readable string value as bytes.
+
+    Returns int
+    """
+    BYTE_POWER = {
+        'K': 1,
+        'KB': 1,
+        'M': 2,
+        'MB': 2,
+        'G': 3,
+        'GB': 3,
+        'T': 4,
+        'TB': 4,
+        'P': 5,
+        'PB': 5,
+    }
+    if isinstance(value, six.string_types):
+        value = six.text_type(value)
+    else:
+        msg = "Unable to interpret non-string value '%s' as boolean" % (value)
+        raise ValueError(msg)
+    matches = re.match("([0-9]+)([a-zA-Z]+)", value)
+    if not matches:
+        msg = "Unable to interpret string value '%s' as bytes" % (value)
+        raise ValueError(msg)
+    return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])

hooks/etcd-proxy-relation-broken

@@ -0,0 +1 @@
+neutron_api_hooks.py

hooks/etcd-proxy-relation-changed

@@ -0,0 +1 @@
+neutron_api_hooks.py

hooks/etcd-proxy-relation-departed

@@ -0,0 +1 @@
+neutron_api_hooks.py

hooks/etcd-proxy-relation-joined

@@ -0,0 +1 @@
+neutron_api_hooks.py

hooks/install

@@ -1 +0,0 @@
-neutron_api_hooks.py

hooks/install

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Wrapper to deal with newer Ubuntu versions that don't have py2 installed
+# by default.
+
+declare -a DEPS=('apt' 'netaddr' 'netifaces' 'pip' 'yaml')
+
+check_and_install() {
+    pkg="${1}-${2}"
+    if ! dpkg -s ${pkg} 2>&1 > /dev/null; then
+        apt-get -y install ${pkg}
+    fi
+}
+
+PYTHON="python"
+
+for dep in ${DEPS[@]}; do
+    check_and_install ${PYTHON} ${dep}
+done
+
+exec ./hooks/install.real

hooks/install.real

@@ -0,0 +1 @@
+neutron_api_hooks.py

hooks/neutron_api_context.py

@@ -162,6 +162,10 @@ class NeutronCCContext(context.NeutronContext):
                     ','.join(config('nsx-controllers').split())
                 ctxt['nsx_controllers_list'] = \
                     config('nsx-controllers').split()
+        if config('neutron-plugin') == 'plumgrid':
+            ctxt['pg_username'] = config('plumgrid-username')
+            ctxt['pg_password'] = config('plumgrid-password')
+            ctxt['virtual_ip'] = config('plumgrid-virtual-ip')
         ctxt['l2_population'] = self.neutron_l2_population
         ctxt['enable_dvr'] = self.neutron_dvr
         ctxt['l3_ha'] = self.neutron_l3ha
@@ -251,6 +255,28 @@ class HAProxyContext(context.HAProxyContext):
         return ctxt
 
 
+class EtcdContext(context.OSContextGenerator):
+    interfaces = ['etcd-proxy']
+
+    def __call__(self):
+        ctxt = {'cluster': ''}
+        cluster_string = ''
+
+        if not config('neutron-plugin') == 'Calico':
+            return ctxt
+
+        for rid in relation_ids('etcd-proxy'):
+            for unit in related_units(rid):
+                rdata = relation_get(rid=rid, unit=unit)
+                cluster_string = rdata.get('cluster')
+                if cluster_string:
+                    break
+
+        ctxt['cluster'] = cluster_string
+
+        return ctxt
+
+
 class NeutronApiSDNContext(context.SubordinateConfigContext):
     interfaces = 'neutron-plugin-api-subordinate'
 
@@ -279,6 +305,10 @@ class NeutronApiSDNContext(context.SubordinateConfigContext):
                 'templ_key': 'restart_trigger',
                 'value': '',
             },
+            'quota-driver': {
+                'templ_key': 'quota_driver',
+                'value': '',
+            },
         }
         for rid in relation_ids('neutron-plugin-api-subordinate'):
             for unit in related_units(rid):

hooks/neutron_api_hooks.py

@@ -19,6 +19,7 @@ from charmhelpers.core.hookenv import (
     relation_get,
     relation_ids,
     relation_set,
+    status_set,
     open_port,
     unit_get,
 )
@@ -39,16 +40,18 @@ from charmhelpers.fetch import (
 from charmhelpers.contrib.openstack.utils import (
     config_value_changed,
     configure_installation_source,
+    set_os_workload_status,
     git_install_requested,
     openstack_upgrade_available,
     os_requires_version,
     os_release,
-    sync_db_with_multi_ipv6_addresses
+    sync_db_with_multi_ipv6_addresses,
 )
 
 from neutron_api_utils import (
     CLUSTER_RES,
     NEUTRON_CONF,
+    REQUIRED_INTERFACES,
     api_port,
     determine_packages,
     determine_ports,
@@ -63,6 +66,9 @@ from neutron_api_utils import (
     services,
     setup_ipv6,
     get_topics,
+    check_optional_relations,
+    additional_install_locations,
+    force_etcd_restart,
 )
 from neutron_api_context import (
     get_dvr,
@@ -70,6 +76,7 @@ from neutron_api_context import (
     get_l2population,
     get_overlay_network_type,
     IdentityServiceContext,
+    EtcdContext,
 )
 
 from charmhelpers.contrib.hahelpers.cluster import (
@@ -150,14 +157,20 @@ def configure_https():
         identity_joined(rid=rid)
 
 
+@hooks.hook('install.real')
 @hooks.hook()
 def install():
+    status_set('maintenance', 'Executing pre-install')
     execd_preinstall()
     configure_installation_source(config('openstack-origin'))
-    packages = determine_packages(config('openstack-origin'))
+    additional_install_locations(
+        config('neutron-plugin'), config('openstack-origin')
+    )
 
     add_source(config('extra-source'), config('extra-key'))
+    status_set('maintenance', 'Installing apt packages')
     apt_update()
+    packages = determine_packages(config('openstack-origin'))
     apt_install(packages, fatal=True)
 
     if config('neutron-plugin') == 'vsp':
@@ -183,6 +196,7 @@ def install():
                 log('install failed with error: {}'.format(e.message))
                 raise Exception(e)
 
+    status_set('maintenance', 'Git install')
     git_install(config('openstack-origin-git'))
 
     [open_port(port) for port in determine_ports()]
@@ -213,14 +227,16 @@ def config_changed():
         if l3ha_router_present() and not get_l3ha():
             e = ('Cannot disable Router HA while ha enabled routers exist.'
                  ' Please remove any ha routers')
-            log(e, level=ERROR)
+            status_set('blocked', e)
             raise Exception(e)
         if dvr_router_present() and not get_dvr():
             e = ('Cannot disable dvr while dvr enabled routers exist. Please'
                  ' remove any distributed routers')
             log(e, level=ERROR)
+            status_set('blocked', e)
             raise Exception(e)
     if config('prefer-ipv6'):
+        status_set('maintenance', 'configuring ipv6')
         setup_ipv6()
         sync_db_with_multi_ipv6_addresses(config('database'),
                                           config('database-user'))
@@ -228,11 +244,18 @@ def config_changed():
     global CONFIGS
     if git_install_requested():
         if config_value_changed('openstack-origin-git'):
+            status_set('maintenance', 'Running Git install')
             git_install(config('openstack-origin-git'))
-    else:
-        if openstack_upgrade_available('neutron-server'):
+    elif not config('action-managed-upgrade'):
+        if openstack_upgrade_available('neutron-common'):
+            status_set('maintenance', 'Running openstack upgrade')
             do_openstack_upgrade(CONFIGS)
 
+    additional_install_locations(
+        config('neutron-plugin'),
+        config('openstack-origin')
+    )
+    status_set('maintenance', 'Installing apt packages')
     apt_install(filter_installed_packages(
                 determine_packages(config('openstack-origin'))),
                 fatal=True)
@@ -402,6 +425,7 @@ def neutron_plugin_api_relation_joined(rid=None):
             'enable-dvr': get_dvr(),
             'enable-l3ha': get_l3ha(),
             'overlay-network-type': get_overlay_network_type(),
+            'addr': unit_get('private-address'),
         }
 
         # Provide this value to relations since it needs to be set in multiple
@@ -550,11 +574,27 @@ def update_nrpe_config():
     nrpe_setup.write()
 
 
+@hooks.hook('etcd-proxy-relation-joined')
+@hooks.hook('etcd-proxy-relation-changed')
+def etcd_proxy_force_restart(relation_id=None):
+    # note(cory.benfield): Mostly etcd does not require active management,
+    # but occasionally it does require a full config nuking. This does not
+    # play well with the standard neutron-api config management, so we
+    # treat etcd like the special snowflake it insists on being.
+    CONFIGS.register('/etc/init/etcd.conf', [EtcdContext()])
+    CONFIGS.write('/etc/init/etcd.conf')
+
+    if 'etcd-proxy' in CONFIGS.complete_contexts():
+        force_etcd_restart()
+
+
 def main():
     try:
         hooks.execute(sys.argv)
     except UnregisteredHookError as e:
         log('Unknown hook {} - skipping.'.format(e))
+    set_os_workload_status(CONFIGS, REQUIRED_INTERFACES,
+                           charm_func=check_optional_relations)
 
 
 if __name__ == '__main__':

hooks/neutron_api_utils.py

@@ -4,6 +4,7 @@ from functools import partial
 import os
 import shutil
 import subprocess
+import glob
 from base64 import b64encode
 from charmhelpers.contrib.openstack import context, templating
 from charmhelpers.contrib.openstack.neutron import (
@@ -19,6 +20,7 @@ from charmhelpers.contrib.openstack.utils import (
     git_pip_venv_dir,
     git_yaml_value,
     configure_installation_source,
+    set_os_workload_status,
 )
 
 from charmhelpers.contrib.python.packages import (
@@ -28,6 +30,8 @@ from charmhelpers.contrib.python.packages import (
 from charmhelpers.core.hookenv import (
     config,
     log,
+    relation_ids,
+    status_get,
 )
 
 from charmhelpers.fetch import (
@@ -38,15 +42,22 @@ from charmhelpers.fetch import (
 )
 
 from charmhelpers.core.host import (
+    lsb_release,
     adduser,
     add_group,
     add_user_to_group,
     mkdir,
-    lsb_release,
+    service_stop,
+    service_start,
     service_restart,
     write_file,
 )
 
+from charmhelpers.contrib.hahelpers.cluster import (
+    get_hacluster_config,
+)
+
+
 from charmhelpers.core.templating import render
 from charmhelpers.contrib.hahelpers.cluster import is_elected_leader
 
@@ -111,6 +122,8 @@ API_PORTS = {
 NEUTRON_CONF_DIR = "/etc/neutron"
 
 NEUTRON_CONF = '%s/neutron.conf' % NEUTRON_CONF_DIR
+NEUTRON_LBAAS_CONF = '%s/neutron_lbaas.conf' % NEUTRON_CONF_DIR
+NEUTRON_VPNAAS_CONF = '%s/neutron_vpnaas.conf' % NEUTRON_CONF_DIR
 HAPROXY_CONF = '/etc/haproxy/haproxy.cfg'
 APACHE_CONF = '/etc/apache2/sites-available/openstack_https_frontend'
 APACHE_24_CONF = '/etc/apache2/sites-available/openstack_https_frontend.conf'
@@ -155,11 +168,61 @@ BASE_RESOURCE_MAP = OrderedDict([
     }),
 ])
 
+# The interface is said to be satisfied if anyone of the interfaces in the
+# list has a complete context.
+REQUIRED_INTERFACES = {
+    'database': ['shared-db', 'pgsql-db'],
+    'messaging': ['amqp', 'zeromq-configuration'],
+    'identity': ['identity-service'],
+}
+
+LIBERTY_RESOURCE_MAP = OrderedDict([
+    (NEUTRON_LBAAS_CONF, {
+        'services': ['neutron-server'],
+        'contexts': [],
+    }),
+    (NEUTRON_VPNAAS_CONF, {
+        'services': ['neutron-server'],
+        'contexts': [],
+    }),
+])
+
 
 def api_port(service):
     return API_PORTS[service]
 
 
+def additional_install_locations(plugin, source):
+    '''
+    Add any required additional package locations for the charm, based
+    on the Neutron plugin being used. This will also force an immediate
+    package upgrade.
+    '''
+    if plugin == 'Calico':
+        if config('calico-origin'):
+            calico_source = config('calico-origin')
+        else:
+            release = get_os_codename_install_source(source)
+            calico_source = 'ppa:project-calico/%s' % release
+
+        add_source(calico_source)
+
+        apt_update()
+        apt_upgrade()
+
+
+def force_etcd_restart():
+    '''
+    If etcd has been reconfigured we need to force it to fully restart.
+    This is necessary because etcd has some config flags that it ignores
+    after the first time it starts, so we need to make it forget them.
+    '''
+    service_stop('etcd')
+    for directory in glob.glob('/var/lib/etcd/*'):
+        shutil.rmtree(directory)
+    service_start('etcd')
+
+
 def manage_plugin():
     return config('manage-neutron-plugin-legacy-mode')
 
@@ -210,12 +273,16 @@ def determine_ports():
     return list(set(ports))
 
 
-def resource_map():
+def resource_map(release=None):
     '''
     Dynamically generate a map of resources that will be managed for a single
     hook execution.
     '''
+    release = release or os_release('neutron-common')
+
     resource_map = deepcopy(BASE_RESOURCE_MAP)
+    if release >= 'liberty':
+        resource_map.update(LIBERTY_RESOURCE_MAP)
 
     if os.path.exists('/etc/apache2/conf-available'):
         resource_map.pop(APACHE_CONF)
@@ -251,7 +318,7 @@ def resource_map():
 
 
 def register_configs(release=None):
-    release = release or os_release('neutron-server')
+    release = release or os_release('neutron-common')
     configs = templating.OSConfigRenderer(templates_dir=TEMPLATES,
                                           openstack_release=release)
     for cfg, rscs in resource_map().iteritems():
@@ -289,7 +356,7 @@ def do_openstack_upgrade(configs):
 
     :param configs: The charms main OSConfigRenderer object.
     """
-    cur_os_rel = os_release('neutron-server')
+    cur_os_rel = os_release('neutron-common')
     new_src = config('openstack-origin')
     new_os_rel = get_os_codename_install_source(new_src)
 
@@ -396,12 +463,11 @@ def setup_ipv6():
         raise Exception("IPv6 is not supported in the charms for Ubuntu "
                         "versions less than Trusty 14.04")
 
-    # NOTE(xianghui): Need to install haproxy(1.5.3) from trusty-backports
-    # to support ipv6 address, so check is required to make sure not
-    # breaking other versions, IPv6 only support for >= Trusty
-    if ubuntu_rel == 'trusty':
-        add_source('deb http://archive.ubuntu.com/ubuntu trusty-backports'
-                   ' main')
+    # Need haproxy >= 1.5.3 for ipv6 so for Trusty if we are <= Kilo we need to
+    # use trusty-backports otherwise we can use the UCA.
+    if ubuntu_rel == 'trusty' and os_release('neutron-server') < 'liberty':
+        add_source('deb http://archive.ubuntu.com/ubuntu trusty-backports '
+                   'main')
         apt_update()
         apt_install('haproxy/trusty-backports', fatal=True)
 
@@ -540,3 +606,21 @@ def git_post_install(projects_yaml):
            neutron_api_context, perms=0o644)
 
     service_restart('neutron-server')
+
+
+def check_optional_relations(configs):
+    required_interfaces = {}
+    if relation_ids('ha'):
+        required_interfaces['ha'] = ['cluster']
+        try:
+            get_hacluster_config()
+        except:
+            return ('blocked',
+                    'hacluster missing configuration: '
+                    'vip, vip_iface, vip_cidr')
+
+    if required_interfaces:
+        set_os_workload_status(configs, required_interfaces)
+        return status_get()
+    else:
+        return 'unknown', 'No optional relations'

metadata.yaml

@@ -42,6 +42,8 @@ requires:
   neutron-plugin-api-subordinate:
     interface: neutron-plugin-api-subordinate
     scope: container
+  etcd-proxy:
+    interface: etcd-proxy
 peers:
   cluster:
     interface: neutron-api-ha

templates/icehouse/etcd.conf

@@ -0,0 +1,16 @@
+# managed by juju, DO NOT EDIT
+description "etcd"
+author "etcd maintainers"
+
+start on stopped rc RUNLEVEL=[2345]
+stop on runlevel [!2345]
+
+respawn
+
+setuid etcd
+
+env ETCD_DATA_DIR=/var/lib/etcd
+export ETCD_DATA_DIR
+
+exec /usr/bin/etcd -proxy on                                                  \
+                   -initial-cluster {{ cluster }}

templates/icehouse/ml2_conf.ini

@@ -4,8 +4,12 @@
 # Configuration file maintained by Juju. Local changes may be overwritten.  
 ###############################################################################
 [ml2]
-type_drivers = {{ overlay_network_type }},vlan,flat
-tenant_network_types = {{ overlay_network_type }},vlan,flat
+{% if neutron_plugin == 'Calico' -%}
+type_drivers = local,flat
+mechanism_drivers = calico
+{% else -%}
+type_drivers = {{ overlay_network_type }},vlan,flat,local
+tenant_network_types = {{ overlay_network_type }},vlan,flat,local
 mechanism_drivers = openvswitch,hyperv,l2population
 
 [ml2_type_gre]
@@ -26,11 +30,16 @@ local_ip = {{ local_ip }}
 
 [agent]
 tunnel_types = {{ overlay_network_type }}
+{% endif -%}
 
 [securitygroup]
 {% if neutron_security_groups -%}
 enable_security_group = True
+{% if neutron_plugin == 'Calico' -%}
+firewall_driver = neutron.agent.linux.iptables_firewall.IptablesFirewallDriver
+{% else -%}
 firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
+{% endif -%}
 {% else -%}
 enable_security_group = False
 {% endif -%}

templates/icehouse/neutron.conf

@@ -30,7 +30,7 @@ core_plugin = {{ core_plugin }}
 {% if service_plugins -%}
 service_plugins = {{ service_plugins }}
 {% else -%}
-{% if neutron_plugin in ['ovs', 'ml2'] -%}
+{% if neutron_plugin in ['ovs', 'ml2', 'Calico'] -%}
 service_plugins = neutron.services.l3_router.l3_router_plugin.L3RouterPlugin,neutron.services.firewall.fwaas_plugin.FirewallPlugin,neutron.services.loadbalancer.plugin.LoadBalancerPlugin,neutron.services.vpn.plugin.VPNDriverPlugin,neutron.services.metering.metering_plugin.MeteringPlugin
 {% endif -%}
 {% endif -%}
@@ -38,8 +38,16 @@ service_plugins = neutron.services.l3_router.l3_router_plugin.L3RouterPlugin,neu
 
 {% if neutron_security_groups -%}
 allow_overlapping_ips = True
+{% if neutron_plugin == 'Calico' -%}
+neutron_firewall_driver = neutron.agent.linux.iptables_firewall.IptablesFirewallDriver
+{% else -%}
 neutron_firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
 {% endif -%}
+{% endif -%}
+
+{% if neutron_plugin == 'Calico' -%}
+dhcp_agents_per_network = 1000
+{% endif -%}
 
 {% include "parts/rabbitmq" %}
 
@@ -61,7 +69,11 @@ nova_admin_auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v2.0
 {% endif %}
 
 [quotas]
+{% if quota_driver -%}
+quota_driver = {{ quota_driver }}
+{% else -%}
 quota_driver = neutron.db.quota_db.DbQuotaDriver
+{% endif -%}
 {% if neutron_security_groups -%}
 quota_items = network,subnet,port,security_group,security_group_rule
 quota_security_group = {{ quota_security_group }}

templates/icehouse/plumgrid.ini

@@ -0,0 +1,20 @@
+# icehouse
+###############################################################################
+# [ WARNING ]
+# Configuration file maintained by Juju. Local changes may be overwritten.
+###############################################################################
+[plumgriddirector]
+# This line should be pointing to the PLUMgrid Director,
+# for the PLUMgrid platform.
+director_server={{ virtual_ip }}
+director_server_port=443
+# Authentification parameters for the Director.
+# These are the admin credentials to manage and control
+# the PLUMgrid Director server.
+username={{ pg_username }}
+password={{ pg_password }}
+servertimeout=70
+
+{% if database_host -%}
+connection = {{ database_type }}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}{% if database_ssl_ca %}?ssl_ca={{ database_ssl_ca }}{% if database_ssl_cert %}&ssl_cert={{ database_ssl_cert }}&ssl_key={{ database_ssl_key }}{% endif %}{% endif %}
+{% endif -%}

templates/juno/neutron.conf

@@ -31,15 +31,23 @@ bind_port = 9696
 
 {% if core_plugin -%}
 core_plugin = {{ core_plugin }}
-{% if neutron_plugin in ['ovs', 'ml2'] -%}
+{% if neutron_plugin in ['ovs', 'ml2', 'Calico'] -%}
 service_plugins = neutron.services.l3_router.l3_router_plugin.L3RouterPlugin,neutron.services.firewall.fwaas_plugin.FirewallPlugin,neutron.services.loadbalancer.plugin.LoadBalancerPlugin,neutron.services.vpn.plugin.VPNDriverPlugin,neutron.services.metering.metering_plugin.MeteringPlugin
 {% endif -%}
 {% endif -%}
 
 {% if neutron_security_groups -%}
 allow_overlapping_ips = True
+{% if neutron_plugin == 'Calico' -%}
+neutron_firewall_driver = neutron.agent.linux.iptables_firewall.IptablesFirewallDriver
+{% else -%}
 neutron_firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
 {% endif -%}
+{% endif -%}
+
+{% if neutron_plugin == 'Calico' -%}
+dhcp_agents_per_network = 1000
+{% endif -%}
 
 {% include "parts/rabbitmq" %}
 
@@ -61,7 +69,11 @@ nova_admin_auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v2.0
 {% endif %}
 
 [quotas]
+{% if quota_driver -%}
+quota_driver = {{ quota_driver }}
+{% else -%}
 quota_driver = neutron.db.quota_db.DbQuotaDriver
+{% endif -%}
 {% if neutron_security_groups -%}
 quota_items = network,subnet,port,security_group,security_group_rule
 {% endif -%}

templates/kilo/ml2_conf.ini

@@ -4,8 +4,12 @@
 # Configuration file maintained by Juju. Local changes may be overwritten.  
 ###############################################################################
 [ml2]
-type_drivers = {{ overlay_network_type }},vlan,flat
-tenant_network_types = {{ overlay_network_type }},vlan,flat
+{% if neutron_plugin == 'Calico' -%}
+type_drivers = local,flat
+mechanism_drivers = calico
+{% else -%}
+type_drivers = {{ overlay_network_type }},vlan,flat,local
+tenant_network_types = {{ overlay_network_type }},vlan,flat,local
 mechanism_drivers = openvswitch,l2population
 
 [ml2_type_gre]
@@ -26,11 +30,16 @@ local_ip = {{ local_ip }}
 
 [agent]
 tunnel_types = {{ overlay_network_type }}
+{% endif -%}
 
 [securitygroup]
 {% if neutron_security_groups -%}
 enable_security_group = True
+{% if neutron_plugin == 'Calico' -%}
+firewall_driver = neutron.agent.linux.iptables_firewall.IptablesFirewallDriver
+{% else -%}
 firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
+{% endif -%}
 {% else -%}
 enable_security_group = False
 {% endif -%}

templates/kilo/neutron.conf

@@ -34,7 +34,7 @@ core_plugin = {{ core_plugin }}
 {% if service_plugins -%}
 service_plugins = {{ service_plugins }}
 {% else -%}
-{% if neutron_plugin in ['ovs', 'ml2'] -%}
+{% if neutron_plugin in ['ovs', 'ml2', 'Calico'] -%}
 service_plugins = router,firewall,lbaas,vpnaas,metering
 {% endif -%}
 {% endif -%}
@@ -42,8 +42,16 @@ service_plugins = router,firewall,lbaas,vpnaas,metering
 
 {% if neutron_security_groups -%}
 allow_overlapping_ips = True
+{% if neutron_plugin == 'Calico' -%}
+neutron_firewall_driver = neutron.agent.linux.iptables_firewall.IptablesFirewallDriver
+{% else -%}
 neutron_firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
 {% endif -%}
+{% endif -%}
+
+{% if neutron_plugin == 'Calico' -%}
+dhcp_agents_per_network = 1000
+{% endif -%}
 
 notify_nova_on_port_status_changes = True
 notify_nova_on_port_data_changes = True
@@ -65,7 +73,11 @@ nova_admin_auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v2.0
 {% include "section-zeromq" %}
 
 [quotas]
+{% if quota_driver -%}
+quota_driver = {{ quota_driver }}
+{% else -%}
 quota_driver = neutron.db.quota_db.DbQuotaDriver
+{% endif -%}
 {% if neutron_security_groups -%}
 quota_items = network,subnet,port,security_group,security_group_rule
 quota_security_group = {{ quota_security_group }}

templates/liberty/neutron.conf

@@ -0,0 +1,108 @@
+# liberty
+###############################################################################
+# [ WARNING ]
+# Configuration file maintained by Juju. Local changes may be overwritten.
+# Restart trigger {{ restart_trigger }}
+###############################################################################
+[DEFAULT]
+verbose = {{ verbose }}
+debug = {{ debug }}
+use_syslog = {{ use_syslog }}
+state_path = /var/lib/neutron
+bind_host = {{ bind_host }}
+auth_strategy = keystone
+notification_driver = neutron.openstack.common.notifier.rpc_notifier
+api_workers = {{ workers }}
+rpc_workers = {{ workers }}
+
+router_distributed = {{ enable_dvr }}
+
+l3_ha = {{ l3_ha }}
+{% if l3_ha -%}
+max_l3_agents_per_router = {{ max_l3_agents_per_router }}
+min_l3_agents_per_router = {{ min_l3_agents_per_router }}
+{% endif -%}
+
+{% if neutron_bind_port -%}
+bind_port = {{ neutron_bind_port }}
+{% else -%}
+bind_port = 9696
+{% endif -%}
+
+{% if core_plugin -%}
+core_plugin = {{ core_plugin }}
+{% if service_plugins -%}
+service_plugins = {{ service_plugins }}
+{% else -%}
+{% if neutron_plugin in ['ovs', 'ml2', 'Calico'] -%}
+service_plugins = router,firewall,lbaas,vpnaas,metering
+{% endif -%}
+{% endif -%}
+{% endif -%}
+
+{% if neutron_security_groups -%}
+allow_overlapping_ips = True
+{% if neutron_plugin == 'Calico' -%}
+neutron_firewall_driver = neutron.agent.linux.iptables_firewall.IptablesFirewallDriver
+{% else -%}
+neutron_firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
+{% endif -%}
+{% endif -%}
+
+{% if neutron_plugin == 'Calico' -%}
+dhcp_agents_per_network = 1000
+{% endif -%}
+
+notify_nova_on_port_status_changes = True
+notify_nova_on_port_data_changes = True
+nova_url = {{ nova_url }}
+nova_region_name = {{ region }}
+{% if auth_host -%}
+nova_admin_username = {{ admin_user }}
+nova_admin_tenant_id = {{ admin_tenant_id }}
+nova_admin_password = {{ admin_password }}
+nova_admin_auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v2.0
+{% endif -%}
+
+{% if sections and 'DEFAULT' in sections -%}
+{% for key, value in sections['DEFAULT'] -%}
+{{ key }} = {{ value }}
+{% endfor -%}
+{% endif %}
+
+{% include "section-zeromq" %}
+
+[quotas]
+{% if quota_driver -%}
+quota_driver = {{ quota_driver }}
+{% else -%}
+quota_driver = neutron.db.quota_db.DbQuotaDriver
+{% endif -%}
+{% if neutron_security_groups -%}
+quota_items = network,subnet,port,security_group,security_group_rule
+quota_security_group = {{ quota_security_group }}
+quota_security_group_rule = {{ quota_security_group_rule }}
+{% else -%}
+quota_items = network,subnet,port
+{% endif -%}
+quota_network = {{ quota_network }}
+quota_subnet = {{ quota_subnet }}
+quota_port = {{ quota_port }}
+quota_vip = {{ quota_vip }}
+quota_pool = {{ quota_pool }}
+quota_member = {{ quota_member }}
+quota_health_monitors = {{ quota_health_monitors }}
+quota_router = {{ quota_router }}
+quota_floatingip = {{ quota_floatingip }}
+
+[agent]
+root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
+
+{% include "section-keystone-authtoken" %}
+
+{% include "parts/section-database" %}
+
+{% include "section-rabbitmq-oslo" %}
+
+[oslo_concurrency]
+lock_path = $state_path/lock

templates/liberty/neutron_lbaas.conf

@@ -0,0 +1,19 @@
+# liberty
+###############################################################################
+# [ WARNING ]
+# Configuration file maintained by Juju. Local changes may be overwritten.
+###############################################################################
+# [service_auth]
+# auth_url = http://127.0.0.1:5000/v2.0
+# admin_tenant_name = %SERVICE_TENANT_NAME%
+# admin_user = %SERVICE_USER%
+# admin_password = %SERVICE_PASSWORD%
+# admin_user_domain = %SERVICE_USER_DOMAIN%
+# admin_project_domain = %SERVICE_PROJECT_DOMAIN%
+# region = %REGION%
+# service_name = lbaas
+# auth_version = 2
+
+[service_providers]
+service_provider=LOADBALANCER:Haproxy:neutron_lbaas.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
+# service_provider=LOADBALANCERV2:Haproxy:neutron_lbaas.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default

templates/liberty/neutron_vpnaas.conf

@@ -0,0 +1,7 @@
+# liberty
+###############################################################################
+# [ WARNING ]
+# Configuration file maintained by Juju. Local changes may be overwritten.
+###############################################################################
+[service_providers]
+service_provider=VPN:openswan:neutron_vpnaas.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default

tests/00-setup

@@ -4,9 +4,14 @@ set -ex
 
 sudo add-apt-repository --yes ppa:juju/stable
 sudo apt-get update --yes
-sudo apt-get install --yes python-amulet \
+sudo apt-get install --yes amulet \
+                           distro-info-data \
+                           python-cinderclient \
                            python-distro-info \
-                           python-neutronclient \
+                           python-glanceclient \
+                           python-heatclient \
                            python-keystoneclient \
+                           python-neutronclient \
                            python-novaclient \
-                           python-glanceclient
+                           python-pika \
+                           python-swiftclient

tests/020-basic-trusty-liberty

@@ -0,0 +1,11 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic neutron-api deployment on trusty-liberty."""
+
+from basic_deployment import NeutronAPIBasicDeployment
+
+if __name__ == '__main__':
+    deployment = NeutronAPIBasicDeployment(series='trusty',
+                                           openstack='cloud:trusty-liberty',
+                                           source='cloud:trusty-updates/liberty')
+    deployment.run_tests()

tests/021-basic-wily-liberty

@@ -0,0 +1,9 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic neutron-api deployment on wily-liberty."""
+
+from basic_deployment import NeutronAPIBasicDeployment
+
+if __name__ == '__main__':
+    deployment = NeutronAPIBasicDeployment(series='wily')
+    deployment.run_tests()

tests/README

@@ -1,6 +1,39 @@
 This directory provides Amulet tests that focus on verification of
 neutron-api deployments.
 
+test_* methods are called in lexical sort order, although each individual test
+should be idempotent, and expected to pass regardless of run order.
+
+Test name convention to ensure desired test order:
+    1xx service and endpoint checks
+    2xx relation checks
+    3xx config checks
+    4xx functional checks
+    9xx restarts and other final checks
+
+Common relation definitions:
+  - [ neutron-api, mysql ]
+  - [ neutron-api, rabbitmq-server ]
+  - [ neutron-api, nova-cloud-controller ]
+  - [ neutron-api, neutron-openvswitch ]
+  - [ neutron-api, keystone ]
+  - [ neutron-api, neutron-gateway ]
+
+Resultant relations of neutron-api service:
+    relations:
+      amqp:
+      - rabbitmq-server
+      cluster:
+      - neutron-api
+      identity-service:
+      - keystone
+      neutron-api:
+      - nova-cloud-controller
+      neutron-plugin-api:
+      - neutron-openvswitch
+      shared-db:
+      - mysql
+
 In order to run tests, you'll need charm-tools installed (in addition to
 juju, of course):
     sudo add-apt-repository ppa:juju/stable

tests/basic_deployment.py

@@ -1,43 +1,5 @@
-#!/usr/bin/python
-"""
-Basic neutron-api functional test.
-
-test_* methods are called in sort order.
-
-Convention to ensure desired test order:
-    1xx service and endpoint checks
-    2xx relation checks
-    3xx config checks
-    4xx functional checks
-    9xx restarts and other final checks
-
-Common relation definitions:
-  - [ neutron-api, mysql ]
-  - [ neutron-api, rabbitmq-server ]
-  - [ neutron-api, nova-cloud-controller ]
-  - [ neutron-api, neutron-openvswitch ]
-  - [ neutron-api, keystone ]
-  - [ neutron-api, neutron-gateway ]
-
-Resultant relations of neutron-api service:
-    relations:
-      amqp:
-      - rabbitmq-server
-      cluster:
-      - neutron-api
-      identity-service:
-      - keystone
-      neutron-api:
-      - nova-cloud-controller
-      neutron-plugin-api:  # not inspected due to
-      - neutron-openvswitch  # bug 1421388
-      shared-db:
-      - mysql
-"""
-
 import amulet
 import os
-import time
 import yaml
 
 from charmhelpers.contrib.openstack.amulet.deployment import (
@@ -46,8 +8,8 @@ from charmhelpers.contrib.openstack.amulet.deployment import (
 
 from charmhelpers.contrib.openstack.amulet.utils import (
     OpenStackAmuletUtils,
-    DEBUG, # flake8: noqa
-    ERROR
+    DEBUG,
+    # ERROR
 )
 
 # Use DEBUG to turn on debug logging
@@ -67,6 +29,11 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
         self._add_relations()
         self._configure_services()
         self._deploy()
+
+        u.log.info('Waiting on extended status checks...')
+        exclude_services = ['mysql']
+        self._auto_wait_for_status(exclude_services=exclude_services)
+
         self._initialize_tests()
 
     def _add_services(self):
@@ -78,7 +45,9 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
            """
         this_service = {'name': 'neutron-api'}
         other_services = [{'name': 'mysql'},
-                          {'name': 'rabbitmq-server'}, {'name': 'keystone'},
+                          {'name': 'rabbitmq-server'},
+                          {'name': 'keystone'},
+                          {'name': 'glance'},  # to satisfy workload status
                           {'name': 'neutron-openvswitch'},
                           {'name': 'nova-cloud-controller'},
                           {'name': 'neutron-gateway'},
@@ -94,14 +63,34 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
             'neutron-api:neutron-api': 'nova-cloud-controller:neutron-api',
             'neutron-api:neutron-plugin-api': 'neutron-gateway:'
                                               'neutron-plugin-api',
-            'neutron-api:neutron-plugin-api': 'neutron-openvswitch:'
-                                              'neutron-plugin-api',
             'neutron-api:identity-service': 'keystone:identity-service',
             'keystone:shared-db': 'mysql:shared-db',
-            'nova-compute:neutron-plugin': 'neutron-openvswitch:neutron-plugin',
+            'nova-compute:neutron-plugin': 'neutron-openvswitch:'
+                                           'neutron-plugin',
             'nova-cloud-controller:shared-db': 'mysql:shared-db',
+            'neutron-gateway:amqp': 'rabbitmq-server:amqp',
+            'nova-cloud-controller:amqp': 'rabbitmq-server:amqp',
+            'nova-compute:amqp': 'rabbitmq-server:amqp',
+            'neutron-openvswitch:amqp': 'rabbitmq-server:amqp',
+            'nova-cloud-controller:identity-service': 'keystone:'
+                                                      'identity-service',
+            'nova-cloud-controller:cloud-compute': 'nova-compute:'
+                                                   'cloud-compute',
+            'glance:identity-service': 'keystone:identity-service',
+            'glance:shared-db': 'mysql:shared-db',
+            'glance:amqp': 'rabbitmq-server:amqp',
+            'nova-compute:image-service': 'glance:image-service',
+            'nova-cloud-controller:image-service': 'glance:image-service',
+        }
+
+        # NOTE(beisner): relate this separately due to the resulting
+        # duplicate dictionary key if included in the relations dict.
+        relations_more = {
+            'neutron-api:neutron-plugin-api': 'neutron-openvswitch:'
+                                              'neutron-plugin-api',
         }
         super(NeutronAPIBasicDeployment, self)._add_relations(relations)
+        super(NeutronAPIBasicDeployment, self)._add_relations(relations_more)
 
     def _configure_services(self):
         """Configure all of the services."""
@@ -115,16 +104,16 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
                 openstack_origin_git = {
                     'repositories': [
                         {'name': 'requirements',
-                         'repository': 'git://github.com/openstack/requirements',
+                         'repository': 'git://github.com/openstack/requirements',  # noqa
                          'branch': branch},
                         {'name': 'neutron-fwaas',
-                         'repository': 'git://github.com/openstack/neutron-fwaas',
+                         'repository': 'git://github.com/openstack/neutron-fwaas',  # noqa
                          'branch': branch},
                         {'name': 'neutron-lbaas',
-                         'repository': 'git://github.com/openstack/neutron-lbaas',
+                         'repository': 'git://github.com/openstack/neutron-lbaas',  # noqa
                          'branch': branch},
                         {'name': 'neutron-vpnaas',
-                         'repository': 'git://github.com/openstack/neutron-vpnaas',
+                         'repository': 'git://github.com/openstack/neutron-vpnaas',  # noqa
                          'branch': branch},
                         {'name': 'neutron',
                          'repository': 'git://github.com/openstack/neutron',
@@ -154,7 +143,9 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
                     'http_proxy': amulet_http_proxy,
                     'https_proxy': amulet_http_proxy,
                 }
-            neutron_api_config['openstack-origin-git'] = yaml.dump(openstack_origin_git)
+            neutron_api_config['openstack-origin-git'] = \
+                yaml.dump(openstack_origin_git)
+
         keystone_config = {'admin-password': 'openstack',
                            'admin-token': 'ubuntutesting'}
         nova_cc_config = {'network-manager': 'Quantum',
@@ -171,58 +162,56 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
         self.keystone_sentry = self.d.sentry.unit['keystone/0']
         self.rabbitmq_sentry = self.d.sentry.unit['rabbitmq-server/0']
         self.nova_cc_sentry = self.d.sentry.unit['nova-cloud-controller/0']
-        self.neutron_gateway_sentry = self.d.sentry.unit['neutron-gateway/0']
+        self.neutron_gw_sentry = self.d.sentry.unit['neutron-gateway/0']
         self.neutron_api_sentry = self.d.sentry.unit['neutron-api/0']
+        self.neutron_ovs_sentry = self.d.sentry.unit['neutron-openvswitch/0']
         self.nova_compute_sentry = self.d.sentry.unit['nova-compute/0']
+
         u.log.debug('openstack release val: {}'.format(
             self._get_openstack_release()))
         u.log.debug('openstack release str: {}'.format(
             self._get_openstack_release_string()))
-        # Let things settle a bit before moving forward
-        time.sleep(30)
-
 
     def test_100_services(self):
         """Verify the expected services are running on the corresponding
            service units."""
         u.log.debug('Checking status of system services...')
-        # Fails vivid-kilo, bug 1454754
-        neutron_api_services = ['status neutron-server']
-        neutron_services = ['status neutron-dhcp-agent',
-                            'status neutron-lbaas-agent',
-                            'status neutron-metadata-agent',
-                            'status neutron-plugin-openvswitch-agent',
-                            'status neutron-ovs-cleanup']
+        neutron_api_services = ['neutron-server']
+        neutron_services = ['neutron-dhcp-agent',
+                            'neutron-lbaas-agent',
+                            'neutron-metadata-agent',
+                            'neutron-plugin-openvswitch-agent',
+                            'neutron-ovs-cleanup']
 
         if self._get_openstack_release() <= self.trusty_juno:
-            neutron_services.append('status neutron-vpn-agent')
+            neutron_services.append('neutron-vpn-agent')
 
         if self._get_openstack_release() < self.trusty_kilo:
             # Juno or earlier
-            neutron_services.append('status neutron-metering-agent')
+            neutron_services.append('neutron-metering-agent')
 
-        nova_cc_services = ['status nova-api-ec2',
-                            'status nova-api-os-compute',
-                            'status nova-objectstore',
-                            'status nova-cert',
-                            'status nova-scheduler',
-                            'status nova-conductor']
+        nova_cc_services = ['nova-api-ec2',
+                            'nova-api-os-compute',
+                            'nova-objectstore',
+                            'nova-cert',
+                            'nova-scheduler',
+                            'nova-conductor']
 
-        commands = {
-            self.mysql_sentry: ['status mysql'],
-            self.keystone_sentry: ['status keystone'],
+        services = {
+            self.mysql_sentry: ['mysql'],
+            self.keystone_sentry: ['keystone'],
             self.nova_cc_sentry: nova_cc_services,
-            self.neutron_gateway_sentry: neutron_services,
+            self.neutron_gw_sentry: neutron_services,
             self.neutron_api_sentry: neutron_api_services,
         }
 
-        ret = u.validate_services(commands)
+        ret = u.validate_services_by_name(services)
         if ret:
             amulet.raise_status(amulet.FAIL, msg=ret)
 
     def test_200_neutron_api_shared_db_relation(self):
         """Verify the neutron-api to mysql shared-db relation data"""
-        u.log.debug('Checking neutron-api:mysql relation data...')
+        u.log.debug('Checking neutron-api:mysql db relation data...')
         unit = self.neutron_api_sentry
         relation = ['shared-db', 'mysql:shared-db']
         expected = {
@@ -239,24 +228,17 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
 
     def test_201_shared_db_neutron_api_relation(self):
         """Verify the mysql to neutron-api shared-db relation data"""
-        u.log.debug('Checking mysql:neutron-api relation data...')
+        u.log.debug('Checking mysql:neutron-api db relation data...')
         unit = self.mysql_sentry
         relation = ['shared-db', 'neutron-api:shared-db']
         expected = {
             'db_host': u.valid_ip,
             'private-address': u.valid_ip,
+            'password': u.not_null
         }
 
-        if self._get_openstack_release() == self.precise_icehouse:
-            # Precise
-            expected['allowed_units'] = 'nova-cloud-controller/0 neutron-api/0'
-        else:
-            # Not Precise
-            expected['allowed_units'] = 'neutron-api/0'
-
         ret = u.validate_relation_data(unit, relation, expected)
-        rel_data = unit.relation('shared-db', 'neutron-api:shared-db')
-        if ret or 'password' not in rel_data:
+        if ret:
             message = u.relation_error('mysql shared-db', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
@@ -281,25 +263,25 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
         u.log.debug('Checking amqp:neutron-api relation data...')
         unit = self.rabbitmq_sentry
         relation = ['amqp', 'neutron-api:amqp']
-        rel_data = unit.relation('amqp', 'neutron-api:amqp')
         expected = {
             'hostname': u.valid_ip,
             'private-address': u.valid_ip,
+            'password': u.not_null
         }
 
         ret = u.validate_relation_data(unit, relation, expected)
-        if ret or not 'password' in rel_data:
+        if ret:
             message = u.relation_error('rabbitmq amqp', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_204_neutron_api_identity_relation(self):
+    def test_204_neutron_api_keystone_identity_relation(self):
         """Verify the neutron-api to keystone identity-service relation data"""
-        u.log.debug('Checking neutron-api:keystone relation data...')
+        u.log.debug('Checking neutron-api:keystone id relation data...')
         unit = self.neutron_api_sentry
         relation = ['identity-service', 'keystone:identity-service']
         api_ip = unit.relation('identity-service',
                                'keystone:identity-service')['private-address']
-        api_endpoint = "http://%s:9696" % (api_ip)
+        api_endpoint = 'http://{}:9696'.format(api_ip)
         expected = {
             'private-address': u.valid_ip,
             'quantum_region': 'RegionOne',
@@ -316,12 +298,12 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
 
     def test_205_keystone_neutron_api_identity_relation(self):
         """Verify the keystone to neutron-api identity-service relation data"""
-        u.log.debug('Checking keystone:neutron-api relation data...')
+        u.log.debug('Checking keystone:neutron-api id relation data...')
         unit = self.keystone_sentry
         relation = ['identity-service', 'neutron-api:identity-service']
-        id_relation = unit.relation('identity-service',
-                                    'neutron-api:identity-service')
-        id_ip = id_relation['private-address']
+        rel_ks_id = unit.relation('identity-service',
+                                  'neutron-api:identity-service')
+        id_ip = rel_ks_id['private-address']
         expected = {
             'admin_token': 'ubuntutesting',
             'auth_host': id_ip,
@@ -335,12 +317,46 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('neutron-api identity-service', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_206_neutron_api_plugin_relation(self):
+    def test_206_neutron_api_neutron_ovs_plugin_api_relation(self):
         """Verify neutron-api to neutron-openvswitch neutron-plugin-api"""
-        u.log.debug('Checking neutron-api:neutron-ovs relation data...')
+        u.log.debug('Checking neutron-api:neutron-ovs plugin-api '
+                    'relation data...')
         unit = self.neutron_api_sentry
         relation = ['neutron-plugin-api',
                     'neutron-openvswitch:neutron-plugin-api']
+
+        u.log.debug(unit.relation(relation[0], relation[1]))
+        expected = {
+            'auth_host': u.valid_ip,
+            'auth_port': '35357',
+            'auth_protocol': 'http',
+            'enable-dvr': 'False',
+            'enable-l3ha': 'False',
+            'l2-population': 'True',
+            'neutron-security-groups': 'False',
+            'overlay-network-type': 'gre',
+            'private-address': u.valid_ip,
+            'region': 'RegionOne',
+            'service_host': u.valid_ip,
+            'service_password': u.not_null,
+            'service_port': '5000',
+            'service_protocol': 'http',
+            'service_tenant': 'services',
+            'service_username': 'quantum',
+        }
+        ret = u.validate_relation_data(unit, relation, expected)
+        if ret:
+            message = u.relation_error(
+                'neutron-api neutron-ovs neutronplugin-api', ret)
+            amulet.raise_status(amulet.FAIL, msg=message)
+
+    def test_207_neutron_ovs_neutron_api_plugin_api_relation(self):
+        """Verify neutron-openvswitch to neutron-api neutron-plugin-api"""
+        u.log.debug('Checking neutron-ovs:neutron-api plugin-api '
+                    'relation data...')
+        unit = self.neutron_ovs_sentry
+        relation = ['neutron-plugin-api',
+                    'neutron-api:neutron-plugin-api']
         expected = {
             'private-address': u.valid_ip,
         }
@@ -349,14 +365,14 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('neutron-api neutron-plugin-api', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_207_neutron_api_novacc_relation(self):
+    def test_208_neutron_api_novacc_relation(self):
         """Verify the neutron-api to nova-cloud-controller relation data"""
         u.log.debug('Checking neutron-api:novacc relation data...')
         unit = self.neutron_api_sentry
         relation = ['neutron-api', 'nova-cloud-controller:neutron-api']
         api_ip = unit.relation('identity-service',
                                'keystone:identity-service')['private-address']
-        api_endpoint = "http://%s:9696" % (api_ip)
+        api_endpoint = 'http://{}:9696'.format(api_ip)
         expected = {
             'private-address': api_ip,
             'neutron-plugin': 'ovs',
@@ -368,14 +384,14 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('neutron-api neutron-api', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_208_novacc_neutron_api_relation(self):
+    def test_209_novacc_neutron_api_relation(self):
         """Verify the nova-cloud-controller to neutron-api relation data"""
         u.log.debug('Checking novacc:neutron-api relation data...')
         unit = self.nova_cc_sentry
         relation = ['neutron-api', 'neutron-api:neutron-api']
         cc_ip = unit.relation('neutron-api',
                               'neutron-api:neutron-api')['private-address']
-        cc_endpoint = "http://%s:8774/v2" % (cc_ip)
+        cc_endpoint = 'http://{}:8774/v2'.format(cc_ip)
         expected = {
             'private-address': cc_ip,
             'nova_url': cc_endpoint,
@@ -385,34 +401,6 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
             message = u.relation_error('nova-cc neutron-api', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    # XXX Test missing to examine the relation data neutron-openvswitch is
-    #     receiving. Current;y this data cannot be interegated due to
-    #     Bug#1421388
-
-    def test_900_restart_on_config_change(self):
-        """Verify that the specified services are restarted when the config
-           is changed.
-
-           Note(coreycb): The method name with the _z_ is a little odd
-           but it forces the test to run last.  It just makes things
-           easier because restarting services requires re-authorization.
-           """
-        u.log.debug('Checking novacc neutron-api relation data...')
-        conf = '/etc/neutron/neutron.conf'
-        services = ['neutron-server']
-        u.log.debug('Making config change on neutron-api service...')
-        self.d.configure('neutron-api', {'use-syslog': 'True'})
-        stime = 60
-        for s in services:
-            u.log.debug("Checking that service restarted: {}".format(s))
-            if not u.service_restarted(self.neutron_api_sentry, s, conf,
-                                       pgrep_full=True, sleep_time=stime):
-                self.d.configure('neutron-api', {'use-syslog': 'False'})
-                msg = "service {} didn't restart after config change".format(s)
-                amulet.raise_status(amulet.FAIL, msg=msg)
-            stime = 0
-        self.d.configure('neutron-api', {'use-syslog': 'False'})
-
     def test_300_neutron_config(self):
         """Verify the data in the neutron config file."""
         u.log.debug('Checking neutron.conf config file data...')
@@ -421,16 +409,17 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
                                                    'neutron-api:neutron-api')
         rabbitmq_relation = self.rabbitmq_sentry.relation('amqp',
                                                           'neutron-api:amqp')
-        ks_rel = self.keystone_sentry.relation('identity-service',
-                                               'neutron-api:identity-service')
+        rel_napi_ks = self.keystone_sentry.relation(
+            'identity-service', 'neutron-api:identity-service')
 
-        nova_auth_url = '%s://%s:%s/v2.0' % (ks_rel['auth_protocol'],
-                                             ks_rel['auth_host'],
-                                             ks_rel['auth_port'])
-        db_relation = self.mysql_sentry.relation('shared-db',
+        nova_auth_url = '{}://{}:{}/v2.0'.format(rel_napi_ks['auth_protocol'],
+                                                 rel_napi_ks['auth_host'],
+                                                 rel_napi_ks['auth_port'])
+        rel_napi_db = self.mysql_sentry.relation('shared-db',
                                                  'neutron-api:shared-db')
-        db_conn = 'mysql://neutron:%s@%s/neutron' % (db_relation['password'],
-                                                     db_relation['db_host'])
+        db_conn = 'mysql://neutron:{}@{}/neutron'.format(
+            rel_napi_db['password'], rel_napi_db['db_host'])
+
         conf = '/etc/neutron/neutron.conf'
         expected = {
             'DEFAULT': {
@@ -439,16 +428,16 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
                 'bind_port': '9686',
                 'nova_url': cc_relation['nova_url'],
                 'nova_region_name': 'RegionOne',
-                'nova_admin_username': ks_rel['service_username'],
-                'nova_admin_tenant_id': ks_rel['service_tenant_id'],
-                'nova_admin_password': ks_rel['service_password'],
+                'nova_admin_username': rel_napi_ks['service_username'],
+                'nova_admin_tenant_id': rel_napi_ks['service_tenant_id'],
+                'nova_admin_password': rel_napi_ks['service_password'],
                 'nova_admin_auth_url': nova_auth_url,
             },
             'keystone_authtoken': {
                 'signing_dir': '/var/cache/neutron',
                 'admin_tenant_name': 'services',
                 'admin_user': 'quantum',
-                'admin_password': ks_rel['service_password'],
+                'admin_password': rel_napi_ks['service_password'],
             },
             'database': {
                 'connection': db_conn,
@@ -457,36 +446,28 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
 
         if self._get_openstack_release() >= self.trusty_kilo:
             # Kilo or later
-            expected.update(
-                {
-                    'oslo_messaging_rabbit': {
-                        'rabbit_userid': 'neutron',
-                        'rabbit_virtual_host': 'openstack',
-                        'rabbit_password': rabbitmq_relation['password'],
-                        'rabbit_host': rabbitmq_relation['hostname']
-                    }
-                }
-            )
+            expected['oslo_messaging_rabbit'] = {
+                'rabbit_userid': 'neutron',
+                'rabbit_virtual_host': 'openstack',
+                'rabbit_password': rabbitmq_relation['password'],
+                'rabbit_host': rabbitmq_relation['hostname']
+            }
         else:
             # Juno or earlier
-            expected['DEFAULT'].update(
-                {
-                    'rabbit_userid': 'neutron',
-                    'rabbit_virtual_host': 'openstack',
-                    'rabbit_password': rabbitmq_relation['password'],
-                    'rabbit_host': rabbitmq_relation['hostname']
-                }
-            )
-            expected['keystone_authtoken'].update(
-                {
-                    'service_protocol': ks_rel['service_protocol'],
-                    'service_host': ks_rel['service_host'],
-                    'service_port': ks_rel['service_port'],
-                    'auth_host': ks_rel['auth_host'],
-                    'auth_port': ks_rel['auth_port'],
-                    'auth_protocol':  ks_rel['auth_protocol']
-                }
-            )
+            expected['DEFAULT'].update({
+                'rabbit_userid': 'neutron',
+                'rabbit_virtual_host': 'openstack',
+                'rabbit_password': rabbitmq_relation['password'],
+                'rabbit_host': rabbitmq_relation['hostname']
+            })
+            expected['keystone_authtoken'].update({
+                'service_protocol': rel_napi_ks['service_protocol'],
+                'service_host': rel_napi_ks['service_host'],
+                'service_port': rel_napi_ks['service_port'],
+                'auth_host': rel_napi_ks['auth_host'],
+                'auth_port': rel_napi_ks['auth_port'],
+                'auth_protocol':  rel_napi_ks['auth_protocol']
+            })
 
         for section, pairs in expected.iteritems():
             ret = u.validate_config_data(unit, conf, section, pairs)
@@ -504,8 +485,8 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
 
         expected = {
             'ml2': {
-                'type_drivers': 'gre,vlan,flat',
-                'tenant_network_types': 'gre,vlan,flat',
+                'type_drivers': 'gre,vlan,flat,local',
+                'tenant_network_types': 'gre,vlan,flat,local',
             },
             'ml2_type_gre': {
                 'tunnel_id_ranges': '1:1000'
@@ -527,21 +508,51 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
 
         if self._get_openstack_release() >= self.trusty_kilo:
             # Kilo or later
-            expected['ml2'].update(
-                {
-                    'mechanism_drivers': 'openvswitch,l2population'
-                }
-            )
+            expected['ml2'].update({
+                'mechanism_drivers': 'openvswitch,l2population'
+            })
         else:
             # Juno or earlier
-            expected['ml2'].update(
-                {
-                    'mechanism_drivers': 'openvswitch,hyperv,l2population'
-                }
-            )
+            expected['ml2'].update({
+                'mechanism_drivers': 'openvswitch,hyperv,l2population'
+            })
 
         for section, pairs in expected.iteritems():
             ret = u.validate_config_data(unit, conf, section, pairs)
             if ret:
                 message = "ml2 config error: {}".format(ret)
                 amulet.raise_status(amulet.FAIL, msg=message)
+
+    def test_900_restart_on_config_change(self):
+        """Verify that the specified services are restarted when the
+        config is changed."""
+
+        sentry = self.neutron_api_sentry
+        juju_service = 'neutron-api'
+
+        # Expected default and alternate values
+        set_default = {'debug': 'False'}
+        set_alternate = {'debug': 'True'}
+
+        # Services which are expected to restart upon config change,
+        # and corresponding config files affected by the change
+        services = {'neutron-server': '/etc/neutron/neutron.conf'}
+
+        # Make config change, check for service restarts
+        u.log.debug('Making config change on {}...'.format(juju_service))
+        mtime = u.get_sentry_time(sentry)
+        self.d.configure(juju_service, set_alternate)
+
+        for s, conf_file in services.iteritems():
+            u.log.debug("Checking that service restarted: {}".format(s))
+            if not u.validate_service_config_changed(sentry, mtime, s,
+                                                     conf_file,
+                                                     retry_count=4,
+                                                     retry_sleep_time=20,
+                                                     sleep_time=20):
+                self.d.configure(juju_service, set_default)
+                msg = "service {} didn't restart after config change".format(s)
+                amulet.raise_status(amulet.FAIL, msg=msg)
+
+        self.d.configure(juju_service, set_default)
+        u.log.debug('OK')

tests/charmhelpers/contrib/amulet/deployment.py

@@ -51,7 +51,8 @@ class AmuletDeployment(object):
         if 'units' not in this_service:
             this_service['units'] = 1
 
-        self.d.add(this_service['name'], units=this_service['units'])
+        self.d.add(this_service['name'], units=this_service['units'],
+                   constraints=this_service.get('constraints'))
 
         for svc in other_services:
             if 'location' in svc:
@@ -64,7 +65,8 @@ class AmuletDeployment(object):
             if 'units' not in svc:
                 svc['units'] = 1
 
-            self.d.add(svc['name'], charm=branch_location, units=svc['units'])
+            self.d.add(svc['name'], charm=branch_location, units=svc['units'],
+                       constraints=svc.get('constraints'))
 
     def _add_relations(self, relations):
         """Add all of the relations for the services."""

tests/charmhelpers/contrib/amulet/utils.py

@@ -19,9 +19,11 @@ import json
 import logging
 import os
 import re
+import socket
 import subprocess
 import sys
 import time
+import uuid
 
 import amulet
 import distro_info
@@ -114,7 +116,7 @@ class AmuletUtils(object):
         # /!\ DEPRECATION WARNING (beisner):
         # New and existing tests should be rewritten to use
         # validate_services_by_name() as it is aware of init systems.
-        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+        self.log.warn('DEPRECATION WARNING:  use '
                       'validate_services_by_name instead of validate_services '
                       'due to init system differences.')
 
@@ -269,33 +271,52 @@ class AmuletUtils(object):
         """Get last modification time of directory."""
         return sentry_unit.directory_stat(directory)['mtime']
 
-    def _get_proc_start_time(self, sentry_unit, service, pgrep_full=False):
-        """Get process' start time.
+    def _get_proc_start_time(self, sentry_unit, service, pgrep_full=None):
+        """Get start time of a process based on the last modification time
+           of the /proc/pid directory.
 
-           Determine start time of the process based on the last modification
-           time of the /proc/pid directory. If pgrep_full is True, the process
-           name is matched against the full command line.
-           """
-        if pgrep_full:
-            cmd = 'pgrep -o -f {}'.format(service)
-        else:
-            cmd = 'pgrep -o {}'.format(service)
-        cmd = cmd + '  | grep  -v pgrep || exit 0'
-        cmd_out = sentry_unit.run(cmd)
-        self.log.debug('CMDout: ' + str(cmd_out))
-        if cmd_out[0]:
-            self.log.debug('Pid for %s %s' % (service, str(cmd_out[0])))
-            proc_dir = '/proc/{}'.format(cmd_out[0].strip())
-            return self._get_dir_mtime(sentry_unit, proc_dir)
+        :sentry_unit:  The sentry unit to check for the service on
+        :service:  service name to look for in process table
+        :pgrep_full:  [Deprecated] Use full command line search mode with pgrep
+        :returns:  epoch time of service process start
+        :param commands:  list of bash commands
+        :param sentry_units:  list of sentry unit pointers
+        :returns:  None if successful; Failure message otherwise
+        """
+        if pgrep_full is not None:
+            # /!\ DEPRECATION WARNING (beisner):
+            # No longer implemented, as pidof is now used instead of pgrep.
+            # https://bugs.launchpad.net/charm-helpers/+bug/1474030
+            self.log.warn('DEPRECATION WARNING:  pgrep_full bool is no '
+                          'longer implemented re: lp 1474030.')
+
+        pid_list = self.get_process_id_list(sentry_unit, service)
+        pid = pid_list[0]
+        proc_dir = '/proc/{}'.format(pid)
+        self.log.debug('Pid for {} on {}: {}'.format(
+            service, sentry_unit.info['unit_name'], pid))
+
+        return self._get_dir_mtime(sentry_unit, proc_dir)
 
     def service_restarted(self, sentry_unit, service, filename,
-                          pgrep_full=False, sleep_time=20):
+                          pgrep_full=None, sleep_time=20):
         """Check if service was restarted.
 
            Compare a service's start time vs a file's last modification time
            (such as a config file for that service) to determine if the service
            has been restarted.
            """
+        # /!\ DEPRECATION WARNING (beisner):
+        # This method is prone to races in that no before-time is known.
+        # Use validate_service_config_changed instead.
+
+        # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
+        # used instead of pgrep.  pgrep_full is still passed through to ensure
+        # deprecation WARNS.  lp1474030
+        self.log.warn('DEPRECATION WARNING:  use '
+                      'validate_service_config_changed instead of '
+                      'service_restarted due to known races.')
+
         time.sleep(sleep_time)
         if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >=
                 self._get_file_mtime(sentry_unit, filename)):
@@ -304,78 +325,122 @@ class AmuletUtils(object):
             return False
 
     def service_restarted_since(self, sentry_unit, mtime, service,
-                                pgrep_full=False, sleep_time=20,
-                                retry_count=2):
+                                pgrep_full=None, sleep_time=20,
+                                retry_count=30, retry_sleep_time=10):
         """Check if service was been started after a given time.
 
         Args:
           sentry_unit (sentry): The sentry unit to check for the service on
           mtime (float): The epoch time to check against
           service (string): service name to look for in process table
-          pgrep_full (boolean): Use full command line search mode with pgrep
-          sleep_time (int): Seconds to sleep before looking for process
-          retry_count (int): If service is not found, how many times to retry
+          pgrep_full: [Deprecated] Use full command line search mode with pgrep
+          sleep_time (int): Initial sleep time (s) before looking for file
+          retry_sleep_time (int): Time (s) to sleep between retries
+          retry_count (int): If file is not found, how many times to retry
 
         Returns:
           bool: True if service found and its start time it newer than mtime,
                 False if service is older than mtime or if service was
                 not found.
         """
-        self.log.debug('Checking %s restarted since %s' % (service, mtime))
+        # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
+        # used instead of pgrep.  pgrep_full is still passed through to ensure
+        # deprecation WARNS.  lp1474030
+
+        unit_name = sentry_unit.info['unit_name']
+        self.log.debug('Checking that %s service restarted since %s on '
+                       '%s' % (service, mtime, unit_name))
         time.sleep(sleep_time)
-        proc_start_time = self._get_proc_start_time(sentry_unit, service,
-                                                    pgrep_full)
-        while retry_count > 0 and not proc_start_time:
-            self.log.debug('No pid file found for service %s, will retry %i '
-                           'more times' % (service, retry_count))
-            time.sleep(30)
-            proc_start_time = self._get_proc_start_time(sentry_unit, service,
-                                                        pgrep_full)
-            retry_count = retry_count - 1
+        proc_start_time = None
+        tries = 0
+        while tries <= retry_count and not proc_start_time:
+            try:
+                proc_start_time = self._get_proc_start_time(sentry_unit,
+                                                            service,
+                                                            pgrep_full)
+                self.log.debug('Attempt {} to get {} proc start time on {} '
+                               'OK'.format(tries, service, unit_name))
+            except IOError as e:
+                # NOTE(beisner) - race avoidance, proc may not exist yet.
+                # https://bugs.launchpad.net/charm-helpers/+bug/1474030
+                self.log.debug('Attempt {} to get {} proc start time on {} '
+                               'failed\n{}'.format(tries, service,
+                                                   unit_name, e))
+                time.sleep(retry_sleep_time)
+                tries += 1
 
         if not proc_start_time:
             self.log.warn('No proc start time found, assuming service did '
                           'not start')
             return False
         if proc_start_time >= mtime:
-            self.log.debug('proc start time is newer than provided mtime'
-                           '(%s >= %s)' % (proc_start_time, mtime))
+            self.log.debug('Proc start time is newer than provided mtime'
+                           '(%s >= %s) on %s (OK)' % (proc_start_time,
+                                                      mtime, unit_name))
             return True
         else:
-            self.log.warn('proc start time (%s) is older than provided mtime '
-                          '(%s), service did not restart' % (proc_start_time,
-                                                             mtime))
+            self.log.warn('Proc start time (%s) is older than provided mtime '
+                          '(%s) on %s, service did not '
+                          'restart' % (proc_start_time, mtime, unit_name))
             return False
 
     def config_updated_since(self, sentry_unit, filename, mtime,
-                             sleep_time=20):
+                             sleep_time=20, retry_count=30,
+                             retry_sleep_time=10):
         """Check if file was modified after a given time.
 
         Args:
           sentry_unit (sentry): The sentry unit to check the file mtime on
           filename (string): The file to check mtime of
           mtime (float): The epoch time to check against
-          sleep_time (int): Seconds to sleep before looking for process
+          sleep_time (int): Initial sleep time (s) before looking for file
+          retry_sleep_time (int): Time (s) to sleep between retries
+          retry_count (int): If file is not found, how many times to retry
 
         Returns:
           bool: True if file was modified more recently than mtime, False if
-                file was modified before mtime,
+                file was modified before mtime, or if file not found.
         """
-        self.log.debug('Checking %s updated since %s' % (filename, mtime))
+        unit_name = sentry_unit.info['unit_name']
+        self.log.debug('Checking that %s updated since %s on '
+                       '%s' % (filename, mtime, unit_name))
         time.sleep(sleep_time)
-        file_mtime = self._get_file_mtime(sentry_unit, filename)
+        file_mtime = None
+        tries = 0
+        while tries <= retry_count and not file_mtime:
+            try:
+                file_mtime = self._get_file_mtime(sentry_unit, filename)
+                self.log.debug('Attempt {} to get {} file mtime on {} '
+                               'OK'.format(tries, filename, unit_name))
+            except IOError as e:
+                # NOTE(beisner) - race avoidance, file may not exist yet.
+                # https://bugs.launchpad.net/charm-helpers/+bug/1474030
+                self.log.debug('Attempt {} to get {} file mtime on {} '
+                               'failed\n{}'.format(tries, filename,
+                                                   unit_name, e))
+                time.sleep(retry_sleep_time)
+                tries += 1
+
+        if not file_mtime:
+            self.log.warn('Could not determine file mtime, assuming '
+                          'file does not exist')
+            return False
+
         if file_mtime >= mtime:
             self.log.debug('File mtime is newer than provided mtime '
-                           '(%s >= %s)' % (file_mtime, mtime))
+                           '(%s >= %s) on %s (OK)' % (file_mtime,
+                                                      mtime, unit_name))
             return True
         else:
-            self.log.warn('File mtime %s is older than provided mtime %s'
-                          % (file_mtime, mtime))
+            self.log.warn('File mtime is older than provided mtime'
+                          '(%s < on %s) on %s' % (file_mtime,
+                                                  mtime, unit_name))
             return False
 
     def validate_service_config_changed(self, sentry_unit, mtime, service,
-                                        filename, pgrep_full=False,
-                                        sleep_time=20, retry_count=2):
+                                        filename, pgrep_full=None,
+                                        sleep_time=20, retry_count=30,
+                                        retry_sleep_time=10):
         """Check service and file were updated after mtime
 
         Args:
@@ -383,9 +448,10 @@ class AmuletUtils(object):
           mtime (float): The epoch time to check against
           service (string): service name to look for in process table
           filename (string): The file to check mtime of
-          pgrep_full (boolean): Use full command line search mode with pgrep
-          sleep_time (int): Seconds to sleep before looking for process
+          pgrep_full: [Deprecated] Use full command line search mode with pgrep
+          sleep_time (int): Initial sleep in seconds to pass to test helpers
           retry_count (int): If service is not found, how many times to retry
+          retry_sleep_time (int): Time in seconds to wait between retries
 
         Typical Usage:
             u = OpenStackAmuletUtils(ERROR)
@@ -402,15 +468,27 @@ class AmuletUtils(object):
                 mtime, False if service is older than mtime or if service was
                 not found or if filename was modified before mtime.
         """
-        self.log.debug('Checking %s restarted since %s' % (service, mtime))
-        time.sleep(sleep_time)
-        service_restart = self.service_restarted_since(sentry_unit, mtime,
-                                                       service,
-                                                       pgrep_full=pgrep_full,
-                                                       sleep_time=0,
-                                                       retry_count=retry_count)
-        config_update = self.config_updated_since(sentry_unit, filename, mtime,
-                                                  sleep_time=0)
+
+        # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
+        # used instead of pgrep.  pgrep_full is still passed through to ensure
+        # deprecation WARNS.  lp1474030
+
+        service_restart = self.service_restarted_since(
+            sentry_unit, mtime,
+            service,
+            pgrep_full=pgrep_full,
+            sleep_time=sleep_time,
+            retry_count=retry_count,
+            retry_sleep_time=retry_sleep_time)
+
+        config_update = self.config_updated_since(
+            sentry_unit,
+            filename,
+            mtime,
+            sleep_time=sleep_time,
+            retry_count=retry_count,
+            retry_sleep_time=retry_sleep_time)
+
         return service_restart and config_update
 
     def get_sentry_time(self, sentry_unit):
@@ -428,7 +506,6 @@ class AmuletUtils(object):
         """Return a list of all Ubuntu releases in order of release."""
         _d = distro_info.UbuntuDistroInfo()
         _release_list = _d.all
-        self.log.debug('Ubuntu release list: {}'.format(_release_list))
         return _release_list
 
     def file_to_url(self, file_rel_path):
@@ -568,6 +645,142 @@ class AmuletUtils(object):
 
         return None
 
+    def validate_sectionless_conf(self, file_contents, expected):
+        """A crude conf parser.  Useful to inspect configuration files which
+        do not have section headers (as would be necessary in order to use
+        the configparser).  Such as openstack-dashboard or rabbitmq confs."""
+        for line in file_contents.split('\n'):
+            if '=' in line:
+                args = line.split('=')
+                if len(args) <= 1:
+                    continue
+                key = args[0].strip()
+                value = args[1].strip()
+                if key in expected.keys():
+                    if expected[key] != value:
+                        msg = ('Config mismatch.  Expected, actual:  {}, '
+                               '{}'.format(expected[key], value))
+                        amulet.raise_status(amulet.FAIL, msg=msg)
+
+    def get_unit_hostnames(self, units):
+        """Return a dict of juju unit names to hostnames."""
+        host_names = {}
+        for unit in units:
+            host_names[unit.info['unit_name']] = \
+                str(unit.file_contents('/etc/hostname').strip())
+        self.log.debug('Unit host names: {}'.format(host_names))
+        return host_names
+
+    def run_cmd_unit(self, sentry_unit, cmd):
+        """Run a command on a unit, return the output and exit code."""
+        output, code = sentry_unit.run(cmd)
+        if code == 0:
+            self.log.debug('{} `{}` command returned {} '
+                           '(OK)'.format(sentry_unit.info['unit_name'],
+                                         cmd, code))
+        else:
+            msg = ('{} `{}` command returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+        return str(output), code
+
+    def file_exists_on_unit(self, sentry_unit, file_name):
+        """Check if a file exists on a unit."""
+        try:
+            sentry_unit.file_stat(file_name)
+            return True
+        except IOError:
+            return False
+        except Exception as e:
+            msg = 'Error checking file {}: {}'.format(file_name, e)
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+    def file_contents_safe(self, sentry_unit, file_name,
+                           max_wait=60, fatal=False):
+        """Get file contents from a sentry unit.  Wrap amulet file_contents
+        with retry logic to address races where a file checks as existing,
+        but no longer exists by the time file_contents is called.
+        Return None if file not found. Optionally raise if fatal is True."""
+        unit_name = sentry_unit.info['unit_name']
+        file_contents = False
+        tries = 0
+        while not file_contents and tries < (max_wait / 4):
+            try:
+                file_contents = sentry_unit.file_contents(file_name)
+            except IOError:
+                self.log.debug('Attempt {} to open file {} from {} '
+                               'failed'.format(tries, file_name,
+                                               unit_name))
+                time.sleep(4)
+                tries += 1
+
+        if file_contents:
+            return file_contents
+        elif not fatal:
+            return None
+        elif fatal:
+            msg = 'Failed to get file contents from unit.'
+            amulet.raise_status(amulet.FAIL, msg)
+
+    def port_knock_tcp(self, host="localhost", port=22, timeout=15):
+        """Open a TCP socket to check for a listening sevice on a host.
+
+        :param host: host name or IP address, default to localhost
+        :param port: TCP port number, default to 22
+        :param timeout: Connect timeout, default to 15 seconds
+        :returns: True if successful, False if connect failed
+        """
+
+        # Resolve host name if possible
+        try:
+            connect_host = socket.gethostbyname(host)
+            host_human = "{} ({})".format(connect_host, host)
+        except socket.error as e:
+            self.log.warn('Unable to resolve address: '
+                          '{} ({}) Trying anyway!'.format(host, e))
+            connect_host = host
+            host_human = connect_host
+
+        # Attempt socket connection
+        try:
+            knock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            knock.settimeout(timeout)
+            knock.connect((connect_host, port))
+            knock.close()
+            self.log.debug('Socket connect OK for host '
+                           '{} on port {}.'.format(host_human, port))
+            return True
+        except socket.error as e:
+            self.log.debug('Socket connect FAIL for'
+                           ' {} port {} ({})'.format(host_human, port, e))
+            return False
+
+    def port_knock_units(self, sentry_units, port=22,
+                         timeout=15, expect_success=True):
+        """Open a TCP socket to check for a listening sevice on each
+        listed juju unit.
+
+        :param sentry_units: list of sentry unit pointers
+        :param port: TCP port number, default to 22
+        :param timeout: Connect timeout, default to 15 seconds
+        :expect_success: True by default, set False to invert logic
+        :returns: None if successful, Failure message otherwise
+        """
+        for unit in sentry_units:
+            host = unit.info['public-address']
+            connected = self.port_knock_tcp(host, port, timeout)
+            if not connected and expect_success:
+                return 'Socket connect failed.'
+            elif connected and not expect_success:
+                return 'Socket connected unexpectedly.'
+
+    def get_uuid_epoch_stamp(self):
+        """Returns a stamp string based on uuid4 and epoch time.  Useful in
+        generating test messages which need to be unique-ish."""
+        return '[{}-{}]'.format(uuid.uuid4(), time.time())
+
+# amulet juju action helpers:
     def run_action(self, unit_sentry, action,
                    _check_output=subprocess.check_output):
         """Run the named action on a given unit sentry.
@@ -594,3 +807,12 @@ class AmuletUtils(object):
         output = _check_output(command, universal_newlines=True)
         data = json.loads(output)
         return data.get(u"status") == "completed"
+
+    def status_get(self, unit):
+        """Return the current service status of this unit."""
+        raw_status, return_code = unit.run(
+            "status-get --format=json --include-data")
+        if return_code != 0:
+            return ("unknown", "")
+        status = json.loads(raw_status)
+        return (status["status"], status["message"])

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import re
 import six
 from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
@@ -44,20 +45,31 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            Determine if the local branch being tested is derived from its
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
+
+        # Charms outside the lp:~openstack-charmers namespace
         base_charms = ['mysql', 'mongodb', 'nrpe']
 
+        # Force these charms to current series even when using an older series.
+        # ie. Use trusty/nrpe even when series is precise, as the P charm
+        # does not possess the necessary external master config and hooks.
+        force_series_current = ['nrpe']
+
         if self.series in ['precise', 'trusty']:
             base_series = self.series
         else:
             base_series = self.current_next
 
-        if self.stable:
-            for svc in other_services:
+        for svc in other_services:
+            if svc['name'] in force_series_current:
+                base_series = self.current_next
+            # If a location has been explicitly set, use it
+            if svc.get('location'):
+                continue
+            if self.stable:
                 temp = 'lp:charms/{}/{}'
                 svc['location'] = temp.format(base_series,
                                               svc['name'])
-        else:
-            for svc in other_services:
+            else:
                 if svc['name'] in base_charms:
                     temp = 'lp:charms/{}/{}'
                     svc['location'] = temp.format(base_series,
@@ -66,6 +78,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
                     temp = 'lp:~openstack-charmers/charms/{}/{}/next'
                     svc['location'] = temp.format(self.current_next,
                                                   svc['name'])
+
         return other_services
 
     def _add_services(self, this_service, other_services):
@@ -77,21 +90,23 @@ class OpenStackAmuletDeployment(AmuletDeployment):
 
         services = other_services
         services.append(this_service)
+
+        # Charms which should use the source config option
         use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
                       'ceph-osd', 'ceph-radosgw']
-        # Most OpenStack subordinate charms do not expose an origin option
-        # as that is controlled by the principle.
-        ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
+
+        # Charms which can not use openstack-origin, ie. many subordinates
+        no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
 
         if self.openstack:
             for svc in services:
-                if svc['name'] not in use_source + ignore:
+                if svc['name'] not in use_source + no_origin:
                     config = {'openstack-origin': self.openstack}
                     self.d.configure(svc['name'], config)
 
         if self.source:
             for svc in services:
-                if svc['name'] in use_source and svc['name'] not in ignore:
+                if svc['name'] in use_source and svc['name'] not in no_origin:
                     config = {'source': self.source}
                     self.d.configure(svc['name'], config)
 
@@ -100,6 +115,45 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         for service, config in six.iteritems(configs):
             self.d.configure(service, config)
 
+    def _auto_wait_for_status(self, message=None, exclude_services=None,
+                              timeout=1800):
+        """Wait for all units to have a specific extended status, except
+        for any defined as excluded.  Unless specified via message, any
+        status containing any case of 'ready' will be considered a match.
+
+        Examples of message usage:
+
+          Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
+              message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
+
+          Wait for all units to reach this status (exact match):
+              message = 'Unit is ready'
+
+          Wait for all units to reach any one of these (exact match):
+              message = re.compile('Unit is ready|OK|Ready')
+
+          Wait for at least one unit to reach this status (exact match):
+              message = {'ready'}
+
+        See Amulet's sentry.wait_for_messages() for message usage detail.
+        https://github.com/juju/amulet/blob/master/amulet/sentry.py
+
+        :param message: Expected status match
+        :param exclude_services: List of juju service names to ignore
+        :param timeout: Maximum time in seconds to wait for status match
+        :returns: None.  Raises if timeout is hit.
+        """
+
+        if not message:
+            message = re.compile('.*ready.*', re.IGNORECASE)
+
+        if not exclude_services:
+            exclude_services = []
+
+        services = list(set(self.d.services.keys()) - set(exclude_services))
+        service_messages = {service: message for service in services}
+        self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+
     def _get_openstack_release(self):
         """Get openstack release.
 

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -27,6 +27,7 @@ import glanceclient.v1.client as glance_client
 import heatclient.v1.client as heat_client
 import keystoneclient.v2_0 as keystone_client
 import novaclient.v1_1.client as nova_client
+import pika
 import swiftclient
 
 from charmhelpers.contrib.amulet.utils import (
@@ -602,3 +603,361 @@ class OpenStackAmuletUtils(AmuletUtils):
             self.log.debug('Ceph {} samples (OK): '
                            '{}'.format(sample_type, samples))
             return None
+
+# rabbitmq/amqp specific helpers:
+    def add_rmq_test_user(self, sentry_units,
+                          username="testuser1", password="changeme"):
+        """Add a test user via the first rmq juju unit, check connection as
+        the new user against all sentry units.
+
+        :param sentry_units: list of sentry unit pointers
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug('Adding rmq user ({})...'.format(username))
+
+        # Check that user does not already exist
+        cmd_user_list = 'rabbitmqctl list_users'
+        output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
+        if username in output:
+            self.log.warning('User ({}) already exists, returning '
+                             'gracefully.'.format(username))
+            return
+
+        perms = '".*" ".*" ".*"'
+        cmds = ['rabbitmqctl add_user {} {}'.format(username, password),
+                'rabbitmqctl set_permissions {} {}'.format(username, perms)]
+
+        # Add user via first unit
+        for cmd in cmds:
+            output, _ = self.run_cmd_unit(sentry_units[0], cmd)
+
+        # Check connection against the other sentry_units
+        self.log.debug('Checking user connect against units...')
+        for sentry_unit in sentry_units:
+            connection = self.connect_amqp_by_unit(sentry_unit, ssl=False,
+                                                   username=username,
+                                                   password=password)
+            connection.close()
+
+    def delete_rmq_test_user(self, sentry_units, username="testuser1"):
+        """Delete a rabbitmq user via the first rmq juju unit.
+
+        :param sentry_units: list of sentry unit pointers
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :returns: None if successful or no such user.
+        """
+        self.log.debug('Deleting rmq user ({})...'.format(username))
+
+        # Check that the user exists
+        cmd_user_list = 'rabbitmqctl list_users'
+        output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
+
+        if username not in output:
+            self.log.warning('User ({}) does not exist, returning '
+                             'gracefully.'.format(username))
+            return
+
+        # Delete the user
+        cmd_user_del = 'rabbitmqctl delete_user {}'.format(username)
+        output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_del)
+
+    def get_rmq_cluster_status(self, sentry_unit):
+        """Execute rabbitmq cluster status command on a unit and return
+        the full output.
+
+        :param unit: sentry unit
+        :returns: String containing console output of cluster status command
+        """
+        cmd = 'rabbitmqctl cluster_status'
+        output, _ = self.run_cmd_unit(sentry_unit, cmd)
+        self.log.debug('{} cluster_status:\n{}'.format(
+            sentry_unit.info['unit_name'], output))
+        return str(output)
+
+    def get_rmq_cluster_running_nodes(self, sentry_unit):
+        """Parse rabbitmqctl cluster_status output string, return list of
+        running rabbitmq cluster nodes.
+
+        :param unit: sentry unit
+        :returns: List containing node names of running nodes
+        """
+        # NOTE(beisner): rabbitmqctl cluster_status output is not
+        # json-parsable, do string chop foo, then json.loads that.
+        str_stat = self.get_rmq_cluster_status(sentry_unit)
+        if 'running_nodes' in str_stat:
+            pos_start = str_stat.find("{running_nodes,") + 15
+            pos_end = str_stat.find("]},", pos_start) + 1
+            str_run_nodes = str_stat[pos_start:pos_end].replace("'", '"')
+            run_nodes = json.loads(str_run_nodes)
+            return run_nodes
+        else:
+            return []
+
+    def validate_rmq_cluster_running_nodes(self, sentry_units):
+        """Check that all rmq unit hostnames are represented in the
+        cluster_status output of all units.
+
+        :param host_names: dict of juju unit names to host names
+        :param units: list of sentry unit pointers (all rmq units)
+        :returns: None if successful, otherwise return error message
+        """
+        host_names = self.get_unit_hostnames(sentry_units)
+        errors = []
+
+        # Query every unit for cluster_status running nodes
+        for query_unit in sentry_units:
+            query_unit_name = query_unit.info['unit_name']
+            running_nodes = self.get_rmq_cluster_running_nodes(query_unit)
+
+            # Confirm that every unit is represented in the queried unit's
+            # cluster_status running nodes output.
+            for validate_unit in sentry_units:
+                val_host_name = host_names[validate_unit.info['unit_name']]
+                val_node_name = 'rabbit@{}'.format(val_host_name)
+
+                if val_node_name not in running_nodes:
+                    errors.append('Cluster member check failed on {}: {} not '
+                                  'in {}\n'.format(query_unit_name,
+                                                   val_node_name,
+                                                   running_nodes))
+        if errors:
+            return ''.join(errors)
+
+    def rmq_ssl_is_enabled_on_unit(self, sentry_unit, port=None):
+        """Check a single juju rmq unit for ssl and port in the config file."""
+        host = sentry_unit.info['public-address']
+        unit_name = sentry_unit.info['unit_name']
+
+        conf_file = '/etc/rabbitmq/rabbitmq.config'
+        conf_contents = str(self.file_contents_safe(sentry_unit,
+                                                    conf_file, max_wait=16))
+        # Checks
+        conf_ssl = 'ssl' in conf_contents
+        conf_port = str(port) in conf_contents
+
+        # Port explicitly checked in config
+        if port and conf_port and conf_ssl:
+            self.log.debug('SSL is enabled  @{}:{} '
+                           '({})'.format(host, port, unit_name))
+            return True
+        elif port and not conf_port and conf_ssl:
+            self.log.debug('SSL is enabled @{} but not on port {} '
+                           '({})'.format(host, port, unit_name))
+            return False
+        # Port not checked (useful when checking that ssl is disabled)
+        elif not port and conf_ssl:
+            self.log.debug('SSL is enabled  @{}:{} '
+                           '({})'.format(host, port, unit_name))
+            return True
+        elif not conf_ssl:
+            self.log.debug('SSL not enabled @{}:{} '
+                           '({})'.format(host, port, unit_name))
+            return False
+        else:
+            msg = ('Unknown condition when checking SSL status @{}:{} '
+                   '({})'.format(host, port, unit_name))
+            amulet.raise_status(amulet.FAIL, msg)
+
+    def validate_rmq_ssl_enabled_units(self, sentry_units, port=None):
+        """Check that ssl is enabled on rmq juju sentry units.
+
+        :param sentry_units: list of all rmq sentry units
+        :param port: optional ssl port override to validate
+        :returns: None if successful, otherwise return error message
+        """
+        for sentry_unit in sentry_units:
+            if not self.rmq_ssl_is_enabled_on_unit(sentry_unit, port=port):
+                return ('Unexpected condition:  ssl is disabled on unit '
+                        '({})'.format(sentry_unit.info['unit_name']))
+        return None
+
+    def validate_rmq_ssl_disabled_units(self, sentry_units):
+        """Check that ssl is enabled on listed rmq juju sentry units.
+
+        :param sentry_units: list of all rmq sentry units
+        :returns: True if successful.  Raise on error.
+        """
+        for sentry_unit in sentry_units:
+            if self.rmq_ssl_is_enabled_on_unit(sentry_unit):
+                return ('Unexpected condition:  ssl is enabled on unit '
+                        '({})'.format(sentry_unit.info['unit_name']))
+        return None
+
+    def configure_rmq_ssl_on(self, sentry_units, deployment,
+                             port=None, max_wait=60):
+        """Turn ssl charm config option on, with optional non-default
+        ssl port specification.  Confirm that it is enabled on every
+        unit.
+
+        :param sentry_units: list of sentry units
+        :param deployment: amulet deployment object pointer
+        :param port: amqp port, use defaults if None
+        :param max_wait: maximum time to wait in seconds to confirm
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug('Setting ssl charm config option:  on')
+
+        # Enable RMQ SSL
+        config = {'ssl': 'on'}
+        if port:
+            config['ssl_port'] = port
+
+        deployment.configure('rabbitmq-server', config)
+
+        # Confirm
+        tries = 0
+        ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
+        while ret and tries < (max_wait / 4):
+            time.sleep(4)
+            self.log.debug('Attempt {}: {}'.format(tries, ret))
+            ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
+            tries += 1
+
+        if ret:
+            amulet.raise_status(amulet.FAIL, ret)
+
+    def configure_rmq_ssl_off(self, sentry_units, deployment, max_wait=60):
+        """Turn ssl charm config option off, confirm that it is disabled
+        on every unit.
+
+        :param sentry_units: list of sentry units
+        :param deployment: amulet deployment object pointer
+        :param max_wait: maximum time to wait in seconds to confirm
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug('Setting ssl charm config option:  off')
+
+        # Disable RMQ SSL
+        config = {'ssl': 'off'}
+        deployment.configure('rabbitmq-server', config)
+
+        # Confirm
+        tries = 0
+        ret = self.validate_rmq_ssl_disabled_units(sentry_units)
+        while ret and tries < (max_wait / 4):
+            time.sleep(4)
+            self.log.debug('Attempt {}: {}'.format(tries, ret))
+            ret = self.validate_rmq_ssl_disabled_units(sentry_units)
+            tries += 1
+
+        if ret:
+            amulet.raise_status(amulet.FAIL, ret)
+
+    def connect_amqp_by_unit(self, sentry_unit, ssl=False,
+                             port=None, fatal=True,
+                             username="testuser1", password="changeme"):
+        """Establish and return a pika amqp connection to the rabbitmq service
+        running on a rmq juju unit.
+
+        :param sentry_unit: sentry unit pointer
+        :param ssl: boolean, default to False
+        :param port: amqp port, use defaults if None
+        :param fatal: boolean, default to True (raises on connect error)
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :returns: pika amqp connection pointer or None if failed and non-fatal
+        """
+        host = sentry_unit.info['public-address']
+        unit_name = sentry_unit.info['unit_name']
+
+        # Default port logic if port is not specified
+        if ssl and not port:
+            port = 5671
+        elif not ssl and not port:
+            port = 5672
+
+        self.log.debug('Connecting to amqp on {}:{} ({}) as '
+                       '{}...'.format(host, port, unit_name, username))
+
+        try:
+            credentials = pika.PlainCredentials(username, password)
+            parameters = pika.ConnectionParameters(host=host, port=port,
+                                                   credentials=credentials,
+                                                   ssl=ssl,
+                                                   connection_attempts=3,
+                                                   retry_delay=5,
+                                                   socket_timeout=1)
+            connection = pika.BlockingConnection(parameters)
+            assert connection.server_properties['product'] == 'RabbitMQ'
+            self.log.debug('Connect OK')
+            return connection
+        except Exception as e:
+            msg = ('amqp connection failed to {}:{} as '
+                   '{} ({})'.format(host, port, username, str(e)))
+            if fatal:
+                amulet.raise_status(amulet.FAIL, msg)
+            else:
+                self.log.warn(msg)
+                return None
+
+    def publish_amqp_message_by_unit(self, sentry_unit, message,
+                                     queue="test", ssl=False,
+                                     username="testuser1",
+                                     password="changeme",
+                                     port=None):
+        """Publish an amqp message to a rmq juju unit.
+
+        :param sentry_unit: sentry unit pointer
+        :param message: amqp message string
+        :param queue: message queue, default to test
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :param ssl: boolean, default to False
+        :param port: amqp port, use defaults if None
+        :returns: None.  Raises exception if publish failed.
+        """
+        self.log.debug('Publishing message to {} queue:\n{}'.format(queue,
+                                                                    message))
+        connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
+                                               port=port,
+                                               username=username,
+                                               password=password)
+
+        # NOTE(beisner): extra debug here re: pika hang potential:
+        #   https://github.com/pika/pika/issues/297
+        #   https://groups.google.com/forum/#!topic/rabbitmq-users/Ja0iyfF0Szw
+        self.log.debug('Defining channel...')
+        channel = connection.channel()
+        self.log.debug('Declaring queue...')
+        channel.queue_declare(queue=queue, auto_delete=False, durable=True)
+        self.log.debug('Publishing message...')
+        channel.basic_publish(exchange='', routing_key=queue, body=message)
+        self.log.debug('Closing channel...')
+        channel.close()
+        self.log.debug('Closing connection...')
+        connection.close()
+
+    def get_amqp_message_by_unit(self, sentry_unit, queue="test",
+                                 username="testuser1",
+                                 password="changeme",
+                                 ssl=False, port=None):
+        """Get an amqp message from a rmq juju unit.
+
+        :param sentry_unit: sentry unit pointer
+        :param queue: message queue, default to test
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :param ssl: boolean, default to False
+        :param port: amqp port, use defaults if None
+        :returns: amqp message body as string.  Raise if get fails.
+        """
+        connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
+                                               port=port,
+                                               username=username,
+                                               password=password)
+        channel = connection.channel()
+        method_frame, _, body = channel.basic_get(queue)
+
+        if method_frame:
+            self.log.debug('Retreived message from {} queue:\n{}'.format(queue,
+                                                                         body))
+            channel.basic_ack(method_frame.delivery_tag)
+            channel.close()
+            connection.close()
+            return body
+        else:
+            msg = 'No message retrieved.'
+            amulet.raise_status(amulet.FAIL, msg)

tests/tests.yaml

@@ -0,0 +1,20 @@
+bootstrap: true
+reset: true
+virtualenv: true
+makefile:
+    - lint
+    - test
+sources:
+    - ppa:juju/stable
+packages:
+    - amulet
+    - distro-info-data
+    - python-cinderclient
+    - python-distro-info
+    - python-glanceclient
+    - python-heatclient
+    - python-keystoneclient
+    - python-neutronclient
+    - python-novaclient
+    - python-pika
+    - python-swiftclient

unit_tests/test_actions_git_reinstall.py

@@ -1,5 +1,6 @@
 from mock import patch, MagicMock
 
+patch('charmhelpers.core.hookenv.status_set').start()
 with patch('charmhelpers.core.hookenv.config') as config:
     config.return_value = 'neutron'
     import neutron_api_utils as utils  # noqa

unit_tests/test_actions_openstack_upgrade.py

@@ -0,0 +1,57 @@
+from mock import patch
+import os
+
+os.environ['JUJU_UNIT_NAME'] = 'neutron-api'
+
+with patch('charmhelpers.core.hookenv.config') as config:
+    config.return_value = 'ovs'
+    with patch('neutron_api_utils.register_configs') as register_configs:
+        import openstack_upgrade
+
+from test_utils import (
+    CharmTestCase
+)
+
+TO_PATCH = [
+    'do_openstack_upgrade',
+    'config_changed',
+]
+
+
+class TestNeutronAPIUpgradeActions(CharmTestCase):
+
+    def setUp(self):
+        super(TestNeutronAPIUpgradeActions, self).setUp(openstack_upgrade,
+                                                        TO_PATCH)
+
+    @patch('charmhelpers.contrib.openstack.utils.juju_log')
+    @patch('charmhelpers.contrib.openstack.utils.config')
+    @patch('charmhelpers.contrib.openstack.utils.action_set')
+    @patch('charmhelpers.contrib.openstack.utils.git_install_requested')
+    @patch('charmhelpers.contrib.openstack.utils.openstack_upgrade_available')
+    def test_openstack_upgrade_true(self, upgrade_avail, git_requested,
+                                    action_set, config, log):
+        git_requested.return_value = False
+        upgrade_avail.return_value = True
+        config.return_value = True
+
+        openstack_upgrade.openstack_upgrade()
+
+        self.assertTrue(self.do_openstack_upgrade.called)
+        self.assertTrue(self.config_changed.called)
+
+    @patch('charmhelpers.contrib.openstack.utils.juju_log')
+    @patch('charmhelpers.contrib.openstack.utils.config')
+    @patch('charmhelpers.contrib.openstack.utils.action_set')
+    @patch('charmhelpers.contrib.openstack.utils.git_install_requested')
+    @patch('charmhelpers.contrib.openstack.utils.openstack_upgrade_available')
+    def test_openstack_upgrade_false(self, upgrade_avail, git_requested,
+                                     action_set, config, log):
+        git_requested.return_value = False
+        upgrade_avail.return_value = True
+        config.return_value = False
+
+        openstack_upgrade.openstack_upgrade()
+
+        self.assertFalse(self.do_openstack_upgrade.called)
+        self.assertFalse(self.config_changed.called)

unit_tests/test_neutron_api_context.py

@@ -277,6 +277,9 @@ class NeutronCCContextTest(CharmTestCase):
         self.test_config.set('vsd-organization', 'foo')
         self.test_config.set('vsd-base-uri', '/nuage/api/v1_0')
         self.test_config.set('vsd-netpart-name', 'foo-enterprise')
+        self.test_config.set('plumgrid-username', 'plumgrid')
+        self.test_config.set('plumgrid-password', 'plumgrid')
+        self.test_config.set('plumgrid-virtual-ip', '192.168.100.250')
 
     def tearDown(self):
         super(NeutronCCContextTest, self).tearDown()
@@ -459,6 +462,53 @@ class NeutronCCContextTest(CharmTestCase):
             self.assertEquals(napi_ctxt[key], expect[key])
 
 
+class EtcdContextTest(CharmTestCase):
+
+    def setUp(self):
+        super(EtcdContextTest, self).setUp(context, TO_PATCH)
+        self.relation_get.side_effect = self.test_relation.get
+        self.config.side_effect = self.test_config.get
+        self.test_config.set('neutron-plugin', 'Calico')
+
+    def tearDown(self):
+        super(EtcdContextTest, self).tearDown()
+
+    def test_etcd_no_related_units(self):
+        self.related_units.return_value = []
+        ctxt = context.EtcdContext()()
+        expect = {'cluster': ''}
+
+        self.assertEquals(expect, ctxt)
+
+    def test_some_related_units(self):
+        self.related_units.return_value = ['unit1']
+        self.relation_ids.return_value = ['rid2', 'rid3']
+        result = (
+            'testname=http://172.18.18.18:8888,'
+            'testname=http://172.18.18.18:8888'
+        )
+        self.test_relation.set({'cluster': result})
+
+        ctxt = context.EtcdContext()()
+        expect = {'cluster': result}
+
+        self.assertEquals(expect, ctxt)
+
+    def test_early_exit(self):
+        self.test_config.set('neutron-plugin', 'notCalico')
+
+        self.related_units.return_value = ['unit1']
+        self.relation_ids.return_value = ['rid2', 'rid3']
+        self.test_relation.set({'ip': '172.18.18.18',
+                                'port': 8888,
+                                'name': 'testname'})
+
+        ctxt = context.EtcdContext()()
+        expect = {'cluster': ''}
+
+        self.assertEquals(expect, ctxt)
+
+
 class NeutronApiSDNContextTest(CharmTestCase):
 
     def setUp(self):
@@ -503,6 +553,7 @@ class NeutronApiSDNContextTest(CharmTestCase):
                                           'ml2_conf.ini'),
                 'service_plugins': 'router,firewall,lbaas,vpnaas,metering',
                 'restart_trigger': '',
+                'quota_driver': '',
                 'neutron_plugin': 'ovs',
                 'sections': {},
             }
@@ -516,12 +567,14 @@ class NeutronApiSDNContextTest(CharmTestCase):
                 'neutron-plugin-config': '/etc/neutron/plugins/fl/flump.ini',
                 'service-plugins': 'router,unicorn,rainbows',
                 'restart-trigger': 'restartnow',
+                'quota-driver': 'quotadriver',
             },
             {
                 'core_plugin': 'neutron.plugins.ml2.plugin.MidoPlumODL',
                 'neutron_plugin_config': '/etc/neutron/plugins/fl/flump.ini',
                 'service_plugins': 'router,unicorn,rainbows',
                 'restart_trigger': 'restartnow',
+                'quota_driver': 'quotadriver',
                 'neutron_plugin': 'ovs',
                 'sections': {},
             }
@@ -550,6 +603,7 @@ class NeutronApiSDNContextTest(CharmTestCase):
                                           'ml2_conf.ini'),
                 'service_plugins': 'router,firewall,lbaas,vpnaas,metering',
                 'restart_trigger': '',
+                'quota_driver': '',
                 'neutron_plugin': 'ovs',
                 'sections': {u'DEFAULT': [[u'neutronboost', True]]},
             }

unit_tests/test_neutron_api_hooks.py

@@ -64,6 +64,7 @@ TO_PATCH = [
     'service_reload',
     'neutron_plugin_attribute',
     'IdentityServiceContext',
+    'force_etcd_restart',
 ]
 NEUTRON_CONF_DIR = "/etc/neutron"
 
@@ -285,6 +286,16 @@ class NeutronAPIHooksTests(CharmTestCase):
         self.assertTrue(_zmq_joined.called)
         self.assertTrue(_id_cluster_joined.called)
 
+    @patch.object(hooks, 'git_install_requested')
+    def test_config_changed_with_openstack_upgrade_action(self, git_requested):
+        git_requested.return_value = False
+        self.openstack_upgrade_available.return_value = True
+        self.test_config.set('action-managed-upgrade', True)
+
+        self._call_hook('config-changed')
+
+        self.assertFalse(self.do_openstack_upgrade.called)
+
     def test_amqp_joined(self):
         self._call_hook('amqp-relation-joined')
         self.relation_set.assert_called_with(
@@ -492,12 +503,14 @@ class NeutronAPIHooksTests(CharmTestCase):
         self.assertTrue(self.CONFIGS.write.called_with(NEUTRON_CONF))
 
     def test_neutron_plugin_api_relation_joined_nol2(self):
+        self.unit_get.return_value = '172.18.18.18'
         self.IdentityServiceContext.return_value = \
             DummyContext(return_value={})
         _relation_data = {
             'neutron-security-groups': False,
             'enable-dvr': False,
             'enable-l3ha': False,
+            'addr': '172.18.18.18',
             'l2-population': False,
             'overlay-network-type': 'vxlan',
             'service_protocol': None,
@@ -522,12 +535,14 @@ class NeutronAPIHooksTests(CharmTestCase):
         )
 
     def test_neutron_plugin_api_relation_joined_dvr(self):
+        self.unit_get.return_value = '172.18.18.18'
         self.IdentityServiceContext.return_value = \
             DummyContext(return_value={})
         _relation_data = {
             'neutron-security-groups': False,
             'enable-dvr': True,
             'enable-l3ha': False,
+            'addr': '172.18.18.18',
             'l2-population': True,
             'overlay-network-type': 'vxlan',
             'service_protocol': None,
@@ -552,12 +567,14 @@ class NeutronAPIHooksTests(CharmTestCase):
         )
 
     def test_neutron_plugin_api_relation_joined_l3ha(self):
+        self.unit_get.return_value = '172.18.18.18'
         self.IdentityServiceContext.return_value = \
             DummyContext(return_value={})
         _relation_data = {
             'neutron-security-groups': False,
             'enable-dvr': False,
             'enable-l3ha': True,
+            'addr': '172.18.18.18',
             'l2-population': False,
             'overlay-network-type': 'vxlan',
             'service_protocol': None,
@@ -582,11 +599,13 @@ class NeutronAPIHooksTests(CharmTestCase):
         )
 
     def test_neutron_plugin_api_relation_joined_w_mtu(self):
+        self.unit_get.return_value = '172.18.18.18'
         self.IdentityServiceContext.return_value = \
             DummyContext(return_value={})
         self.test_config.set('network-device-mtu', 1500)
         _relation_data = {
             'neutron-security-groups': False,
+            'addr': '172.18.18.18',
             'l2-population': False,
             'overlay-network-type': 'vxlan',
             'network-device-mtu': 1500,
@@ -745,9 +764,8 @@ class NeutronAPIHooksTests(CharmTestCase):
         self.relation_ids.side_effect = self._fake_relids
         _id_rel_joined = self.patch('identity_joined')
         hooks.configure_https()
-        calls = [call('a2dissite', 'openstack_https_frontend'),
-                 call('service', 'apache2', 'reload')]
-        self.check_call.assert_called_has_calls(calls)
+        self.check_call.assert_called_with(['a2ensite',
+                                            'openstack_https_frontend'])
         self.assertTrue(_id_rel_joined.called)
 
     def test_configure_https_nohttps(self):
@@ -755,9 +773,8 @@ class NeutronAPIHooksTests(CharmTestCase):
         self.relation_ids.side_effect = self._fake_relids
         _id_rel_joined = self.patch('identity_joined')
         hooks.configure_https()
-        calls = [call('a2dissite', 'openstack_https_frontend'),
-                 call('service', 'apache2', 'reload')]
-        self.check_call.assert_called_has_calls(calls)
+        self.check_call.assert_called_with(['a2dissite',
+                                            'openstack_https_frontend'])
         self.assertTrue(_id_rel_joined.called)
 
     def test_conditional_neutron_migration_icehouse(self):
@@ -802,3 +819,8 @@ class NeutronAPIHooksTests(CharmTestCase):
             'Not running neutron database migration as migrations are handled '
             'by the neutron-server process or nova-cloud-controller charm.'
         )
+
+    def test_etcd_peer_joined(self):
+        self._call_hook('etcd-proxy-relation-joined')
+        self.assertTrue(self.CONFIGS.register.called)
+        self.CONFIGS.write.assert_called_with('/etc/init/etcd.conf')

unit_tests/test_neutron_api_utils.py

@@ -24,6 +24,7 @@ TO_PATCH = [
     'apt_install',
     'apt_update',
     'apt_upgrade',
+    'add_source',
     'b64encode',
     'config',
     'configure_installation_source',
@@ -34,6 +35,9 @@ TO_PATCH = [
     'pip_install',
     'subprocess',
     'is_elected_leader',
+    'service_stop',
+    'service_start',
+    'glob',
 ]
 
 openstack_origin_git = \
@@ -149,6 +153,19 @@ class TestNeutronAPIUtils(CharmTestCase):
         [self.assertIn(q_conf, _map.keys()) for q_conf in confs]
         self.assertTrue(nutils.APACHE_24_CONF not in _map.keys())
 
+    @patch.object(nutils, 'manage_plugin')
+    @patch('os.path.exists')
+    def test_resource_map_liberty(self, _path_exists, _manage_plugin):
+        _path_exists.return_value = False
+        _manage_plugin.return_value = True
+        self.os_release.return_value = 'liberty'
+        _map = nutils.resource_map()
+        confs = [nutils.NEUTRON_CONF, nutils.NEUTRON_DEFAULT,
+                 nutils.APACHE_CONF, nutils.NEUTRON_LBAAS_CONF,
+                 nutils.NEUTRON_VPNAAS_CONF]
+        [self.assertIn(q_conf, _map.keys()) for q_conf in confs]
+        self.assertTrue(nutils.APACHE_24_CONF not in _map.keys())
+
     @patch.object(nutils, 'manage_plugin')
     @patch('os.path.exists')
     def test_resource_map_apache24(self, _path_exists, _manage_plugin):
@@ -248,8 +265,8 @@ class TestNeutronAPIUtils(CharmTestCase):
         self.get_os_codename_install_source.return_value = 'juno'
         configs = MagicMock()
         nutils.do_openstack_upgrade(configs)
-        self.os_release.assert_called_with('neutron-server')
-        self.log.assert_called()
+        self.os_release.assert_called_with('neutron-common')
+        self.assertTrue(self.log.called)
         self.configure_installation_source.assert_called_with(
             'cloud:trusty-juno'
         )
@@ -287,8 +304,8 @@ class TestNeutronAPIUtils(CharmTestCase):
         self.get_os_codename_install_source.return_value = 'kilo'
         configs = MagicMock()
         nutils.do_openstack_upgrade(configs)
-        self.os_release.assert_called_with('neutron-server')
-        self.log.assert_called()
+        self.os_release.assert_called_with('neutron-common')
+        self.assertTrue(self.log.called)
         self.configure_installation_source.assert_called_with(
             'cloud:trusty-kilo'
         )
@@ -327,8 +344,8 @@ class TestNeutronAPIUtils(CharmTestCase):
         self.get_os_codename_install_source.return_value = 'kilo'
         configs = MagicMock()
         nutils.do_openstack_upgrade(configs)
-        self.os_release.assert_called_with('neutron-server')
-        self.log.assert_called()
+        self.os_release.assert_called_with('neutron-common')
+        self.assertTrue(self.log.called)
         self.configure_installation_source.assert_called_with(
             'cloud:trusty-kilo'
         )
@@ -580,3 +597,30 @@ class TestNeutronAPIUtils(CharmTestCase):
         self.test_config.set('manage-neutron-plugin-legacy-mode', False)
         manage = nutils.manage_plugin()
         self.assertFalse(manage)
+
+    def test_additional_install_locations_calico(self):
+        self.get_os_codename_install_source.return_value = 'icehouse'
+        nutils.additional_install_locations('Calico', '')
+        self.add_source.assert_called_with('ppa:project-calico/icehouse')
+
+    def test_unusual_calico_install_location(self):
+        self.test_config.set('calico-origin', 'ppa:testppa/project-calico')
+        nutils.additional_install_locations('Calico', '')
+        self.add_source.assert_called_with('ppa:testppa/project-calico')
+
+    def test_follows_openstack_origin(self):
+        self.get_os_codename_install_source.return_value = 'juno'
+        nutils.additional_install_locations('Calico', 'cloud:trusty-juno')
+        self.add_source.assert_called_with('ppa:project-calico/juno')
+
+    @patch('shutil.rmtree')
+    def test_force_etcd_restart(self, rmtree):
+        self.glob.glob.return_value = [
+            '/var/lib/etcd/one', '/var/lib/etcd/two'
+        ]
+        nutils.force_etcd_restart()
+        self.service_stop.assert_called_once_with('etcd')
+        self.glob.glob.assert_called_once_with('/var/lib/etcd/*')
+        rmtree.assert_any_call('/var/lib/etcd/one')
+        rmtree.assert_any_call('/var/lib/etcd/two')
+        self.service_start.assert_called_once_with('etcd')

unit_tests/test_utils.py

@@ -6,6 +6,9 @@ import yaml
 from contextlib import contextmanager
 from mock import patch, MagicMock
 
+patch('charmhelpers.contrib.openstack.utils.set_os_workload_status').start()
+patch('charmhelpers.core.hookenv.status_set').start()
+
 
 def load_config():
     '''