Merge lp:~tealeg/charms/trusty/ceilometer/pause-and-resume [a=tealeg] [r=tribaal, ack]

Add pause and resume actions for the ceilometer services.

actions.yaml

@@ -0,0 +1,4 @@
+pause:
+  description: Pause the Ceilometer unit.  This action will stop Ceilometer services.
+resume:
+  descrpition: Resume the Ceilometer unit.  This action will start Ceilometer services.

actions/actions.py

@@ -0,0 +1,52 @@
+#!/usr/bin/python
+
+import os
+import sys
+
+from charmhelpers.core.host import service_pause, service_resume
+from charmhelpers.core.hookenv import action_fail, status_set
+from ceilometer_utils import CEILOMETER_SERVICES
+
+
+def pause(args):
+    """Pause the Ceilometer services.
+
+    @raises Exception should the service fail to stop.
+    """
+    for service in CEILOMETER_SERVICES:
+        if not service_pause(service):
+            raise Exception("Failed to %s." % service)
+    status_set(
+        "maintenance", "Paused. Use 'resume' action to resume normal service.")
+
+def resume(args):
+    """Resume the Ceilometer services.
+
+    @raises Exception should the service fail to start."""
+    for service in CEILOMETER_SERVICES:
+        if not service_resume(service):
+            raise Exception("Failed to resume %s." % service)
+    status_set("active", "")
+
+
+# A dictionary of all the defined actions to callables (which take
+# parsed arguments).
+ACTIONS = {"pause": pause, "resume": resume}
+
+
+def main(args):
+    action_name = os.path.basename(args[0])
+    try:
+        action = ACTIONS[action_name]
+    except KeyError:
+        return "Action %s undefined" % action_name
+    else:
+        try:
+            action(args)
+        except Exception as e:
+            action_fail(str(e))
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))
+

actions/ceilometer_contexts.py

@@ -0,0 +1 @@
+../ceilometer_contexts.py

actions/ceilometer_utils.py

@@ -0,0 +1 @@
+../ceilometer_utils.py

actions/charmhelpers

@@ -0,0 +1 @@
+../charmhelpers

actions/pause

@@ -0,0 +1 @@
+actions.py

actions/resume

@@ -0,0 +1 @@
+actions.py

ceilometer_contexts.py

@@ -0,0 +1,121 @@
+from charmhelpers.core.hookenv import (
+    relation_ids,
+    relation_get,
+    related_units,
+    config
+)
+
+from charmhelpers.contrib.openstack.utils import os_release
+
+from charmhelpers.contrib.openstack.context import (
+    OSContextGenerator,
+    context_complete,
+    ApacheSSLContext as SSLContext,
+)
+
+from charmhelpers.contrib.hahelpers.cluster import (
+    determine_apache_port,
+    determine_api_port
+)
+
+CEILOMETER_DB = 'ceilometer'
+
+
+class LoggingConfigContext(OSContextGenerator):
+    def __call__(self):
+        return {'debug': config('debug'), 'verbose': config('verbose')}
+
+
+class MongoDBContext(OSContextGenerator):
+    interfaces = ['mongodb']
+
+    def __call__(self):
+        mongo_servers = []
+        replset = None
+        use_replset = os_release('ceilometer-api') >= 'icehouse'
+
+        for relid in relation_ids('shared-db'):
+            rel_units = related_units(relid)
+            use_replset = use_replset and (len(rel_units) > 1)
+
+            for unit in rel_units:
+                host = relation_get('hostname', unit, relid)
+                port = relation_get('port', unit, relid)
+
+                conf = {
+                    "db_host": host,
+                    "db_port": port,
+                    "db_name": CEILOMETER_DB
+                }
+
+                if not context_complete(conf):
+                    continue
+
+                if not use_replset:
+                    return conf
+
+                if replset is None:
+                    replset = relation_get('replset', unit, relid)
+
+                mongo_servers.append('{}:{}'.format(host, port))
+
+        if mongo_servers:
+            return {
+                'db_mongo_servers': ','.join(mongo_servers),
+                'db_name': CEILOMETER_DB,
+                'db_replset': replset
+            }
+
+        return {}
+
+
+CEILOMETER_PORT = 8777
+
+
+class CeilometerContext(OSContextGenerator):
+    def __call__(self):
+        # Lazy-import to avoid a circular dependency in the imports
+        from ceilometer_utils import get_shared_secret
+
+        ctxt = {
+            'port': CEILOMETER_PORT,
+            'metering_secret': get_shared_secret()
+        }
+        return ctxt
+
+
+class CeilometerServiceContext(OSContextGenerator):
+    interfaces = ['ceilometer-service']
+
+    def __call__(self):
+        for relid in relation_ids('ceilometer-service'):
+            for unit in related_units(relid):
+                conf = relation_get(unit=unit, rid=relid)
+                if context_complete(conf):
+                    return conf
+        return {}
+
+
+class HAProxyContext(OSContextGenerator):
+    interfaces = ['ceilometer-haproxy']
+
+    def __call__(self):
+        '''Extends the main charmhelpers HAProxyContext with a port mapping
+        specific to this charm.
+        '''
+        haproxy_port = CEILOMETER_PORT
+        api_port = determine_api_port(CEILOMETER_PORT, singlenode_mode=True)
+        apache_port = determine_apache_port(CEILOMETER_PORT,
+                                            singlenode_mode=True)
+
+        ctxt = {
+            'service_ports': {'ceilometer_api': [haproxy_port, apache_port]},
+            'port': api_port
+        }
+        return ctxt
+
+
+class ApacheSSLContext(SSLContext):
+
+    external_ports = [CEILOMETER_PORT]
+    service_namespace = "ceilometer"

ceilometer_utils.py

@@ -0,0 +1,225 @@
+import os
+import uuid
+
+from collections import OrderedDict
+
+from charmhelpers.contrib.openstack import (
+    templating,
+    context,
+)
+from ceilometer_contexts import (
+    ApacheSSLContext,
+    LoggingConfigContext,
+    MongoDBContext,
+    CeilometerContext,
+    HAProxyContext
+)
+from charmhelpers.contrib.openstack.utils import (
+    get_os_codename_package,
+    get_os_codename_install_source,
+    configure_installation_source
+)
+from charmhelpers.core.hookenv import config, log
+from charmhelpers.fetch import apt_update, apt_install, apt_upgrade
+from copy import deepcopy
+
+HAPROXY_CONF = '/etc/haproxy/haproxy.cfg'
+CEILOMETER_CONF_DIR = "/etc/ceilometer"
+CEILOMETER_CONF = "%s/ceilometer.conf" % CEILOMETER_CONF_DIR
+HTTPS_APACHE_CONF = "/etc/apache2/sites-available/openstack_https_frontend"
+HTTPS_APACHE_24_CONF = "/etc/apache2/sites-available/" \
+    "openstack_https_frontend.conf"
+CLUSTER_RES = 'grp_ceilometer_vips'
+
+CEILOMETER_SERVICES = [
+    'ceilometer-agent-central',
+    'ceilometer-collector',
+    'ceilometer-api',
+    'ceilometer-alarm-evaluator',
+    'ceilometer-alarm-notifier',
+    'ceilometer-agent-notification',
+]
+
+CEILOMETER_DB = "ceilometer"
+CEILOMETER_SERVICE = "ceilometer"
+
+CEILOMETER_PACKAGES = [
+    'haproxy',
+    'apache2',
+    'ceilometer-agent-central',
+    'ceilometer-collector',
+    'ceilometer-api',
+    'python-pymongo',
+]
+
+ICEHOUSE_PACKAGES = [
+    'ceilometer-alarm-notifier',
+    'ceilometer-alarm-evaluator',
+    'ceilometer-agent-notification'
+]
+
+ICEHOUSE_SERVICES = [
+    'ceilometer-alarm-notifier',
+    'ceilometer-alarm-evaluator',
+    'ceilometer-agent-notification'
+]
+
+CEILOMETER_ROLE = "ResellerAdmin"
+SVC = 'ceilometer'
+
+CONFIG_FILES = OrderedDict([
+    (CEILOMETER_CONF, {
+        'hook_contexts': [context.IdentityServiceContext(service=SVC,
+                                                         service_user=SVC),
+                          context.AMQPContext(ssl_dir=CEILOMETER_CONF_DIR),
+                          LoggingConfigContext(),
+                          MongoDBContext(),
+                          CeilometerContext(),
+                          context.SyslogContext(),
+                          HAProxyContext()],
+        'services': CEILOMETER_SERVICES
+    }),
+    (HAPROXY_CONF, {
+        'hook_contexts': [context.HAProxyContext(singlenode_mode=True),
+                          HAProxyContext()],
+        'services': ['haproxy'],
+    }),
+    (HTTPS_APACHE_CONF, {
+        'hook_contexts': [ApacheSSLContext()],
+        'services': ['apache2'],
+    }),
+    (HTTPS_APACHE_24_CONF, {
+        'hook_contexts': [ApacheSSLContext()],
+        'services': ['apache2'],
+    })
+])
+
+TEMPLATES = 'templates'
+
+SHARED_SECRET = "/etc/ceilometer/secret.txt"
+
+
+def register_configs():
+    """
+    Register config files with their respective contexts.
+    Regstration of some configs may not be required depending on
+    existing of certain relations.
+    """
+    # if called without anything installed (eg during install hook)
+    # just default to earliest supported release. configs dont get touched
+    # till post-install, anyway.
+    release = get_os_codename_package('ceilometer-common', fatal=False) \
+        or 'grizzly'
+    configs = templating.OSConfigRenderer(templates_dir=TEMPLATES,
+                                          openstack_release=release)
+
+    if (get_os_codename_install_source(
+            config('openstack-origin')) >= 'icehouse'):
+        CONFIG_FILES[CEILOMETER_CONF]['services'] = \
+            CONFIG_FILES[CEILOMETER_CONF]['services'] + ICEHOUSE_SERVICES
+
+    for conf in CONFIG_FILES:
+        configs.register(conf, CONFIG_FILES[conf]['hook_contexts'])
+
+    if os.path.exists('/etc/apache2/conf-available'):
+        configs.register(HTTPS_APACHE_24_CONF,
+                         CONFIG_FILES[HTTPS_APACHE_24_CONF]['hook_contexts'])
+    else:
+        configs.register(HTTPS_APACHE_CONF,
+                         CONFIG_FILES[HTTPS_APACHE_CONF]['hook_contexts'])
+    return configs
+
+
+def restart_map():
+    '''
+    Determine the correct resource map to be passed to
+    charmhelpers.core.restart_on_change() based on the services configured.
+
+    :returns: dict: A dictionary mapping config file to lists of services
+                    that should be restarted when file changes.
+    '''
+    _map = {}
+    for f, ctxt in CONFIG_FILES.iteritems():
+        svcs = []
+        for svc in ctxt['services']:
+            svcs.append(svc)
+        if svcs:
+            _map[f] = svcs
+    return _map
+
+
+def services():
+    ''' Returns a list of services associate with this charm '''
+    _services = []
+    for v in restart_map().values():
+        _services = _services + v
+    return list(set(_services))
+
+
+def get_ceilometer_context():
+    ''' Retrieve a map of all current relation data for agent configuration '''
+    ctxt = {}
+    for hcontext in CONFIG_FILES[CEILOMETER_CONF]['hook_contexts']:
+        ctxt.update(hcontext())
+    return ctxt
+
+
+def do_openstack_upgrade(configs):
+    """
+    Perform an upgrade.  Takes care of upgrading packages, rewriting
+    configs, database migrations and potentially any other post-upgrade
+    actions.
+
+    :param configs: The charms main OSConfigRenderer object.
+    """
+    new_src = config('openstack-origin')
+    new_os_rel = get_os_codename_install_source(new_src)
+
+    log('Performing OpenStack upgrade to %s.' % (new_os_rel))
+
+    configure_installation_source(new_src)
+    dpkg_opts = [
+        '--option', 'Dpkg::Options::=--force-confnew',
+        '--option', 'Dpkg::Options::=--force-confdef',
+    ]
+    apt_update(fatal=True)
+    apt_upgrade(options=dpkg_opts, fatal=True, dist=True)
+    apt_install(packages=get_packages(),
+                options=dpkg_opts,
+                fatal=True)
+
+    # set CONFIGS to load templates from new release
+    configs.set_release(openstack_release=new_os_rel)
+
+
+def get_packages():
+    packages = deepcopy(CEILOMETER_PACKAGES)
+    if (get_os_codename_install_source(
+            config('openstack-origin')) >= 'icehouse'):
+        packages = packages + ICEHOUSE_PACKAGES
+    return packages
+
+
+def get_shared_secret():
+    """
+    Returns the current shared secret for the ceilometer node. If the shared
+    secret does not exist, this method will generate one.
+    """
+    secret = None
+    if not os.path.exists(SHARED_SECRET):
+        secret = str(uuid.uuid4())
+        set_shared_secret(secret)
+    else:
+        with open(SHARED_SECRET, 'r') as secret_file:
+            secret = secret_file.read().strip()
+    return secret
+
+
+def set_shared_secret(secret):
+    """
+    Sets the shared secret which is used to sign ceilometer messages.
+
+    :param secret: the secret to set
+    """
+    with open(SHARED_SECRET, 'w') as secret_file:
+        secret_file.write(secret)

charmhelpers/contrib/openstack/amulet/deployment.py

@@ -44,8 +44,15 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            Determine if the local branch being tested is derived from its
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
+
+        # Charms outside the lp:~openstack-charmers namespace
         base_charms = ['mysql', 'mongodb', 'nrpe']
 
+        # Force these charms to current series even when using an older series.
+        # ie. Use trusty/nrpe even when series is precise, as the P charm
+        # does not possess the necessary external master config and hooks.
+        force_series_current = ['nrpe']
+
         if self.series in ['precise', 'trusty']:
             base_series = self.series
         else:
@@ -53,11 +60,17 @@ class OpenStackAmuletDeployment(AmuletDeployment):
 
         if self.stable:
             for svc in other_services:
+                if svc['name'] in force_series_current:
+                    base_series = self.current_next
+
                 temp = 'lp:charms/{}/{}'
                 svc['location'] = temp.format(base_series,
                                               svc['name'])
         else:
             for svc in other_services:
+                if svc['name'] in force_series_current:
+                    base_series = self.current_next
+
                 if svc['name'] in base_charms:
                     temp = 'lp:charms/{}/{}'
                     svc['location'] = temp.format(base_series,
@@ -77,21 +90,23 @@ class OpenStackAmuletDeployment(AmuletDeployment):
 
         services = other_services
         services.append(this_service)
+
+        # Charms which should use the source config option
         use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
                       'ceph-osd', 'ceph-radosgw']
-        # Most OpenStack subordinate charms do not expose an origin option
-        # as that is controlled by the principle.
-        ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
+
+        # Charms which can not use openstack-origin, ie. many subordinates
+        no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
 
         if self.openstack:
             for svc in services:
-                if svc['name'] not in use_source + ignore:
+                if svc['name'] not in use_source + no_origin:
                     config = {'openstack-origin': self.openstack}
                     self.d.configure(svc['name'], config)
 
         if self.source:
             for svc in services:
-                if svc['name'] in use_source and svc['name'] not in ignore:
+                if svc['name'] in use_source and svc['name'] not in no_origin:
                     config = {'source': self.source}
                     self.d.configure(svc['name'], config)
 

charmhelpers/contrib/openstack/amulet/utils.py

@@ -27,6 +27,7 @@ import glanceclient.v1.client as glance_client
 import heatclient.v1.client as heat_client
 import keystoneclient.v2_0 as keystone_client
 import novaclient.v1_1.client as nova_client
+import pika
 import swiftclient
 
 from charmhelpers.contrib.amulet.utils import (
@@ -602,3 +603,361 @@ class OpenStackAmuletUtils(AmuletUtils):
             self.log.debug('Ceph {} samples (OK): '
                            '{}'.format(sample_type, samples))
             return None
+
+# rabbitmq/amqp specific helpers:
+    def add_rmq_test_user(self, sentry_units,
+                          username="testuser1", password="changeme"):
+        """Add a test user via the first rmq juju unit, check connection as
+        the new user against all sentry units.
+
+        :param sentry_units: list of sentry unit pointers
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug('Adding rmq user ({})...'.format(username))
+
+        # Check that user does not already exist
+        cmd_user_list = 'rabbitmqctl list_users'
+        output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
+        if username in output:
+            self.log.warning('User ({}) already exists, returning '
+                             'gracefully.'.format(username))
+            return
+
+        perms = '".*" ".*" ".*"'
+        cmds = ['rabbitmqctl add_user {} {}'.format(username, password),
+                'rabbitmqctl set_permissions {} {}'.format(username, perms)]
+
+        # Add user via first unit
+        for cmd in cmds:
+            output, _ = self.run_cmd_unit(sentry_units[0], cmd)
+
+        # Check connection against the other sentry_units
+        self.log.debug('Checking user connect against units...')
+        for sentry_unit in sentry_units:
+            connection = self.connect_amqp_by_unit(sentry_unit, ssl=False,
+                                                   username=username,
+                                                   password=password)
+            connection.close()
+
+    def delete_rmq_test_user(self, sentry_units, username="testuser1"):
+        """Delete a rabbitmq user via the first rmq juju unit.
+
+        :param sentry_units: list of sentry unit pointers
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :returns: None if successful or no such user.
+        """
+        self.log.debug('Deleting rmq user ({})...'.format(username))
+
+        # Check that the user exists
+        cmd_user_list = 'rabbitmqctl list_users'
+        output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
+
+        if username not in output:
+            self.log.warning('User ({}) does not exist, returning '
+                             'gracefully.'.format(username))
+            return
+
+        # Delete the user
+        cmd_user_del = 'rabbitmqctl delete_user {}'.format(username)
+        output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_del)
+
+    def get_rmq_cluster_status(self, sentry_unit):
+        """Execute rabbitmq cluster status command on a unit and return
+        the full output.
+
+        :param unit: sentry unit
+        :returns: String containing console output of cluster status command
+        """
+        cmd = 'rabbitmqctl cluster_status'
+        output, _ = self.run_cmd_unit(sentry_unit, cmd)
+        self.log.debug('{} cluster_status:\n{}'.format(
+            sentry_unit.info['unit_name'], output))
+        return str(output)
+
+    def get_rmq_cluster_running_nodes(self, sentry_unit):
+        """Parse rabbitmqctl cluster_status output string, return list of
+        running rabbitmq cluster nodes.
+
+        :param unit: sentry unit
+        :returns: List containing node names of running nodes
+        """
+        # NOTE(beisner): rabbitmqctl cluster_status output is not
+        # json-parsable, do string chop foo, then json.loads that.
+        str_stat = self.get_rmq_cluster_status(sentry_unit)
+        if 'running_nodes' in str_stat:
+            pos_start = str_stat.find("{running_nodes,") + 15
+            pos_end = str_stat.find("]},", pos_start) + 1
+            str_run_nodes = str_stat[pos_start:pos_end].replace("'", '"')
+            run_nodes = json.loads(str_run_nodes)
+            return run_nodes
+        else:
+            return []
+
+    def validate_rmq_cluster_running_nodes(self, sentry_units):
+        """Check that all rmq unit hostnames are represented in the
+        cluster_status output of all units.
+
+        :param host_names: dict of juju unit names to host names
+        :param units: list of sentry unit pointers (all rmq units)
+        :returns: None if successful, otherwise return error message
+        """
+        host_names = self.get_unit_hostnames(sentry_units)
+        errors = []
+
+        # Query every unit for cluster_status running nodes
+        for query_unit in sentry_units:
+            query_unit_name = query_unit.info['unit_name']
+            running_nodes = self.get_rmq_cluster_running_nodes(query_unit)
+
+            # Confirm that every unit is represented in the queried unit's
+            # cluster_status running nodes output.
+            for validate_unit in sentry_units:
+                val_host_name = host_names[validate_unit.info['unit_name']]
+                val_node_name = 'rabbit@{}'.format(val_host_name)
+
+                if val_node_name not in running_nodes:
+                    errors.append('Cluster member check failed on {}: {} not '
+                                  'in {}\n'.format(query_unit_name,
+                                                   val_node_name,
+                                                   running_nodes))
+        if errors:
+            return ''.join(errors)
+
+    def rmq_ssl_is_enabled_on_unit(self, sentry_unit, port=None):
+        """Check a single juju rmq unit for ssl and port in the config file."""
+        host = sentry_unit.info['public-address']
+        unit_name = sentry_unit.info['unit_name']
+
+        conf_file = '/etc/rabbitmq/rabbitmq.config'
+        conf_contents = str(self.file_contents_safe(sentry_unit,
+                                                    conf_file, max_wait=16))
+        # Checks
+        conf_ssl = 'ssl' in conf_contents
+        conf_port = str(port) in conf_contents
+
+        # Port explicitly checked in config
+        if port and conf_port and conf_ssl:
+            self.log.debug('SSL is enabled  @{}:{} '
+                           '({})'.format(host, port, unit_name))
+            return True
+        elif port and not conf_port and conf_ssl:
+            self.log.debug('SSL is enabled @{} but not on port {} '
+                           '({})'.format(host, port, unit_name))
+            return False
+        # Port not checked (useful when checking that ssl is disabled)
+        elif not port and conf_ssl:
+            self.log.debug('SSL is enabled  @{}:{} '
+                           '({})'.format(host, port, unit_name))
+            return True
+        elif not port and not conf_ssl:
+            self.log.debug('SSL not enabled @{}:{} '
+                           '({})'.format(host, port, unit_name))
+            return False
+        else:
+            msg = ('Unknown condition when checking SSL status @{}:{} '
+                   '({})'.format(host, port, unit_name))
+            amulet.raise_status(amulet.FAIL, msg)
+
+    def validate_rmq_ssl_enabled_units(self, sentry_units, port=None):
+        """Check that ssl is enabled on rmq juju sentry units.
+
+        :param sentry_units: list of all rmq sentry units
+        :param port: optional ssl port override to validate
+        :returns: None if successful, otherwise return error message
+        """
+        for sentry_unit in sentry_units:
+            if not self.rmq_ssl_is_enabled_on_unit(sentry_unit, port=port):
+                return ('Unexpected condition:  ssl is disabled on unit '
+                        '({})'.format(sentry_unit.info['unit_name']))
+        return None
+
+    def validate_rmq_ssl_disabled_units(self, sentry_units):
+        """Check that ssl is enabled on listed rmq juju sentry units.
+
+        :param sentry_units: list of all rmq sentry units
+        :returns: True if successful.  Raise on error.
+        """
+        for sentry_unit in sentry_units:
+            if self.rmq_ssl_is_enabled_on_unit(sentry_unit):
+                return ('Unexpected condition:  ssl is enabled on unit '
+                        '({})'.format(sentry_unit.info['unit_name']))
+        return None
+
+    def configure_rmq_ssl_on(self, sentry_units, deployment,
+                             port=None, max_wait=60):
+        """Turn ssl charm config option on, with optional non-default
+        ssl port specification.  Confirm that it is enabled on every
+        unit.
+
+        :param sentry_units: list of sentry units
+        :param deployment: amulet deployment object pointer
+        :param port: amqp port, use defaults if None
+        :param max_wait: maximum time to wait in seconds to confirm
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug('Setting ssl charm config option:  on')
+
+        # Enable RMQ SSL
+        config = {'ssl': 'on'}
+        if port:
+            config['ssl_port'] = port
+
+        deployment.configure('rabbitmq-server', config)
+
+        # Confirm
+        tries = 0
+        ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
+        while ret and tries < (max_wait / 4):
+            time.sleep(4)
+            self.log.debug('Attempt {}: {}'.format(tries, ret))
+            ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
+            tries += 1
+
+        if ret:
+            amulet.raise_status(amulet.FAIL, ret)
+
+    def configure_rmq_ssl_off(self, sentry_units, deployment, max_wait=60):
+        """Turn ssl charm config option off, confirm that it is disabled
+        on every unit.
+
+        :param sentry_units: list of sentry units
+        :param deployment: amulet deployment object pointer
+        :param max_wait: maximum time to wait in seconds to confirm
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug('Setting ssl charm config option:  off')
+
+        # Disable RMQ SSL
+        config = {'ssl': 'off'}
+        deployment.configure('rabbitmq-server', config)
+
+        # Confirm
+        tries = 0
+        ret = self.validate_rmq_ssl_disabled_units(sentry_units)
+        while ret and tries < (max_wait / 4):
+            time.sleep(4)
+            self.log.debug('Attempt {}: {}'.format(tries, ret))
+            ret = self.validate_rmq_ssl_disabled_units(sentry_units)
+            tries += 1
+
+        if ret:
+            amulet.raise_status(amulet.FAIL, ret)
+
+    def connect_amqp_by_unit(self, sentry_unit, ssl=False,
+                             port=None, fatal=True,
+                             username="testuser1", password="changeme"):
+        """Establish and return a pika amqp connection to the rabbitmq service
+        running on a rmq juju unit.
+
+        :param sentry_unit: sentry unit pointer
+        :param ssl: boolean, default to False
+        :param port: amqp port, use defaults if None
+        :param fatal: boolean, default to True (raises on connect error)
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :returns: pika amqp connection pointer or None if failed and non-fatal
+        """
+        host = sentry_unit.info['public-address']
+        unit_name = sentry_unit.info['unit_name']
+
+        # Default port logic if port is not specified
+        if ssl and not port:
+            port = 5671
+        elif not ssl and not port:
+            port = 5672
+
+        self.log.debug('Connecting to amqp on {}:{} ({}) as '
+                       '{}...'.format(host, port, unit_name, username))
+
+        try:
+            credentials = pika.PlainCredentials(username, password)
+            parameters = pika.ConnectionParameters(host=host, port=port,
+                                                   credentials=credentials,
+                                                   ssl=ssl,
+                                                   connection_attempts=3,
+                                                   retry_delay=5,
+                                                   socket_timeout=1)
+            connection = pika.BlockingConnection(parameters)
+            assert connection.server_properties['product'] == 'RabbitMQ'
+            self.log.debug('Connect OK')
+            return connection
+        except Exception as e:
+            msg = ('amqp connection failed to {}:{} as '
+                   '{} ({})'.format(host, port, username, str(e)))
+            if fatal:
+                amulet.raise_status(amulet.FAIL, msg)
+            else:
+                self.log.warn(msg)
+                return None
+
+    def publish_amqp_message_by_unit(self, sentry_unit, message,
+                                     queue="test", ssl=False,
+                                     username="testuser1",
+                                     password="changeme",
+                                     port=None):
+        """Publish an amqp message to a rmq juju unit.
+
+        :param sentry_unit: sentry unit pointer
+        :param message: amqp message string
+        :param queue: message queue, default to test
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :param ssl: boolean, default to False
+        :param port: amqp port, use defaults if None
+        :returns: None.  Raises exception if publish failed.
+        """
+        self.log.debug('Publishing message to {} queue:\n{}'.format(queue,
+                                                                    message))
+        connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
+                                               port=port,
+                                               username=username,
+                                               password=password)
+
+        # NOTE(beisner): extra debug here re: pika hang potential:
+        #   https://github.com/pika/pika/issues/297
+        #   https://groups.google.com/forum/#!topic/rabbitmq-users/Ja0iyfF0Szw
+        self.log.debug('Defining channel...')
+        channel = connection.channel()
+        self.log.debug('Declaring queue...')
+        channel.queue_declare(queue=queue, auto_delete=False, durable=True)
+        self.log.debug('Publishing message...')
+        channel.basic_publish(exchange='', routing_key=queue, body=message)
+        self.log.debug('Closing channel...')
+        channel.close()
+        self.log.debug('Closing connection...')
+        connection.close()
+
+    def get_amqp_message_by_unit(self, sentry_unit, queue="test",
+                                 username="testuser1",
+                                 password="changeme",
+                                 ssl=False, port=None):
+        """Get an amqp message from a rmq juju unit.
+
+        :param sentry_unit: sentry unit pointer
+        :param queue: message queue, default to test
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :param ssl: boolean, default to False
+        :param port: amqp port, use defaults if None
+        :returns: amqp message body as string.  Raise if get fails.
+        """
+        connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
+                                               port=port,
+                                               username=username,
+                                               password=password)
+        channel = connection.channel()
+        method_frame, _, body = channel.basic_get(queue)
+
+        if method_frame:
+            self.log.debug('Retreived message from {} queue:\n{}'.format(queue,
+                                                                         body))
+            channel.basic_ack(method_frame.delivery_tag)
+            channel.close()
+            connection.close()
+            return body
+        else:
+            msg = 'No message retrieved.'
+            amulet.raise_status(amulet.FAIL, msg)

charmhelpers/contrib/openstack/utils.py

@@ -114,6 +114,7 @@ SWIFT_CODENAMES = OrderedDict([
     ('2.2.1', 'kilo'),
     ('2.2.2', 'kilo'),
     ('2.3.0', 'liberty'),
+    ('2.4.0', 'liberty'),
 ])
 
 # >= Liberty version->codename mapping
@@ -142,6 +143,9 @@ PACKAGE_CODENAMES = {
     'glance-common': OrderedDict([
         ('11.0.0', 'liberty'),
     ]),
+    'openstack-dashboard': OrderedDict([
+        ('8.0.0', 'liberty'),
+    ]),
 }
 
 DEFAULT_LOOPBACK_SIZE = '5G'

charmhelpers/contrib/storage/linux/ceph.py

@@ -56,6 +56,8 @@ from charmhelpers.fetch import (
     apt_install,
 )
 
+from charmhelpers.core.kernel import modprobe
+
 KEYRING = '/etc/ceph/ceph.client.{}.keyring'
 KEYFILE = '/etc/ceph/ceph.client.{}.key'
 
@@ -288,17 +290,6 @@ def place_data_on_block_device(blk_device, data_src_dst):
     os.chown(data_src_dst, uid, gid)
 
 
-# TODO: re-use
-def modprobe(module):
-    """Load a kernel module and configure for auto-load on reboot."""
-    log('Loading kernel module', level=INFO)
-    cmd = ['modprobe', module]
-    check_call(cmd)
-    with open('/etc/modules', 'r+') as modules:
-        if module not in modules.read():
-            modules.write(module)
-
-
 def copy_files(src, dst, symlinks=False, ignore=None):
     """Copy files from src to dst."""
     for item in os.listdir(src):

charmhelpers/core/host.py

@@ -63,32 +63,48 @@ def service_reload(service_name, restart_on_failure=False):
     return service_result
 
 
-def service_pause(service_name, init_dir=None):
+def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"):
     """Pause a system service.
 
     Stop it, and prevent it from starting again at boot."""
-    if init_dir is None:
-        init_dir = "/etc/init"
     stopped = service_stop(service_name)
-    # XXX: Support systemd too
-    override_path = os.path.join(
-        init_dir, '{}.override'.format(service_name))
-    with open(override_path, 'w') as fh:
-        fh.write("manual\n")
+    upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
+    sysv_file = os.path.join(initd_dir, service_name)
+    if os.path.exists(upstart_file):
+        override_path = os.path.join(
+            init_dir, '{}.override'.format(service_name))
+        with open(override_path, 'w') as fh:
+            fh.write("manual\n")
+    elif os.path.exists(sysv_file):
+        subprocess.check_call(["update-rc.d", service_name, "disable"])
+    else:
+        # XXX: Support SystemD too
+        raise ValueError(
+            "Unable to detect {0} as either Upstart {1} or SysV {2}".format(
+                service_name, upstart_file, sysv_file))
     return stopped
 
 
-def service_resume(service_name, init_dir=None):
+def service_resume(service_name, init_dir="/etc/init",
+                   initd_dir="/etc/init.d"):
     """Resume a system service.
 
     Reenable starting again at boot. Start the service"""
-    # XXX: Support systemd too
-    if init_dir is None:
-        init_dir = "/etc/init"
-    override_path = os.path.join(
-        init_dir, '{}.override'.format(service_name))
-    if os.path.exists(override_path):
-        os.unlink(override_path)
+    upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
+    sysv_file = os.path.join(initd_dir, service_name)
+    if os.path.exists(upstart_file):
+        override_path = os.path.join(
+            init_dir, '{}.override'.format(service_name))
+        if os.path.exists(override_path):
+            os.unlink(override_path)
+    elif os.path.exists(sysv_file):
+        subprocess.check_call(["update-rc.d", service_name, "enable"])
+    else:
+        # XXX: Support SystemD too
+        raise ValueError(
+            "Unable to detect {0} as either Upstart {1} or SysV {2}".format(
+                service_name, upstart_file, sysv_file))
+
     started = service_start(service_name)
     return started
 

charmhelpers/core/kernel.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
+
+from charmhelpers.core.hookenv import (
+    log,
+    INFO
+)
+
+from subprocess import check_call, check_output
+import re
+
+
+def modprobe(module, persist=True):
+    """Load a kernel module and configure for auto-load on reboot."""
+    cmd = ['modprobe', module]
+
+    log('Loading kernel module %s' % module, level=INFO)
+
+    check_call(cmd)
+    if persist:
+        with open('/etc/modules', 'r+') as modules:
+            if module not in modules.read():
+                modules.write(module)
+
+
+def rmmod(module, force=False):
+    """Remove a module from the linux kernel"""
+    cmd = ['rmmod']
+    if force:
+        cmd.append('-f')
+    cmd.append(module)
+    log('Removing kernel module %s' % module, level=INFO)
+    return check_call(cmd)
+
+
+def lsmod():
+    """Shows what kernel modules are currently loaded"""
+    return check_output(['lsmod'],
+                        universal_newlines=True)
+
+
+def is_module_loaded(module):
+    """Checks if a kernel module is already loaded"""
+    matches = re.findall('^%s[ ]+' % module, lsmod(), re.M)
+    return len(matches) > 0
+
+
+def update_initramfs(version='all'):
+    """Updates an initramfs image"""
+    return check_call(["update-initramfs", "-k", version, "-u"])

hooks/ceilometer_contexts.py

@@ -1,121 +0,0 @@
-from charmhelpers.core.hookenv import (
-    relation_ids,
-    relation_get,
-    related_units,
-    config
-)
-
-from charmhelpers.contrib.openstack.utils import os_release
-
-from charmhelpers.contrib.openstack.context import (
-    OSContextGenerator,
-    context_complete,
-    ApacheSSLContext as SSLContext,
-)
-
-from charmhelpers.contrib.hahelpers.cluster import (
-    determine_apache_port,
-    determine_api_port
-)
-
-CEILOMETER_DB = 'ceilometer'
-
-
-class LoggingConfigContext(OSContextGenerator):
-    def __call__(self):
-        return {'debug': config('debug'), 'verbose': config('verbose')}
-
-
-class MongoDBContext(OSContextGenerator):
-    interfaces = ['mongodb']
-
-    def __call__(self):
-        mongo_servers = []
-        replset = None
-        use_replset = os_release('ceilometer-api') >= 'icehouse'
-
-        for relid in relation_ids('shared-db'):
-            rel_units = related_units(relid)
-            use_replset = use_replset and (len(rel_units) > 1)
-
-            for unit in rel_units:
-                host = relation_get('hostname', unit, relid)
-                port = relation_get('port', unit, relid)
-
-                conf = {
-                    "db_host": host,
-                    "db_port": port,
-                    "db_name": CEILOMETER_DB
-                }
-
-                if not context_complete(conf):
-                    continue
-
-                if not use_replset:
-                    return conf
-
-                if replset is None:
-                    replset = relation_get('replset', unit, relid)
-
-                mongo_servers.append('{}:{}'.format(host, port))
-
-        if mongo_servers:
-            return {
-                'db_mongo_servers': ','.join(mongo_servers),
-                'db_name': CEILOMETER_DB,
-                'db_replset': replset
-            }
-
-        return {}
-
-
-CEILOMETER_PORT = 8777
-
-
-class CeilometerContext(OSContextGenerator):
-    def __call__(self):
-        # Lazy-import to avoid a circular dependency in the imports
-        from ceilometer_utils import get_shared_secret
-
-        ctxt = {
-            'port': CEILOMETER_PORT,
-            'metering_secret': get_shared_secret()
-        }
-        return ctxt
-
-
-class CeilometerServiceContext(OSContextGenerator):
-    interfaces = ['ceilometer-service']
-
-    def __call__(self):
-        for relid in relation_ids('ceilometer-service'):
-            for unit in related_units(relid):
-                conf = relation_get(unit=unit, rid=relid)
-                if context_complete(conf):
-                    return conf
-        return {}
-
-
-class HAProxyContext(OSContextGenerator):
-    interfaces = ['ceilometer-haproxy']
-
-    def __call__(self):
-        '''Extends the main charmhelpers HAProxyContext with a port mapping
-        specific to this charm.
-        '''
-        haproxy_port = CEILOMETER_PORT
-        api_port = determine_api_port(CEILOMETER_PORT, singlenode_mode=True)
-        apache_port = determine_apache_port(CEILOMETER_PORT,
-                                            singlenode_mode=True)
-
-        ctxt = {
-            'service_ports': {'ceilometer_api': [haproxy_port, apache_port]},
-            'port': api_port
-        }
-        return ctxt
-
-
-class ApacheSSLContext(SSLContext):
-
-    external_ports = [CEILOMETER_PORT]
-    service_namespace = "ceilometer"

hooks/ceilometer_contexts.py

@@ -0,0 +1 @@
+../ceilometer_contexts.py

hooks/ceilometer_hooks.py

@@ -66,8 +66,7 @@ CONFIGS = register_configs()
 def install():
     execd_preinstall()
     origin = config('openstack-origin')
-    if (lsb_release()['DISTRIB_CODENAME'] == 'precise'
-            and origin == 'distro'):
+    if (lsb_release()['DISTRIB_CODENAME'] == 'precise' and origin == 'distro'):
         origin = 'cloud:precise-grizzly'
     configure_installation_source(origin)
     apt_update(fatal=True)

hooks/ceilometer_utils.py

@@ -1,225 +0,0 @@
-import os
-import uuid
-
-from collections import OrderedDict
-
-from charmhelpers.contrib.openstack import (
-    templating,
-    context,
-)
-from ceilometer_contexts import (
-    ApacheSSLContext,
-    LoggingConfigContext,
-    MongoDBContext,
-    CeilometerContext,
-    HAProxyContext
-)
-from charmhelpers.contrib.openstack.utils import (
-    get_os_codename_package,
-    get_os_codename_install_source,
-    configure_installation_source
-)
-from charmhelpers.core.hookenv import config, log
-from charmhelpers.fetch import apt_update, apt_install, apt_upgrade
-from copy import deepcopy
-
-HAPROXY_CONF = '/etc/haproxy/haproxy.cfg'
-CEILOMETER_CONF_DIR = "/etc/ceilometer"
-CEILOMETER_CONF = "%s/ceilometer.conf" % CEILOMETER_CONF_DIR
-HTTPS_APACHE_CONF = "/etc/apache2/sites-available/openstack_https_frontend"
-HTTPS_APACHE_24_CONF = "/etc/apache2/sites-available/" \
-    "openstack_https_frontend.conf"
-CLUSTER_RES = 'grp_ceilometer_vips'
-
-CEILOMETER_SERVICES = [
-    'ceilometer-agent-central',
-    'ceilometer-collector',
-    'ceilometer-api',
-    'ceilometer-alarm-evaluator',
-    'ceilometer-alarm-notifier',
-    'ceilometer-agent-notification',
-]
-
-CEILOMETER_DB = "ceilometer"
-CEILOMETER_SERVICE = "ceilometer"
-
-CEILOMETER_PACKAGES = [
-    'haproxy',
-    'apache2',
-    'ceilometer-agent-central',
-    'ceilometer-collector',
-    'ceilometer-api',
-    'python-pymongo',
-]
-
-ICEHOUSE_PACKAGES = [
-    'ceilometer-alarm-notifier',
-    'ceilometer-alarm-evaluator',
-    'ceilometer-agent-notification'
-]
-
-ICEHOUSE_SERVICES = [
-    'ceilometer-alarm-notifier',
-    'ceilometer-alarm-evaluator',
-    'ceilometer-agent-notification'
-]
-
-CEILOMETER_ROLE = "ResellerAdmin"
-SVC = 'ceilometer'
-
-CONFIG_FILES = OrderedDict([
-    (CEILOMETER_CONF, {
-        'hook_contexts': [context.IdentityServiceContext(service=SVC,
-                                                         service_user=SVC),
-                          context.AMQPContext(ssl_dir=CEILOMETER_CONF_DIR),
-                          LoggingConfigContext(),
-                          MongoDBContext(),
-                          CeilometerContext(),
-                          context.SyslogContext(),
-                          HAProxyContext()],
-        'services': CEILOMETER_SERVICES
-    }),
-    (HAPROXY_CONF, {
-        'hook_contexts': [context.HAProxyContext(singlenode_mode=True),
-                          HAProxyContext()],
-        'services': ['haproxy'],
-    }),
-    (HTTPS_APACHE_CONF, {
-        'hook_contexts': [ApacheSSLContext()],
-        'services': ['apache2'],
-    }),
-    (HTTPS_APACHE_24_CONF, {
-        'hook_contexts': [ApacheSSLContext()],
-        'services': ['apache2'],
-    })
-])
-
-TEMPLATES = 'templates'
-
-SHARED_SECRET = "/etc/ceilometer/secret.txt"
-
-
-def register_configs():
-    """
-    Register config files with their respective contexts.
-    Regstration of some configs may not be required depending on
-    existing of certain relations.
-    """
-    # if called without anything installed (eg during install hook)
-    # just default to earliest supported release. configs dont get touched
-    # till post-install, anyway.
-    release = get_os_codename_package('ceilometer-common', fatal=False) \
-        or 'grizzly'
-    configs = templating.OSConfigRenderer(templates_dir=TEMPLATES,
-                                          openstack_release=release)
-
-    if (get_os_codename_install_source(config('openstack-origin'))
-            >= 'icehouse'):
-        CONFIG_FILES[CEILOMETER_CONF]['services'] = \
-            CONFIG_FILES[CEILOMETER_CONF]['services'] + ICEHOUSE_SERVICES
-
-    for conf in CONFIG_FILES:
-        configs.register(conf, CONFIG_FILES[conf]['hook_contexts'])
-
-    if os.path.exists('/etc/apache2/conf-available'):
-        configs.register(HTTPS_APACHE_24_CONF,
-                         CONFIG_FILES[HTTPS_APACHE_24_CONF]['hook_contexts'])
-    else:
-        configs.register(HTTPS_APACHE_CONF,
-                         CONFIG_FILES[HTTPS_APACHE_CONF]['hook_contexts'])
-    return configs
-
-
-def restart_map():
-    '''
-    Determine the correct resource map to be passed to
-    charmhelpers.core.restart_on_change() based on the services configured.
-
-    :returns: dict: A dictionary mapping config file to lists of services
-                    that should be restarted when file changes.
-    '''
-    _map = {}
-    for f, ctxt in CONFIG_FILES.iteritems():
-        svcs = []
-        for svc in ctxt['services']:
-            svcs.append(svc)
-        if svcs:
-            _map[f] = svcs
-    return _map
-
-
-def services():
-    ''' Returns a list of services associate with this charm '''
-    _services = []
-    for v in restart_map().values():
-        _services = _services + v
-    return list(set(_services))
-
-
-def get_ceilometer_context():
-    ''' Retrieve a map of all current relation data for agent configuration '''
-    ctxt = {}
-    for hcontext in CONFIG_FILES[CEILOMETER_CONF]['hook_contexts']:
-        ctxt.update(hcontext())
-    return ctxt
-
-
-def do_openstack_upgrade(configs):
-    """
-    Perform an upgrade.  Takes care of upgrading packages, rewriting
-    configs, database migrations and potentially any other post-upgrade
-    actions.
-
-    :param configs: The charms main OSConfigRenderer object.
-    """
-    new_src = config('openstack-origin')
-    new_os_rel = get_os_codename_install_source(new_src)
-
-    log('Performing OpenStack upgrade to %s.' % (new_os_rel))
-
-    configure_installation_source(new_src)
-    dpkg_opts = [
-        '--option', 'Dpkg::Options::=--force-confnew',
-        '--option', 'Dpkg::Options::=--force-confdef',
-    ]
-    apt_update(fatal=True)
-    apt_upgrade(options=dpkg_opts, fatal=True, dist=True)
-    apt_install(packages=get_packages(),
-                options=dpkg_opts,
-                fatal=True)
-
-    # set CONFIGS to load templates from new release
-    configs.set_release(openstack_release=new_os_rel)
-
-
-def get_packages():
-    packages = deepcopy(CEILOMETER_PACKAGES)
-    if (get_os_codename_install_source(config('openstack-origin'))
-            >= 'icehouse'):
-        packages = packages + ICEHOUSE_PACKAGES
-    return packages
-
-
-def get_shared_secret():
-    """
-    Returns the current shared secret for the ceilometer node. If the shared
-    secret does not exist, this method will generate one.
-    """
-    secret = None
-    if not os.path.exists(SHARED_SECRET):
-        secret = str(uuid.uuid4())
-        set_shared_secret(secret)
-    else:
-        with open(SHARED_SECRET, 'r') as secret_file:
-            secret = secret_file.read().strip()
-    return secret
-
-
-def set_shared_secret(secret):
-    """
-    Sets the shared secret which is used to sign ceilometer messages.
-
-    :param secret: the secret to set
-    """
-    with open(SHARED_SECRET, 'w') as secret_file:
-        secret_file.write(secret)

hooks/ceilometer_utils.py

@@ -0,0 +1 @@
+../ceilometer_utils.py

hooks/charmhelpers

@@ -0,0 +1 @@
+../charmhelpers

tests/basic_deployment.py

@@ -1,9 +1,12 @@
 #!/usr/bin/python
 
+import subprocess
+
 """
 Basic ceilometer functional tests.
 """
 import amulet
+import json
 import time
 from ceilometerclient.v2 import client as ceilclient
 
@@ -107,6 +110,32 @@ class CeilometerBasicDeployment(OpenStackAmuletDeployment):
                                                    endpoint_type='publicURL')
         self.ceil = ceilclient.Client(endpoint=ep, token=self._get_token)
 
+    def _run_action(self, unit_id, action, *args):
+        command = ["juju", "action", "do", "--format=json", unit_id, action]
+        command.extend(args)
+        print("Running command: %s\n" % " ".join(command))
+        output = subprocess.check_output(command)
+        output_json = output.decode(encoding="UTF-8")
+        data = json.loads(output_json)
+        action_id = data[u'Action queued with id']
+        return action_id
+
+    def _wait_on_action(self, action_id):
+        command = ["juju", "action", "fetch", "--format=json", action_id]
+        while True:
+            try:
+                output = subprocess.check_output(command)
+            except Exception as e:
+                print(e)
+                return False
+            output_json = output.decode(encoding="UTF-8")
+            data = json.loads(output_json)
+            if data[u"status"] == "completed":
+                return True
+            elif data[u"status"] == "failed":
+                return False
+            time.sleep(2)
+
     def test_100_services(self):
         """Verify the expected services are running on the corresponding
            service units."""
@@ -569,3 +598,18 @@ class CeilometerBasicDeployment(OpenStackAmuletDeployment):
             sleep_time = 0
 
         self.d.configure(juju_service, set_default)
+
+    def test_1000_pause_and_resume(self):
+        """The services can be paused and resumed. """
+        unit_name = "ceilometer/0"
+        unit = self.d.sentry.unit[unit_name]
+
+        assert u.status_get(unit)[0] == "unknown"
+
+        action_id = self._run_action(unit_name, "pause")
+        assert self._wait_on_action(action_id), "Pause action failed."
+        assert u.status_get(unit)[0] == "maintenance"
+
+        action_id = self._run_action(unit_name, "resume")
+        assert self._wait_on_action(action_id), "Resume action failed."
+        assert u.status_get(unit)[0] == "active"

tests/charmhelpers/contrib/amulet/utils.py

@@ -594,3 +594,12 @@ class AmuletUtils(object):
         output = _check_output(command, universal_newlines=True)
         data = json.loads(output)
         return data.get(u"status") == "completed"
+
+    def status_get(self, unit):
+        """Return the current service status of this unit."""
+        raw_status, return_code = unit.run(
+            "status-get --format=json --include-data")
+        if return_code != 0:
+            return ("unknown", "")
+        status = json.loads(raw_status)
+        return (status["status"], status["message"])