Workload status

hooks/charmhelpers/contrib/openstack/context.py

@@ -194,10 +194,50 @@ def config_flags_parser(config_flags):
 class OSContextGenerator(object):
     """Base class for all context generators."""
     interfaces = []
+    related = False
+    complete = False
+    missing_data = []
 
     def __call__(self):
         raise NotImplementedError
 
+    def context_complete(self, ctxt):
+        """Check for missing data for the required context data.
+        Set self.missing_data if it exists and return False.
+        Set self.complete if no missing data and return True.
+        """
+        # Fresh start
+        self.complete = False
+        self.missing_data = []
+        for k, v in six.iteritems(ctxt):
+            if v is None or v == '':
+                if k not in self.missing_data:
+                    self.missing_data.append(k)
+
+        if self.missing_data:
+            self.complete = False
+            log('Missing required data: %s' % ' '.join(self.missing_data), level=INFO)
+        else:
+            self.complete = True
+        return self.complete
+
+    def get_related(self):
+        """Check if any of the context interfaces have relation ids.
+        Set self.related and return True if one of the interfaces
+        has relation ids.
+        """
+        # Fresh start
+        self.related = False
+        try:
+            for interface in self.interfaces:
+                if relation_ids(interface):
+                    self.related = True
+            return self.related
+        except AttributeError as e:
+            log("{} {}"
+                "".format(self, e), 'INFO')
+            return self.related
+
 
 class SharedDBContext(OSContextGenerator):
     interfaces = ['shared-db']
@@ -213,6 +253,7 @@ class SharedDBContext(OSContextGenerator):
         self.database = database
         self.user = user
         self.ssl_dir = ssl_dir
+        self.rel_name = self.interfaces[0]
 
     def __call__(self):
         self.database = self.database or config('database')
@@ -246,6 +287,7 @@ class SharedDBContext(OSContextGenerator):
             password_setting = self.relation_prefix + '_password'
 
         for rid in relation_ids(self.interfaces[0]):
+            self.related = True
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 host = rdata.get('db_host')
@@ -257,7 +299,7 @@ class SharedDBContext(OSContextGenerator):
                     'database_password': rdata.get(password_setting),
                     'database_type': 'mysql'
                 }
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     db_ssl(rdata, ctxt, self.ssl_dir)
                     return ctxt
         return {}
@@ -278,6 +320,7 @@ class PostgresqlDBContext(OSContextGenerator):
 
         ctxt = {}
         for rid in relation_ids(self.interfaces[0]):
+            self.related = True
             for unit in related_units(rid):
                 rel_host = relation_get('host', rid=rid, unit=unit)
                 rel_user = relation_get('user', rid=rid, unit=unit)
@@ -287,7 +330,7 @@ class PostgresqlDBContext(OSContextGenerator):
                         'database_user': rel_user,
                         'database_password': rel_passwd,
                         'database_type': 'postgresql'}
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     return ctxt
 
         return {}
@@ -348,6 +391,7 @@ class IdentityServiceContext(OSContextGenerator):
             ctxt['signing_dir'] = cachedir
 
         for rid in relation_ids(self.rel_name):
+            self.related = True
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 serv_host = rdata.get('service_host')
@@ -366,7 +410,7 @@ class IdentityServiceContext(OSContextGenerator):
                              'service_protocol': svc_protocol,
                              'auth_protocol': auth_protocol})
 
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     # NOTE(jamespage) this is required for >= icehouse
                     # so a missing value just indicates keystone needs
                     # upgrading
@@ -405,6 +449,7 @@ class AMQPContext(OSContextGenerator):
         ctxt = {}
         for rid in relation_ids(self.rel_name):
             ha_vip_only = False
+            self.related = True
             for unit in related_units(rid):
                 if relation_get('clustered', rid=rid, unit=unit):
                     ctxt['clustered'] = True
@@ -437,7 +482,7 @@ class AMQPContext(OSContextGenerator):
                 ha_vip_only = relation_get('ha-vip-only',
                                            rid=rid, unit=unit) is not None
 
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     if 'rabbit_ssl_ca' in ctxt:
                         if not self.ssl_dir:
                             log("Charm not setup for ssl support but ssl ca "
@@ -469,7 +514,7 @@ class AMQPContext(OSContextGenerator):
             ctxt['oslo_messaging_flags'] = config_flags_parser(
                 oslo_messaging_flags)
 
-        if not context_complete(ctxt):
+        if not self.complete:
             return {}
 
         return ctxt
@@ -507,7 +552,7 @@ class CephContext(OSContextGenerator):
         if not os.path.isdir('/etc/ceph'):
             os.mkdir('/etc/ceph')
 
-        if not context_complete(ctxt):
+        if not self.context_complete(ctxt):
             return {}
 
         ensure_packages(['ceph-common'])
@@ -1366,6 +1411,6 @@ class NetworkServiceContext(OSContextGenerator):
                     'auth_protocol':
                     rdata.get('auth_protocol') or 'http',
                 }
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     return ctxt
         return {}

hooks/charmhelpers/contrib/openstack/templating.py

@@ -18,7 +18,7 @@ import os
 
 import six
 
-from charmhelpers.fetch import apt_install
+from charmhelpers.fetch import apt_install, apt_update
 from charmhelpers.core.hookenv import (
     log,
     ERROR,
@@ -29,6 +29,7 @@ from charmhelpers.contrib.openstack.utils import OPENSTACK_CODENAMES
 try:
     from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 except ImportError:
+    apt_update(fatal=True)
     apt_install('python-jinja2', fatal=True)
     from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 
@@ -112,7 +113,7 @@ class OSConfigTemplate(object):
 
     def complete_contexts(self):
         '''
-        Return a list of interfaces that have atisfied contexts.
+        Return a list of interfaces that have satisfied contexts.
         '''
         if self._complete_contexts:
             return self._complete_contexts
@@ -293,3 +294,30 @@ class OSConfigRenderer(object):
         [interfaces.extend(i.complete_contexts())
          for i in six.itervalues(self.templates)]
         return interfaces
+
+    def get_incomplete_context_data(self, interfaces):
+        '''
+        Return dictionary of relation status of interfaces and any missing
+        required context data. Example:
+            {'amqp': {'missing_data': ['rabbitmq_password'], 'related': True},
+             'zeromq-configuration': {'related': False}}
+        '''
+        incomplete_context_data = {}
+
+        for i in six.itervalues(self.templates):
+            for context in i.contexts:
+                for interface in interfaces:
+                    related = False
+                    if interface in context.interfaces:
+                        related = context.get_related()
+                        missing_data = context.missing_data
+                        if missing_data:
+                            incomplete_context_data[interface] = {'missing_data': missing_data}
+                        if related:
+                            if incomplete_context_data.get(interface):
+                                incomplete_context_data[interface].update({'related': True})
+                            else:
+                                incomplete_context_data[interface] = {'related': True}
+                        else:
+                            incomplete_context_data[interface] = {'related': False}
+        return incomplete_context_data

hooks/charmhelpers/contrib/openstack/utils.py

@@ -42,7 +42,9 @@ from charmhelpers.core.hookenv import (
     charm_dir,
     INFO,
     relation_ids,
-    relation_set
+    relation_set,
+    status_set,
+    hook_name
 )
 
 from charmhelpers.contrib.storage.linux.lvm import (
@@ -754,6 +756,176 @@ def git_yaml_value(projects_yaml, key):
     return None
 
 
+def os_workload_status(configs, required_interfaces, charm_func=None):
+    """
+    Decorator to set workload status based on complete contexts
+    """
+    def wrap(f):
+        @wraps(f)
+        def wrapped_f(*args, **kwargs):
+            # Run the original function first
+            f(*args, **kwargs)
+            # Set workload status now that contexts have been
+            # acted on
+            set_os_workload_status(configs, required_interfaces, charm_func)
+        return wrapped_f
+    return wrap
+
+
+def set_os_workload_status(configs, required_interfaces, charm_func=None):
+    """
+    Set workload status based on complete contexts.
+    status-set missing or incomplete contexts
+    and juju-log details of missing required data.
+    charm_func is a charm specific function to run checking
+    for charm specific requirements such as a VIP setting.
+    """
+    incomplete_rel_data = incomplete_relation_data(configs, required_interfaces)
+    state = 'active'
+    missing_relations = []
+    incomplete_relations = []
+    message = None
+    charm_state = None
+    charm_message = None
+
+    for generic_interface in incomplete_rel_data.keys():
+        related_interface = None
+        missing_data = {}
+        # Related or not?
+        for interface in incomplete_rel_data[generic_interface]:
+            if incomplete_rel_data[generic_interface][interface].get('related'):
+                related_interface = interface
+                missing_data = incomplete_rel_data[generic_interface][interface].get('missing_data')
+        # No relation ID for the generic_interface
+        if not related_interface:
+            juju_log("{} relation is missing and must be related for "
+                     "functionality. ".format(generic_interface), 'WARN')
+            state = 'blocked'
+            if generic_interface not in missing_relations:
+                missing_relations.append(generic_interface)
+        else:
+            # Relation ID exists but no related unit
+            if not missing_data:
+                # Edge case relation ID exists but departing
+                if ('departed' in hook_name() or 'broken' in hook_name()) \
+                        and related_interface in hook_name():
+                    state = 'blocked'
+                    if generic_interface not in missing_relations:
+                        missing_relations.append(generic_interface)
+                    juju_log("{} relation's interface, {}, "
+                             "relationship is departed or broken "
+                             "and is required for functionality."
+                             "".format(generic_interface, related_interface), "WARN")
+                # Normal case relation ID exists but no related unit
+                # (joining)
+                else:
+                    juju_log("{} relations's interface, {}, is related but has "
+                             "no units in the relation."
+                             "".format(generic_interface, related_interface), "INFO")
+            # Related unit exists and data missing on the relation
+            else:
+                juju_log("{} relation's interface, {}, is related awaiting "
+                         "the following data from the relationship: {}. "
+                         "".format(generic_interface, related_interface,
+                                   ", ".join(missing_data)), "INFO")
+            if state != 'blocked':
+                state = 'waiting'
+            if generic_interface not in incomplete_relations \
+                    and generic_interface not in missing_relations:
+                incomplete_relations.append(generic_interface)
+
+    if missing_relations:
+        message = "Missing relations: {}".format(", ".join(missing_relations))
+        if incomplete_relations:
+            message += "; incomplete relations: {}" \
+                       "".format(", ".join(incomplete_relations))
+        state = 'blocked'
+    elif incomplete_relations:
+        message = "Incomplete relations: {}" \
+                  "".format(", ".join(incomplete_relations))
+        state = 'waiting'
+
+    # Run charm specific checks
+    if charm_func:
+        charm_state, charm_message = charm_func(configs)
+        if charm_state != 'active' and charm_state != 'unknown':
+            state = workload_state_compare(state, charm_state)
+            if message:
+                message = "{} {}".format(message, charm_message)
+            else:
+                message = charm_message
+
+    # Set to active if all requirements have been met
+    if state == 'active':
+        message = "Unit is ready"
+        juju_log(message, "INFO")
+
+    status_set(state, message)
+
+
+def workload_state_compare(current_workload_state, workload_state):
+    """ Return highest priority of two states"""
+    hierarchy = {'unknown': -1,
+                 'active': 0,
+                 'maintenance': 1,
+                 'waiting': 2,
+                 'blocked': 3,
+                 }
+
+    if hierarchy.get(workload_state) is None:
+        workload_state = 'unknown'
+    if hierarchy.get(current_workload_state) is None:
+        current_workload_state = 'unknown'
+
+    # Set workload_state based on hierarchy of statuses
+    if hierarchy.get(current_workload_state) > hierarchy.get(workload_state):
+        return current_workload_state
+    else:
+        return workload_state
+
+
+def incomplete_relation_data(configs, required_interfaces):
+    """
+    Check complete contexts against required_interfaces
+    Return dictionary of incomplete relation data.
+
+    configs is an OSConfigRenderer object with configs registered
+
+    required_interfaces is a dictionary of required general interfaces
+    with dictionary values of possible specific interfaces.
+    Example:
+    required_interfaces = {'database': ['shared-db', 'pgsql-db']}
+
+    The interface is said to be satisfied if anyone of the interfaces in the
+    list has a complete context.
+
+    Return dictionary of incomplete or missing required contexts with relation
+    status of interfaces and any missing data points. Example:
+        {'message':
+             {'amqp': {'missing_data': ['rabbitmq_password'], 'related': True},
+              'zeromq-configuration': {'related': False}},
+         'identity':
+             {'identity-service': {'related': False}},
+         'database':
+             {'pgsql-db': {'related': False},
+              'shared-db': {'related': True}}}
+    """
+    complete_ctxts = configs.complete_contexts()
+    incomplete_relations = []
+    for svc_type in required_interfaces.keys():
+        # Avoid duplicates
+        found_ctxt = False
+        for interface in required_interfaces[svc_type]:
+            if interface in complete_ctxts:
+                found_ctxt = True
+        if not found_ctxt:
+            incomplete_relations.append(svc_type)
+    incomplete_context_data = {}
+    for i in incomplete_relations:
+        incomplete_context_data[i] = configs.get_incomplete_context_data(required_interfaces[i])
+    return incomplete_context_data
+
+
 def do_action_openstack_upgrade(package, upgrade_callback, configs):
     """Perform action-managed OpenStack upgrade.
 

hooks/cinder_hooks.py

@@ -27,6 +27,8 @@ from cinder_utils import (
     setup_ipv6,
     check_db_initialised,
     filesystem_mounted,
+    REQUIRED_INTERFACES,
+    check_ha_settings,
 )
 
 from charmhelpers.core.hookenv import (
@@ -42,6 +44,7 @@ from charmhelpers.core.hookenv import (
     unit_get,
     log,
     ERROR,
+    status_set,
 )
 
 from charmhelpers.fetch import (
@@ -63,6 +66,7 @@ from charmhelpers.contrib.openstack.utils import (
     openstack_upgrade_available,
     sync_db_with_multi_ipv6_addresses,
     os_release,
+    os_workload_status,
 )
 
 from charmhelpers.contrib.storage.linux.ceph import (
@@ -100,7 +104,9 @@ CONFIGS = register_configs()
 
 
 @hooks.hook('install.real')
+@os_workload_status(CONFIGS, REQUIRED_INTERFACES, charm_func=check_ha_settings)
 def install():
+    status_set('maintenance', 'Executing pre-install')
     execd_preinstall()
     conf = config()
     src = conf['openstack-origin']
@@ -109,18 +115,22 @@ def install():
         src = 'cloud:precise-folsom'
     configure_installation_source(src)
 
+    status_set('maintenance', 'Installing apt packages')
     apt_update()
     apt_install(determine_packages(), fatal=True)
 
+    status_set('maintenance', 'Git install')
     git_install(config('openstack-origin-git'))
 
 
 @hooks.hook('config-changed')
+@os_workload_status(CONFIGS, REQUIRED_INTERFACES, charm_func=check_ha_settings)
 @restart_on_change(restart_map(), stopstart=True)
 def config_changed():
     conf = config()
 
     if conf['prefer-ipv6']:
+        status_set('maintenance', 'configuring ipv6')
         setup_ipv6()
         sync_db_with_multi_ipv6_addresses(config('database'),
                                           config('database-user'))
@@ -131,6 +141,7 @@ def config_changed():
 
     if (service_enabled('volume') and
             conf['block-device'] not in [None, 'None', 'none']):
+        status_set('maintenance', 'Configuring lvm storage')
         block_devices = conf['block-device'].split()
         configure_lvm_storage(block_devices,
                               conf['volume-group'],
@@ -140,9 +151,11 @@ def config_changed():
 
     if git_install_requested():
         if config_value_changed('openstack-origin-git'):
+            status_set('maintenance', 'Running Git install')
             git_install(config('openstack-origin-git'))
     elif not config('action-managed-upgrade'):
         if openstack_upgrade_available('cinder-common'):
+            status_set('maintenance', 'Running openstack upgrade')
             do_openstack_upgrade(configs=CONFIGS)
             # NOTE(jamespage) tell any storage-backends we just upgraded
             for rid in relation_ids('storage-backend'):
@@ -164,6 +177,7 @@ def config_changed():
 
 
 @hooks.hook('shared-db-relation-joined')
+@os_workload_status(CONFIGS, REQUIRED_INTERFACES, charm_func=check_ha_settings)
 def db_joined():
     if is_relation_made('pgsql-db'):
         # error, postgresql is used
@@ -184,6 +198,7 @@ def db_joined():
 
 
 @hooks.hook('pgsql-db-relation-joined')
+@os_workload_status(CONFIGS, REQUIRED_INTERFACES, charm_func=check_ha_settings)
 def pgsql_db_joined():
     if is_relation_made('shared-db'):
         # raise error
@@ -197,6 +212,7 @@ def pgsql_db_joined():
 
 
 @hooks.hook('shared-db-relation-changed')
+@os_workload_status(CONFIGS, REQUIRED_INTERFACES, charm_func=check_ha_settings)
 @restart_on_change(restart_map())
 def db_changed():
     if 'shared-db' not in CONFIGS.complete_contexts():
@@ -217,6 +233,7 @@ def db_changed():
 
 
 @hooks.hook('pgsql-db-relation-changed')
+@os_workload_status(CONFIGS, REQUIRED_INTERFACES, charm_func=check_ha_settings)
 @restart_on_change(restart_map())
 def pgsql_db_changed():
     if 'pgsql-db' not in CONFIGS.complete_contexts():
@@ -229,6 +246,7 @@ def pgsql_db_changed():
 
 
 @hooks.hook('amqp-relation-joined')
+@os_workload_status(CONFIGS, REQUIRED_INTERFACES, charm_func=check_ha_settings)
 def amqp_joined(relation_id=None):
     conf = config()
     relation_set(relation_id=relation_id,
@@ -236,6 +254,7 @@ def amqp_joined(relation_id=None):
 
 
 @hooks.hook('amqp-relation-changed')
+@os_workload_status(CONFIGS, REQUIRED_INTERFACES, charm_func=check_ha_settings)
 @restart_on_change(restart_map())
 def amqp_changed():
     if 'amqp' not in CONFIGS.complete_contexts():
@@ -245,6 +264,7 @@ def amqp_changed():
 
 
 @hooks.hook('amqp-relation-departed')
+@os_workload_status(CONFIGS, REQUIRED_INTERFACES, charm_func=check_ha_settings)
 @restart_on_change(restart_map())
 def amqp_departed():
     if 'amqp' not in CONFIGS.complete_contexts():
@@ -254,6 +274,7 @@ def amqp_departed():
 
 
 @hooks.hook('identity-service-relation-joined')
+@os_workload_status(CONFIGS, REQUIRED_INTERFACES, charm_func=check_ha_settings)
 def identity_joined(rid=None):
     public_url = '{}:{}/v1/$(tenant_id)s'.format(
         canonical_url(CONFIGS, PUBLIC),
@@ -304,6 +325,7 @@ def identity_joined(rid=None):
 
 
 @hooks.hook('identity-service-relation-changed')
+@os_workload_status(CONFIGS, REQUIRED_INTERFACES, charm_func=check_ha_settings)
 @restart_on_change(restart_map())
 def identity_changed():
     if 'identity-service' not in CONFIGS.complete_contexts():
@@ -391,6 +413,7 @@ def cluster_changed():
 
 
 @hooks.hook('ha-relation-joined')
+@os_workload_status(CONFIGS, REQUIRED_INTERFACES, charm_func=check_ha_settings)
 def ha_joined(relation_id=None):
     cluster_config = get_hacluster_config()
 
@@ -448,6 +471,7 @@ def ha_joined(relation_id=None):
 
 
 @hooks.hook('ha-relation-changed')
+@os_workload_status(CONFIGS, REQUIRED_INTERFACES, charm_func=check_ha_settings)
 def ha_changed():
     clustered = relation_get('clustered')
     if not clustered or clustered in [None, 'None', '']:

hooks/cinder_utils.py

@@ -46,6 +46,7 @@ from charmhelpers.core.host import (
 from charmhelpers.contrib.openstack.alternatives import install_alternative
 from charmhelpers.contrib.hahelpers.cluster import (
     is_elected_leader,
+    get_hacluster_config,
 )
 
 from charmhelpers.contrib.storage.linux.utils import (
@@ -155,6 +156,15 @@ APACHE_SITE_24_CONF = '/etc/apache2/sites-available/' \
 
 TEMPLATES = 'templates/'
 
+# The interface is said to be satisfied if anyone of the interfaces in
+# the
+# list has a complete context.
+REQUIRED_INTERFACES = {
+    'database': ['shared-db', 'pgsql-db'],
+    'message': ['amqp'],
+    'identity': ['identity-service'],
+}
+
 
 def ceph_config_file():
     return CHARM_CEPH_CONF.format(service_name())
@@ -800,3 +810,16 @@ def git_post_install(projects_yaml):
 
 def filesystem_mounted(fs):
     return subprocess.call(['grep', '-wqs', fs, '/proc/mounts']) == 0
+
+
+def check_ha_settings(configs):
+    if relation_ids('ha'):
+        try:
+            get_hacluster_config()
+            return 'active', 'hacluster configs complete.'
+        except:
+            return ('blocked',
+                    'hacluster missing configuration: '
+                    'vip, vip_iface, vip_cidr')
+    else:
+        return 'unknown', 'No ha clustering'

unit_tests/test_utils.py

@@ -18,6 +18,9 @@ RESTART_MAP = OrderedDict([
     ('/etc/apache2/sites-available/openstack_https_frontend.conf', ['apache2'])
 ])
 
+patch('charmhelpers.contrib.openstack.utils.set_os_workload_status').start()
+patch('charmhelpers.core.hookenv.status_set').start()
+
 
 def load_config():
     '''Walk backwords from __file__ looking for config.yaml,