Charmhelper sync before 1604 testing

Change-Id: I87e827449ac2eaf8652979c5219ef43b60fbe7f7

hooks/charmhelpers/contrib/openstack/context.py

@@ -20,7 +20,7 @@ import os
 import re
 import time
 from base64 import b64decode
-from subprocess import check_call
+from subprocess import check_call, CalledProcessError
 
 import six
 import yaml
@@ -45,6 +45,7 @@ from charmhelpers.core.hookenv import (
     INFO,
     WARNING,
     ERROR,
+    status_set,
 )
 
 from charmhelpers.core.sysctl import create as sysctl_create
@@ -1491,3 +1492,92 @@ class InternalEndpointContext(OSContextGenerator):
     """
     def __call__(self):
         return {'use_internal_endpoints': config('use-internal-endpoints')}
+
+
+class AppArmorContext(OSContextGenerator):
+    """Base class for apparmor contexts."""
+
+    def __init__(self):
+        self._ctxt = None
+        self.aa_profile = None
+        self.aa_utils_packages = ['apparmor-utils']
+
+    @property
+    def ctxt(self):
+        if self._ctxt is not None:
+            return self._ctxt
+        self._ctxt = self._determine_ctxt()
+        return self._ctxt
+
+    def _determine_ctxt(self):
+        """
+        Validate aa-profile-mode settings is disable, enforce, or complain.
+
+        :return ctxt: Dictionary of the apparmor profile or None
+        """
+        if config('aa-profile-mode') in ['disable', 'enforce', 'complain']:
+            ctxt = {'aa-profile-mode': config('aa-profile-mode')}
+        else:
+            ctxt = None
+        return ctxt
+
+    def __call__(self):
+        return self.ctxt
+
+    def install_aa_utils(self):
+        """
+        Install packages required for apparmor configuration.
+        """
+        log("Installing apparmor utils.")
+        ensure_packages(self.aa_utils_packages)
+
+    def manually_disable_aa_profile(self):
+        """
+        Manually disable an apparmor profile.
+
+        If aa-profile-mode is set to disabled (default) this is required as the
+        template has been written but apparmor is yet unaware of the profile
+        and aa-disable aa-profile fails. Without this the profile would kick
+        into enforce mode on the next service restart.
+
+        """
+        profile_path = '/etc/apparmor.d'
+        disable_path = '/etc/apparmor.d/disable'
+        if not os.path.lexists(os.path.join(disable_path, self.aa_profile)):
+            os.symlink(os.path.join(profile_path, self.aa_profile),
+                       os.path.join(disable_path, self.aa_profile))
+
+    def setup_aa_profile(self):
+        """
+        Setup an apparmor profile.
+        The ctxt dictionary will contain the apparmor profile mode and
+        the apparmor profile name.
+        Makes calls out to aa-disable, aa-complain, or aa-enforce to setup
+        the apparmor profile.
+        """
+        self()
+        if not self.ctxt:
+            log("Not enabling apparmor Profile")
+            return
+        self.install_aa_utils()
+        cmd = ['aa-{}'.format(self.ctxt['aa-profile-mode'])]
+        cmd.append(self.ctxt['aa-profile'])
+        log("Setting up the apparmor profile for {} in {} mode."
+            "".format(self.ctxt['aa-profile'], self.ctxt['aa-profile-mode']))
+        try:
+            check_call(cmd)
+        except CalledProcessError as e:
+            # If aa-profile-mode is set to disabled (default) manual
+            # disabling is required as the template has been written but
+            # apparmor is yet unaware of the profile and aa-disable aa-profile
+            # fails. If aa-disable learns to read profile files first this can
+            # be removed.
+            if self.ctxt['aa-profile-mode'] == 'disable':
+                log("Manually disabling the apparmor profile for {}."
+                    "".format(self.ctxt['aa-profile']))
+                self.manually_disable_aa_profile()
+                return
+            status_set('blocked', "Apparmor profile {} failed to be set to {}."
+                                  "".format(self.ctxt['aa-profile'],
+                                            self.ctxt['aa-profile-mode']))
+            raise e

hooks/charmhelpers/contrib/openstack/utils.py

@@ -137,7 +137,7 @@ SWIFT_CODENAMES = OrderedDict([
     ('liberty',
         ['2.3.0', '2.4.0', '2.5.0']),
     ('mitaka',
-        ['2.5.0', '2.6.0']),
+        ['2.5.0', '2.6.0', '2.7.0']),
 ])
 
 # >= Liberty version->codename mapping
@@ -156,6 +156,7 @@ PACKAGE_CODENAMES = {
     ]),
     'keystone': OrderedDict([
         ('8.0', 'liberty'),
+        ('8.1', 'liberty'),
         ('9.0', 'mitaka'),
     ]),
     'horizon-common': OrderedDict([
@@ -1534,7 +1535,8 @@ def make_assess_status_func(*args, **kwargs):
     return _assess_status_func
 
 
-def pausable_restart_on_change(restart_map, stopstart=False):
+def pausable_restart_on_change(restart_map, stopstart=False,
+                               restart_functions=None):
     """A restart_on_change decorator that checks to see if the unit is
     paused. If it is paused then the decorated function doesn't fire.
 
@@ -1567,6 +1569,7 @@ def pausable_restart_on_change(restart_map, stopstart=False):
                 return f(*args, **kwargs)
             # otherwise, normal restart_on_change functionality
             return restart_on_change_helper(
-                (lambda: f(*args, **kwargs)), restart_map, stopstart)
+                (lambda: f(*args, **kwargs)), restart_map, stopstart,
+                restart_functions)
         return wrapped_f
     return wrap

hooks/charmhelpers/core/host.py

@@ -128,6 +128,13 @@ def service(action, service_name):
     return subprocess.call(cmd) == 0
 
 
+def systemv_services_running():
+    output = subprocess.check_output(
+        ['service', '--status-all'],
+        stderr=subprocess.STDOUT).decode('UTF-8')
+    return [row.split()[-1] for row in output.split('\n') if '[ + ]' in row]
+
+
 def service_running(service_name):
     """Determine whether a system service is running"""
     if init_is_systemd():
@@ -140,11 +147,15 @@ def service_running(service_name):
         except subprocess.CalledProcessError:
             return False
         else:
+            # This works for upstart scripts where the 'service' command
+            # returns a consistent string to represent running 'start/running'
             if ("start/running" in output or "is running" in output or
                     "up and running" in output):
                 return True
-            else:
+            # Check System V scripts init script return codes
-                return False
+            if service_name in systemv_services_running():
+                return True
+            return False
 
 
 def service_available(service_name):
@@ -412,7 +423,7 @@ class ChecksumError(ValueError):
     pass
 
 
-def restart_on_change(restart_map, stopstart=False):
+def restart_on_change(restart_map, stopstart=False, restart_functions=None):
     """Restart services based on configuration files changing
 
     This function is used a decorator, for example::
@@ -433,18 +444,22 @@ def restart_on_change(restart_map, stopstart=False):
 
     @param restart_map: {path_file_name: [service_name, ...]
     @param stopstart: DEFAULT false; whether to stop, start OR restart
+    @param restart_functions: nonstandard functions to use to restart services
+                              {svc: func, ...}
     @returns result from decorated function
     """
     def wrap(f):
         @functools.wraps(f)
         def wrapped_f(*args, **kwargs):
             return restart_on_change_helper(
-                (lambda: f(*args, **kwargs)), restart_map, stopstart)
+                (lambda: f(*args, **kwargs)), restart_map, stopstart,
+                restart_functions)
         return wrapped_f
     return wrap
 
 
-def restart_on_change_helper(lambda_f, restart_map, stopstart=False):
+def restart_on_change_helper(lambda_f, restart_map, stopstart=False,
+                             restart_functions=None):
     """Helper function to perform the restart_on_change function.
 
     This is provided for decorators to restart services if files described
@@ -453,8 +468,12 @@ def restart_on_change_helper(lambda_f, restart_map, stopstart=False):
     @param lambda_f: function to call.
     @param restart_map: {file: [service, ...]}
     @param stopstart: whether to stop, start or restart a service
+    @param restart_functions: nonstandard functions to use to restart services
+                              {svc: func, ...}
     @returns result of lambda_f()
     """
+    if restart_functions is None:
+        restart_functions = {}
     checksums = {path: path_hash(path) for path in restart_map}
     r = lambda_f()
     # create a list of lists of the services to restart
@@ -465,9 +484,12 @@ def restart_on_change_helper(lambda_f, restart_map, stopstart=False):
     services_list = list(OrderedDict.fromkeys(itertools.chain(*restarts)))
     if services_list:
         actions = ('stop', 'start') if stopstart else ('restart',)
-        for action in actions:
+        for service_name in services_list:
-            for service_name in services_list:
+            if service_name in restart_functions:
-                service(action, service_name)
+                restart_functions[service_name](service_name)
+            else:
+                for action in actions:
+                    service(action, service_name)
     return r
 
 

requirements.txt

@@ -1,6 +1,7 @@
 # The order of packages is significant, because pip processes them in the order
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
+pbr>=1.8.0,<1.9.0
 PyYAML>=3.1.0
 simplejson>=2.2.0
 netifaces>=0.10.4

tests/charmhelpers/contrib/amulet/utils.py

@@ -601,7 +601,7 @@ class AmuletUtils(object):
                 return ('Process name count mismatch.  expected, actual: {}, '
                         '{}'.format(len(expected), len(actual)))
 
-            for (e_proc_name, e_pids_length), (a_proc_name, a_pids) in \
+            for (e_proc_name, e_pids), (a_proc_name, a_pids) in \
                     zip(e_proc_names.items(), a_proc_names.items()):
                 if e_proc_name != a_proc_name:
                     return ('Process name mismatch.  expected, actual: {}, '
@@ -610,25 +610,31 @@ class AmuletUtils(object):
                 a_pids_length = len(a_pids)
                 fail_msg = ('PID count mismatch. {} ({}) expected, actual: '
                             '{}, {} ({})'.format(e_sentry_name, e_proc_name,
-                                                 e_pids_length, a_pids_length,
+                                                 e_pids, a_pids_length,
                                                  a_pids))
 
-                # If expected is not bool, ensure PID quantities match
+                # If expected is a list, ensure at least one PID quantity match
-                if not isinstance(e_pids_length, bool) and \
+                if isinstance(e_pids, list) and \
-                        a_pids_length != e_pids_length:
+                        a_pids_length not in e_pids:
+                    return fail_msg
+                # If expected is not bool and not list,
+                # ensure PID quantities match
+                elif not isinstance(e_pids, bool) and \
+                        not isinstance(e_pids, list) and \
+                        a_pids_length != e_pids:
                     return fail_msg
                 # If expected is bool True, ensure 1 or more PIDs exist
-                elif isinstance(e_pids_length, bool) and \
+                elif isinstance(e_pids, bool) and \
-                        e_pids_length is True and a_pids_length < 1:
+                        e_pids is True and a_pids_length < 1:
                     return fail_msg
                 # If expected is bool False, ensure 0 PIDs exist
-                elif isinstance(e_pids_length, bool) and \
+                elif isinstance(e_pids, bool) and \
-                        e_pids_length is False and a_pids_length != 0:
+                        e_pids is False and a_pids_length != 0:
                     return fail_msg
                 else:
                     self.log.debug('PID check OK: {} {} {}: '
                                    '{}'.format(e_sentry_name, e_proc_name,
-                                               e_pids_length, a_pids))
+                                               e_pids, a_pids))
         return None
 
     def validate_list_of_identical_dicts(self, list_of_dicts):

unit_tests/test_cluster_hooks.py

@@ -79,15 +79,16 @@ class TestClusterHooks(CharmTestCase):
         hooks.hooks.execute(['hooks/cluster-relation-changed'])
         ex = [
             call('stop', 'cinder-api'),
-            call('stop', 'cinder-volume'),
-            call('stop', 'cinder-scheduler'),
-            call('stop', 'haproxy'),
-            call('stop', 'apache2'),
             call('start', 'cinder-api'),
+            call('stop', 'cinder-volume'),
             call('start', 'cinder-volume'),
+            call('stop', 'cinder-scheduler'),
             call('start', 'cinder-scheduler'),
+            call('stop', 'haproxy'),
             call('start', 'haproxy'),
-            call('start', 'apache2')]
+            call('stop', 'apache2'),
+            call('start', 'apache2'),
+        ]
         self.assertEquals(ex, service.call_args_list)
 
     @patch.object(hooks, 'check_db_initialised', lambda *args, **kwargs: None)